From ae2494caa7f54b07024446c3b095ebc01305c0d2 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 10 May 2024 21:22:54 -0700 Subject: [PATCH 001/199] autorepair v2 framework --- .../cassandra/pages/operating/metrics.adoc | 50 ++ .../org/apache/cassandra/config/Config.java | 5 + .../cassandra/config/DatabaseDescriptor.java | 17 +- .../apache/cassandra/config/DurationSpec.java | 3 +- .../cassandra/cql3/UntypedResultSet.java | 7 + .../statements/ModificationStatement.java | 2 +- .../statements/schema/TableAttributes.java | 8 + .../cassandra/metrics/AutoRepairMetrics.java | 165 ++++ .../metrics/AutoRepairMetricsManager.java | 34 + .../repair/RepairMessageVerbHandler.java | 7 + .../repair/autorepair/AutoRepair.java | 401 +++++++++ .../repair/autorepair/AutoRepairConfig.java | 408 +++++++++ .../repair/autorepair/AutoRepairKeyspace.java | 84 ++ .../repair/autorepair/AutoRepairState.java | 349 ++++++++ .../repair/autorepair/AutoRepairUtils.java | 806 ++++++++++++++++++ .../DefaultAutoRepairTokenSplitter.java | 89 ++ .../IAutoRepairTokenRangeSplitter.java | 31 + .../cassandra/schema/AutoRepairParams.java | 145 ++++ .../cassandra/schema/SchemaKeyspace.java | 12 +- .../apache/cassandra/schema/TableParams.java | 89 +- .../service/ActiveRepairService.java | 32 + .../cassandra/service/AutoRepairService.java | 147 ++++ .../service/AutoRepairServiceMBean.java | 60 ++ .../cassandra/service/StorageService.java | 21 +- .../org/apache/cassandra/tools/NodeProbe.java | 84 ++ .../org/apache/cassandra/tools/NodeTool.java | 2 + .../tools/nodetool/GetAutoRepairConfig.java | 77 ++ .../tools/nodetool/SetAutoRepairConfig.java | 156 ++++ .../apache/cassandra/utils/FBUtilities.java | 8 + test/unit/org/apache/cassandra/Util.java | 13 + .../config/YamlConfigurationLoaderTest.java | 31 +- .../autorepair/AutoRepairConfigTest.java | 453 ++++++++++ .../autorepair/AutoRepairKeyspaceTest.java | 64 ++ .../AutoRepairParameterizedTest.java | 581 +++++++++++++ .../AutoRepairStateFactoryTest.java | 52 ++ .../autorepair/AutoRepairStateTest.java | 382 +++++++++ .../repair/autorepair/AutoRepairTest.java | 119 +++ .../autorepair/AutoRepairUtilsTest.java | 476 +++++++++++ .../service/ActiveRepairServiceTest.java | 44 +- 39 files changed, 5477 insertions(+), 37 deletions(-) create mode 100644 src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java create mode 100644 src/java/org/apache/cassandra/metrics/AutoRepairMetricsManager.java create mode 100644 src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java create mode 100644 src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java create mode 100644 src/java/org/apache/cassandra/repair/autorepair/AutoRepairKeyspace.java create mode 100644 src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java create mode 100644 src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java create mode 100644 src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java create mode 100644 src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java create mode 100644 src/java/org/apache/cassandra/schema/AutoRepairParams.java create mode 100644 src/java/org/apache/cassandra/service/AutoRepairService.java create mode 100644 src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java create mode 100644 src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java create mode 100644 src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java diff --git a/doc/modules/cassandra/pages/operating/metrics.adoc b/doc/modules/cassandra/pages/operating/metrics.adoc index 1eb8156dc20a..68163e627a10 100644 --- a/doc/modules/cassandra/pages/operating/metrics.adoc +++ b/doc/modules/cassandra/pages/operating/metrics.adoc @@ -945,6 +945,56 @@ partitions processed per logged batch partitions processed per unlogged batch |=== +== Automated Repair Metrics + +Metrics specifc to automated repair. + +Reported name format: + +*Metric Name*:: +`org.apache.cassandra.metrics.AutoRepairMetrics.` +*JMX MBean*:: +`org.apache.cassandra.metrics:type=AutoRepairMetrics name=` + +[cols=",,",options="header",] +|=== +|Name |Type |Description +|RepairsInProgress |Gauge |Repair is in progress +on the node + +|NodeRepairTimeInSec |Gauge |Time taken to repair +the node + +|ClusterRepairTimeInSec |Gauge |Time taken to repair +the entire Cassandra cluster + +|SkippedTablesCount |Gauge |Number of tables skipped +on the node + +|LongestUnrepairedSec |Gauge |Time since the last repair +ran on the node + +|FailedTablesCount |Gauge |Number of tables encountered +failure during repair on the node + +|TotalMVTablesConsideredForRepair |Gauge |Number of materialized +views considered on the node + +|TotalDisabledRepairTables |Gauge |Number of tables on which +the automated repair has been disabled on the node + +|RepairTurnMyTurn |Counter |Represents the node's turn to repair + +|RepairTurnMyTurnDueToPriority |Counter |Represents the node's turn to repair +due to priority set in the automated repair + +|RepairTurnMyTurnForceRepair |Counter |Represents the node's turn to repair +due to force repair set in the automated repair + + +|=== + + == JVM Metrics JVM metrics such as memory and garbage collection statistics can either diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index c862861b067e..81c5e9e8e8f2 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -34,6 +34,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -330,6 +331,8 @@ public MemtableOptions() public volatile int concurrent_materialized_view_builders = 1; public volatile int reject_repair_compaction_threshold = Integer.MAX_VALUE; + public volatile double incremental_repair_disk_headroom_reject_ratio = 0.2; // at least 20% of disk must be unused to run incremental repair + /** * @deprecated retry support removed on CASSANDRA-10992 */ @@ -864,6 +867,8 @@ public static void setClientMode(boolean clientMode) public volatile DurationSpec.LongNanosecondsBound repair_state_expires = new DurationSpec.LongNanosecondsBound("3d"); public volatile int repair_state_size = 100_000; + public volatile AutoRepairConfig auto_repair = new AutoRepairConfig(); + /** * The variants of paxos implementation and semantics supported by Cassandra. */ diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index eac1b42037cb..94e7771cc4eb 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -46,6 +46,7 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.util.concurrent.RateLimiter; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -4516,4 +4517,18 @@ public static void setCQLStartTime(Config.CQLStartTime value) { conf.cql_start_time = value; } -} \ No newline at end of file + public static AutoRepairConfig getAutoRepairConfig() + { + return conf.auto_repair; + } + + public static double getIncrementalRepairDiskHeadroomRejectRatio() + { + return conf.incremental_repair_disk_headroom_reject_ratio; + } + + public static void setIncrementalRepairDiskHeadroomRejectRatio(double value) + { + conf.incremental_repair_disk_headroom_reject_ratio = value; + } +} diff --git a/src/java/org/apache/cassandra/config/DurationSpec.java b/src/java/org/apache/cassandra/config/DurationSpec.java index 10d56c23eb71..0c2b32626051 100644 --- a/src/java/org/apache/cassandra/config/DurationSpec.java +++ b/src/java/org/apache/cassandra/config/DurationSpec.java @@ -17,6 +17,7 @@ */ package org.apache.cassandra.config; +import java.io.Serializable; import java.util.Arrays; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -39,7 +40,7 @@ * users the opportunity to be able to provide config with a unit of their choice in cassandra.yaml as per the available * options. (CASSANDRA-15234) */ -public abstract class DurationSpec +public abstract class DurationSpec implements Serializable { /** * The Regexp used to parse the duration provided as String. diff --git a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java index 14b7f76d6bef..5862444e37f8 100644 --- a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java +++ b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java @@ -432,6 +432,13 @@ public long getLong(String column) return LongType.instance.compose(data.get(column)); } + // this function will return the default value if the row doesn't have that column or the column data is null + // This function is used to avoid the nullpointerexception + public long getLong(String column, long ifNull) { + ByteBuffer bytes = data.get(column); + return bytes == null ? ifNull : LongType.instance.compose(bytes); + } + public Set getSet(String column, AbstractType type) { ByteBuffer raw = data.get(column); diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java index dabec10b6a48..3b8e5255f3ca 100644 --- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java @@ -81,7 +81,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa public static final String CUSTOM_EXPRESSIONS_NOT_ALLOWED = "Custom index expressions cannot be used in WHERE clauses for UPDATE or DELETE statements"; - private static final ColumnIdentifier CAS_RESULT_COLUMN = new ColumnIdentifier("[applied]", false); + public static final ColumnIdentifier CAS_RESULT_COLUMN = new ColumnIdentifier("[applied]", false); protected final StatementType type; diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index 829a53a911ef..1e290869bd73 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -26,6 +26,8 @@ import org.apache.cassandra.cql3.statements.PropertyDefinitions; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.schema.AutoRepairParams; import org.apache.cassandra.schema.CachingParams; import org.apache.cassandra.schema.CompactionParams; import org.apache.cassandra.schema.CompressionParams; @@ -155,6 +157,12 @@ private TableParams build(TableParams.Builder builder) if (hasOption(Option.READ_REPAIR)) builder.readRepair(ReadRepairStrategy.fromString(getString(Option.READ_REPAIR))); + if (hasOption(Option.AUTOMATED_REPAIR_FULL)) + builder.automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.full, getMap(Option.AUTOMATED_REPAIR_FULL))); + + if (hasOption(Option.AUTOMATED_REPAIR_INCREMENTAL)) + builder.automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.incremental, getMap(Option.AUTOMATED_REPAIR_INCREMENTAL))); + return builder.build(); } diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java new file mode 100644 index 000000000000..cd46c462fb66 --- /dev/null +++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.metrics; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.Gauge; +import com.google.common.annotations.VisibleForTesting; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils; +import org.apache.cassandra.repair.autorepair.AutoRepair; + +import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; + +/** + * Metrics related to AutoRepair. + */ +public class AutoRepairMetrics +{ + public Gauge repairsInProgress; + public Gauge nodeRepairTimeInSec; + public Gauge clusterRepairTimeInSec; + public Gauge skippedTablesCount; + public Gauge longestUnrepairedSec; + public Gauge failedTablesCount; + public Counter repairTurnMyTurn; + public Counter repairTurnMyTurnDueToPriority; + public Counter repairTurnMyTurnForceRepair; + public Gauge totalMVTablesConsideredForRepair; + public Gauge totalDisabledRepairTables; + + public AutoRepairMetrics(RepairType repairType) + { + AutoRepairMetricsFactory factory = new AutoRepairMetricsFactory(repairType); + + repairsInProgress = Metrics.register(factory.createMetricName("RepairsInProgress"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).isRepairInProgress() ? 1 : 0; + } + }); + + nodeRepairTimeInSec = Metrics.register(factory.createMetricName("NodeRepairTimeInSec"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).getNodeRepairTimeInSec(); + } + }); + + clusterRepairTimeInSec = Metrics.register(factory.createMetricName("ClusterRepairTimeInSec"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).getClusterRepairTimeInSec(); + } + }); + + skippedTablesCount = Metrics.register(factory.createMetricName("SkippedTablesCount"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).getRepairSkippedTablesCount(); + } + }); + + longestUnrepairedSec = Metrics.register(factory.createMetricName("LongestUnrepairedSec"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).getLongestUnrepairedSec(); + } + }); + + failedTablesCount = Metrics.register(factory.createMetricName("FailedTablesCount"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).getRepairFailedTablesCount(); + } + }); + + repairTurnMyTurn = Metrics.counter(factory.createMetricName("RepairTurnMyTurn")); + repairTurnMyTurnDueToPriority = Metrics.counter(factory.createMetricName("RepairTurnMyTurnDueToPriority")); + repairTurnMyTurnForceRepair = Metrics.counter(factory.createMetricName("RepairTurnMyTurnForceRepair")); + + totalMVTablesConsideredForRepair = Metrics.register(factory.createMetricName("TotalMVTablesConsideredForRepair"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).getTotalMVTablesConsideredForRepair(); + } + }); + + totalDisabledRepairTables = Metrics.register(factory.createMetricName("TotalDisabledRepairTables"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).getTotalDisabledTablesRepairCount(); + } + }); + } + + public void recordTurn(AutoRepairUtils.RepairTurn turn) + { + switch (turn) + { + case MY_TURN: + repairTurnMyTurn.inc(); + break; + case MY_TURN_FORCE_REPAIR: + repairTurnMyTurnForceRepair.inc(); + break; + case MY_TURN_DUE_TO_PRIORITY: + repairTurnMyTurnDueToPriority.inc(); + break; + default: + throw new RuntimeException(String.format("Unrecoginized turn: %s", turn.name())); + } + } + + @VisibleForTesting + protected static class AutoRepairMetricsFactory implements MetricNameFactory + { + private static final String TYPE = "AutoRepair"; + @VisibleForTesting + protected final String repairType; + + protected AutoRepairMetricsFactory(RepairType repairType) + { + this.repairType = repairType.toString().toLowerCase(); + } + + @Override + public CassandraMetricsRegistry.MetricName createMetricName(String metricName) + { + StringBuilder mbeanName = new StringBuilder(); + mbeanName.append(DefaultNameFactory.GROUP_NAME).append(':'); + mbeanName.append("type=").append(TYPE); + mbeanName.append(",name=").append(metricName); + mbeanName.append(",repairType=").append(repairType); + + StringBuilder scope = new StringBuilder(); + scope.append("repairType=").append(repairType); + + return new CassandraMetricsRegistry.MetricName(DefaultNameFactory.GROUP_NAME, TYPE.toLowerCase(), + metricName, scope.toString(), mbeanName.toString()); + } + } +} diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetricsManager.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetricsManager.java new file mode 100644 index 000000000000..e293945c9846 --- /dev/null +++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetricsManager.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.metrics; + +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class AutoRepairMetricsManager +{ + private static final Map metrics = new ConcurrentHashMap<>(); + + public static AutoRepairMetrics getMetrics(RepairType repairType) + { + return metrics.computeIfAbsent(repairType, k -> new AutoRepairMetrics(repairType)); + } +} diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java index 12fa6c341cf0..734cb67f95d7 100644 --- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java +++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java @@ -77,6 +77,13 @@ public void doVerb(final Message message) logger.debug("Duplicate prepare message found for {}", state.id); return; } + if (!ActiveRepairService.verifyDiskHeadroomThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind, prepareMessage.isIncremental)) + { + // error is logged in verifyDiskHeadroomThreshold + state.phase.fail("Not enough disk headroom to perform incremental repair"); + sendFailureResponse(message); + return; + } if (!ActiveRepairService.verifyCompactionsPendingThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind)) { // error is logged in verifyCompactionsPendingThreshold diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java new file mode 100644 index 000000000000..9fbc208f95ab --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -0,0 +1,401 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + +import org.apache.cassandra.repair.RepairRunnable; +import org.apache.cassandra.utils.Pair; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.concurrent.NamedThreadFactory; +import org.apache.cassandra.concurrent.ScheduledExecutorPlus; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.Tables; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn; + +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_DUE_TO_PRIORITY; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_FORCE_REPAIR; + +public class AutoRepair +{ + // Initial delay for repair session to start after setup + final static long INITIAL_REPAIR_DELAY_SEC = 30; + + private static final Logger logger = LoggerFactory.getLogger(AutoRepair.class); + + @VisibleForTesting + protected static Supplier timeFunc = System::currentTimeMillis; + + public static AutoRepair instance = new AutoRepair(); + + @VisibleForTesting + protected final Map repairExecutors; + @VisibleForTesting + protected final Map repairStates; + + protected final Map tokenRangeSplitters = new EnumMap<>(AutoRepairConfig.RepairType.class); + + + @VisibleForTesting + protected AutoRepair() + { + AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); + repairExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class); + repairStates = new EnumMap<>(AutoRepairConfig.RepairType.class); + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + repairExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-Repair-" + repairType, Thread.NORM_PRIORITY)); + repairStates.put(repairType, AutoRepairConfig.RepairType.getAutoRepairState(repairType)); + tokenRangeSplitters.put(repairType, FBUtilities.newAutoRepairTokenRangeSplitter(config.getTokenRangeSplitter(repairType))); + } + } + + public void setup() + { + verifyIsSafeToEnable(); + + AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); + AutoRepairService.setup(); + AutoRepairUtils.setup(); + + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + repairExecutors.get(repairType).scheduleWithFixedDelay( + () -> repair(repairType, 60000), + INITIAL_REPAIR_DELAY_SEC, + config.getRepairCheckInterval().toSeconds(), + TimeUnit.SECONDS); + } + } + + @VisibleForTesting + protected void verifyIsSafeToEnable() + { + AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); + if (config.isAutoRepairEnabled(AutoRepairConfig.RepairType.incremental) && + (DatabaseDescriptor.getMaterializedViewsEnabled() || DatabaseDescriptor.isCDCEnabled())) + throw new ConfigurationException("Cannot enable incremental repair with materialized views or CDC enabled"); + } + + // repairAsync runs a repair session of the given type asynchronously. + public void repairAsync(AutoRepairConfig.RepairType repairType, long millisToWait) + { + repairExecutors.get(repairType).submit(() -> repair(repairType, millisToWait)); + } + + // repair runs a repair session of the given type synchronously. + public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + if (!config.isAutoRepairEnabled(repairType)) + { + logger.debug("Auto-repair is disabled for repair type {}", repairType); + return; + } + + + AutoRepairState repairState = repairStates.get(repairType); + + try + { + String localDC = DatabaseDescriptor.getLocalDataCenter(); + if (config.getIgnoreDCs(repairType).contains(localDC)) + { + logger.info("Not running repair as this node belongs to datacenter {}", localDC); + return; + } + + // refresh the longest unrepaired node + repairState.setLongestUnrepairedNode(AutoRepairUtils.getHostWithLongestUnrepairTime(repairType)); + + //consistency level to use for local query + UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); + RepairTurn turn = AutoRepairUtils.myTurnToRunRepair(repairType, myId); + if (turn == MY_TURN || turn == MY_TURN_DUE_TO_PRIORITY || turn == MY_TURN_FORCE_REPAIR) + { + repairState.recordTurn(turn); + // For normal auto repair, we will use primary range only repairs (Repair with -pr option). + // For some cases, we may set the auto_repair_primary_token_range_only flag to false then we will do repair + // without -pr. We may also do force repair for certain node that we want to repair all the data on one node + // When doing force repair, we want to repair without -pr. + boolean primaryRangeOnly = config.getRepairPrimaryTokenRangeOnly(repairType) + && turn != MY_TURN_FORCE_REPAIR; + repairState.setTotalTablesConsideredForRepair(0); + if (repairState.getLastRepairTime() != 0) + { + /** check if it is too soon to run repair. one of the reason we + * should not run frequent repair is because repair triggers + * memtable flush + */ + long timeElapsedSinceLastRepair = TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - repairState.getLastRepairTime()); + if (timeElapsedSinceLastRepair < config.getRepairMinInterval(repairType).toSeconds()) + { + logger.info("Too soon to run repair, last repair was done {} seconds ago", + timeElapsedSinceLastRepair); + return; + } + } + + long startTime = timeFunc.get(); + logger.info("My host id: {}, my turn to run repair...repair primary-ranges only? {}", myId, + config.getRepairPrimaryTokenRangeOnly(repairType)); + AutoRepairUtils.updateStartAutoRepairHistory(repairType, myId, timeFunc.get(), turn); + + repairState.setRepairKeyspaceCount(0); + repairState.setRepairTableSuccessCount(0); + repairState.setRepairFailedTablesCount(0); + repairState.setRepairSkippedTablesCount(0); + repairState.setRepairInProgress(true); + repairState.setTotalMVTablesConsideredForRepair(0); + for (Keyspace keyspace : Keyspace.all()) + { + Tables tables = keyspace.getMetadata().tables; + Iterator iter = tables.iterator(); + String keyspaceName = keyspace.getName(); + if (!AutoRepairUtils.checkNodeContainsKeyspaceReplica(keyspace)) + { + continue; + } + + repairState.setRepairKeyspaceCount(repairState.getRepairKeyspaceCount() + 1); + List tablesToBeRepaired = new ArrayList<>(); + while (iter.hasNext()) + { + repairState.setTotalTablesConsideredForRepair(repairState.getTotalTablesConsideredForRepair() + 1); + TableMetadata tableMetadata = iter.next(); + String tableName = tableMetadata.name; + tablesToBeRepaired.add(tableName); + + // See if we should repair MVs as well that are associated with this given table + List mvs = AutoRepairUtils.getAllMVs(repairType, keyspace, tableMetadata); + if (!mvs.isEmpty()) + { + tablesToBeRepaired.addAll(mvs); + repairState.setTotalMVTablesConsideredForRepair(repairState.getTotalMVTablesConsideredForRepair() + mvs.size()); + } + } + + for (String tableName : tablesToBeRepaired) + { + try + { + ColumnFamilyStore columnFamilyStore = keyspace.getColumnFamilyStore(tableName); + if (!columnFamilyStore.metadata().params.automatedRepair.get(repairType).repairEnabled()) + { + logger.info("Repair is disabled for keyspace {} for tables: {}", keyspaceName, tableName); + repairState.setTotalDisabledTablesRepairCount(repairState.getTotalDisabledTablesRepairCount() + 1); + continue; + } + // this is done to make autorepair safe as running repair on table with more sstables + // may have its own challenges + int size = columnFamilyStore.getLiveSSTables().size(); + if (size > config.getRepairSSTableCountHigherThreshold(repairType)) + { + logger.info("Too many SSTables for repair, not doing repair on table {}.{} " + + "totalSSTables {}", keyspaceName, tableName, columnFamilyStore.getLiveSSTables().size()); + repairState.setRepairSkippedTablesCount(repairState.getRepairSkippedTablesCount() + 1); + continue; + } + + if (config.getRepairByKeyspace(repairType)) + { + logger.info("Repair keyspace {} for tables: {}", keyspaceName, tablesToBeRepaired); + } + else + { + logger.info("Repair table {}.{}", keyspaceName, tableName); + } + long tableStartTime = timeFunc.get(); + boolean repairSuccess = true; + Set> ranges = new HashSet<>(); + List> subRangesToBeRepaired = tokenRangeSplitters.get(repairType).getRange(repairType, primaryRangeOnly, keyspaceName, tableName); + int totalSubRanges = subRangesToBeRepaired.size(); + int totalProcessedSubRanges = 0; + for (Pair token : subRangesToBeRepaired) + { + if (!config.isAutoRepairEnabled(repairType)) + { + logger.error("Auto-repair for type {} is disabled hence not running repair", repairType); + repairState.setRepairInProgress(false); + return; + } + + if (config.getRepairByKeyspace(repairType)) + { + if (AutoRepairUtils.keyspaceMaxRepairTimeExceeded(repairType, tableStartTime, tablesToBeRepaired.size())) + { + repairState.setRepairSkippedTablesCount(repairState.getRepairSkippedTablesCount() + tablesToBeRepaired.size()); + logger.info("Keyspace took too much time to repair hence skipping it {}", + keyspaceName); + break; + } + } + else + { + if (AutoRepairUtils.tableMaxRepairTimeExceeded(repairType, tableStartTime)) + { + repairState.setRepairSkippedTablesCount(repairState.getRepairSkippedTablesCount() + 1); + logger.info("Table took too much time to repair hence skipping it {}.{}", + keyspaceName, tableName); + break; + } + } + Token childStartToken = token.left; + Token childEndToken = token.right; + logger.debug("Current Token Left side {}, right side {}", childStartToken + .toString(), childEndToken.toString()); + + ranges.add(new Range<>(childStartToken, childEndToken)); + totalProcessedSubRanges++; + if ((totalProcessedSubRanges % config.getRepairThreads(repairType) == 0) || + (totalProcessedSubRanges == totalSubRanges)) + { + RepairRunnable task = repairState.getRepairRunnable(keyspaceName, + config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : ImmutableList.of(tableName), + ranges, primaryRangeOnly); + repairState.resetWaitCondition(); + new Thread(NamedThreadFactory.createAnonymousThread(new FutureTask<>(task, null))).start(); + try + { + repairState.waitForRepairToComplete(); + } + catch (InterruptedException e) + { + logger.error("Exception in cond await:", e); + } + + //check repair status + if (repairState.isSuccess()) + { + logger.info("Repair completed for range {}-{} for {}.{}, total subranges: {}," + + "processed subranges: {}", childStartToken, childEndToken, + keyspaceName, config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, totalSubRanges, totalProcessedSubRanges); + } + else + { + repairSuccess = false; + //in future we can add retry, etc. + logger.info("Repair failed for range {}-{} for {}.{} total subranges: {}," + + "processed subranges: {}", childStartToken, childEndToken, + keyspaceName, config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, totalSubRanges, totalProcessedSubRanges); + } + ranges.clear(); + } + } + int touchedTables = config.getRepairByKeyspace(repairType) ? tablesToBeRepaired.size() : 1; + if (repairSuccess) + { + repairState.setRepairTableSuccessCount(repairState.getRepairTableSuccessCount() + touchedTables); + } + else + { + repairState.setRepairFailedTablesCount(repairState.getRepairFailedTablesCount() + touchedTables); + } + if (config.getRepairByKeyspace(repairType)) + { + logger.info("Repair completed for keyspace {}, tables: {}", keyspaceName, tablesToBeRepaired); + break; + } + else + { + logger.info("Repair completed for {}.{}", keyspaceName, tableName); + } + } + catch (Exception e) + { + logger.error("Exception while repairing keyspace {}:", keyspaceName, e); + } + } + } + + //if it was due to priority then remove it now + if (turn == MY_TURN_DUE_TO_PRIORITY) + { + logger.info("Remove current host from priority list"); + AutoRepairUtils.removePriorityStatus(repairType, myId); + } + + repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - startTime)); + long timeInHours = TimeUnit.SECONDS.toHours(repairState.getNodeRepairTimeInSec()); + logger.info("Local {} repair time {} hour(s), stats: repairKeyspaceCount {}, " + + "repairTableSuccessCount {}, repairTableFailureCount {}, " + + "repairTableSkipCount {}", repairType, timeInHours, repairState.getRepairKeyspaceCount(), + repairState.getRepairTableSuccessCount(), repairState.getRepairFailedTablesCount(), + repairState.getRepairSkippedTablesCount()); + if (repairState.getLastRepairTime() != 0) + { + repairState.setClusterRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - + repairState.getLastRepairTime())); + logger.info("Cluster repair time for repair type {}: {} day(s)", repairType, + TimeUnit.SECONDS.toDays(repairState.getClusterRepairTimeInSec())); + } + repairState.setLastRepairTime(timeFunc.get()); + if (timeInHours == 0 && millisToWait > 0) + { + //If repair finished quickly, happens for an empty instance, in such case + //wait for a minute so that the JMX metrics can detect the repairInProgress + logger.info("Wait for {} milliseconds for repair type {}.", millisToWait, repairType); + Thread.sleep(millisToWait); + } + repairState.setRepairInProgress(false); + AutoRepairUtils.updateFinishAutoRepairHistory(repairType, myId, timeFunc.get()); + } + else + { + logger.info("Waiting for my turn..."); + } + } + catch (Exception e) + { + logger.error("Exception in autorepair:", e); + } + } + + public AutoRepairState getRepairState(AutoRepairConfig.RepairType repairType) + { + return repairStates.get(repairType); + } +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java new file mode 100644 index 000000000000..bd20c1d3c548 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -0,0 +1,408 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.exceptions.ConfigurationException; + +public class AutoRepairConfig implements Serializable +{ + // enable/disable auto repair globally, overrides all other settings. Cannot be modified dynamically. + public final Boolean enabled; + // the interval in seconds between checks for eligible repair operations. Cannot be modified dynamically. + public final DurationSpec.IntSecondsBound repair_check_interval = new DurationSpec.IntSecondsBound("5m"); + // configures how long repair history is kept for a replaced node + public volatile DurationSpec.IntSecondsBound history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound("2h"); + // global_settings overides Options.defaultOptions for all repair types + public volatile Options global_settings; + + public enum RepairType + { + full, + incremental; + + public static AutoRepairState getAutoRepairState(RepairType repairType) + { + switch (repairType) + { + case full: + return new FullRepairState(); + case incremental: + return new IncrementalRepairState(); + } + + throw new IllegalArgumentException("Invalid repair type: " + repairType); + } + } + + // repair_type_overrides overrides the global_settings for a specific repair type + public volatile Map repair_type_overrides = new EnumMap<>(RepairType.class); + + public AutoRepairConfig() + { + this(false); + } + + public AutoRepairConfig(boolean enabled) + { + this.enabled = enabled; + global_settings = Options.getDefaultOptions(); + for (RepairType type : RepairType.values()) + { + repair_type_overrides.put(type, new Options()); + } + } + + public DurationSpec.IntSecondsBound getRepairCheckInterval() + { + return repair_check_interval; + } + + public boolean isAutoRepairSchedulingEnabled() + { + return enabled; + } + + public DurationSpec.IntSecondsBound getAutoRepairHistoryClearDeleteHostsBufferInterval() + { + return history_clear_delete_hosts_buffer_interval; + } + + public void setAutoRepairHistoryClearDeleteHostsBufferInterval(String duration) + { + history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound(duration); + } + + public boolean isAutoRepairEnabled(RepairType repairType) + { + return enabled && applyOverrides(repairType, opt -> opt.enabled); + } + + public void setAutoRepairEnabled(RepairType repairType, boolean enabled) + { + if (enabled && repairType == RepairType.incremental && + (DatabaseDescriptor.getMaterializedViewsEnabled() || DatabaseDescriptor.isCDCEnabled())) + throw new ConfigurationException("Cannot enable incremental repair with materialized views or CDC enabled"); + + ensureOverrides(repairType); + repair_type_overrides.get(repairType).enabled = enabled; + } + + public void setRepairByKeyspace(RepairType repairType, boolean repairByKeyspace) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).repair_by_keyspace = repairByKeyspace; + } + + public boolean getRepairByKeyspace(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.repair_by_keyspace); + } + + public int getRepairThreads(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.number_of_repair_threads); + } + + public void setRepairThreads(RepairType repairType, int repairThreads) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).number_of_repair_threads = repairThreads; + } + + public int getRepairSubRangeNum(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.number_of_subranges); + } + + public void setRepairSubRangeNum(RepairType repairType, int repairSubRanges) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).number_of_subranges = repairSubRanges; + } + + public DurationSpec.IntSecondsBound getRepairMinInterval(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.min_repair_interval); + } + + public void setRepairMinInterval(RepairType repairType, String minRepairInterval) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).min_repair_interval = new DurationSpec.IntSecondsBound(minRepairInterval); + } + + public int getRepairSSTableCountHigherThreshold(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.sstable_upper_threshold); + } + + public void setRepairSSTableCountHigherThreshold(RepairType repairType, int sstableHigherThreshold) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).sstable_upper_threshold = sstableHigherThreshold; + } + + public DurationSpec.IntSecondsBound getAutoRepairTableMaxRepairTime(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.table_max_repair_time); + } + + public void setAutoRepairTableMaxRepairTime(RepairType repairType, String autoRepairTableMaxRepairTime) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).table_max_repair_time = new DurationSpec.IntSecondsBound(autoRepairTableMaxRepairTime); + } + + public Set getIgnoreDCs(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.ignore_dcs); + } + + public void setIgnoreDCs(RepairType repairType, Set ignoreDCs) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).ignore_dcs = ignoreDCs; + } + + public boolean getRepairPrimaryTokenRangeOnly(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.repair_primary_token_range_only); + } + + public void setRepairPrimaryTokenRangeOnly(RepairType repairType, boolean primaryTokenRangeOnly) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).repair_primary_token_range_only = primaryTokenRangeOnly; + } + + public int getParallelRepairPercentageInGroup(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.parallel_repair_percentage_in_group); + } + + public void setParallelRepairPercentageInGroup(RepairType repairType, int percentageInGroup) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).parallel_repair_percentage_in_group = percentageInGroup; + } + + public int getParallelRepairCountInGroup(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.parallel_repair_count_in_group); + } + + public void setParallelRepairCountInGroup(RepairType repairType, int countInGroup) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).parallel_repair_count_in_group = countInGroup; + } + + public boolean getMVRepairEnabled(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.mv_repair_enabled); + } + + public void setMVRepairEnabled(RepairType repairType, boolean enabled) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).mv_repair_enabled = enabled; + } + + public void setForceRepairNewNode(RepairType repairType, boolean forceRepairNewNode) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).force_repair_new_node = forceRepairNewNode; + } + + public boolean getForceRepairNewNode(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.force_repair_new_node); + } + + public String getTokenRangeSplitter(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.token_range_splitter); + } + + // Options configures auto-repair behavior for a given repair type. + // All fields can be modified dynamically. + public static class Options implements Serializable + { + // The separator separating different DCs in repair_dc_groups + public static final String DC_GROUP_SEPARATOR = "\\|"; + + // defaultOptions defines the default auto-repair behavior when no overrides are defined + @VisibleForTesting + protected static final Options defaultOptions = getDefaultOptions(); + + public Options() + { + } + + @VisibleForTesting + protected static Options getDefaultOptions() + { + Options opts = new Options(); + + opts.enabled = false; + opts.repair_by_keyspace = false; + opts.number_of_subranges = 1; + opts.number_of_repair_threads = 1; + opts.parallel_repair_count_in_group = 1; + opts.parallel_repair_percentage_in_group = 0; + opts.sstable_upper_threshold = 10000; + opts.min_repair_interval = new DurationSpec.IntSecondsBound("24h"); + opts.ignore_dcs = new HashSet<>(); + opts.repair_primary_token_range_only = true; + opts.force_repair_new_node = false; + opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h"); + opts.mv_repair_enabled = true; + opts.token_range_splitter = DefaultAutoRepairTokenSplitter.class.getName(); + + return opts; + } + + // enable/disable auto repair for the given repair type + public volatile Boolean enabled; + // auto repair is default repair table by table, if this is enabled, the framework will repair all the tables in a keyspace in one go. + public volatile Boolean repair_by_keyspace; + // the number of subranges to split each to-be-repaired token range into, + // the higher this number, the smaller the repair sessions will be + // How many subranges to divide one range into? The default is 1. + // If you are using v-node, say 256, then the repair will always go one v-node range at a time, this parameter, additionally, will let us further subdivide a given v-node range into sub-ranges. + // With the value “1” and v-nodes of 256, a given table on a node will undergo the repair 256 times. But with a value “2,” the same table on a node will undergo a repair 512 times because every v-node range will be further divided by two. + // If you do not use v-nodes or the number of v-nodes is pretty small, say 8, setting this value to a higher number, say 16, will be useful to repair on a smaller range, and the chance of succeeding is higher. + public volatile Integer number_of_subranges; + // the number of repair threads to run for a given invoked Repair Job. + // Once the scheduler schedules one Job, then howmany threads to use inside that job will be controlled through this parameter. + // This is similar to -j for repair options for the nodetool repair command. + public volatile Integer number_of_repair_threads; + // the number of repair sessions that can run in parallel in a single group + // The number of nodes running repair parallelly. If parallelrepaircount is set, it will choose the larger value of the two. The default is 3. + // This configuration controls how many nodes would run repair in parallel. + // The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. + // If one or more node(s) finish repair, then the framework automatically picks up the next candidate and ensures the maximum number of nodes running repair do not exceed “3”. + public volatile Integer parallel_repair_count_in_group; + // the number of repair sessions that can run in parallel in a single groupas a percentage + // of the total number of nodes in the group [0,100] + // The percentage of nodes in the cluster that run repair parallelly. If parallelrepaircount is set, it will choose the larger value of the two. + //The problem with a fixed number of nodes (the above property) is that in a large-scale environment, + // the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. + //The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. + // So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. + // Extremely fewer manual interventions as it will rarely violate the repair SLA for customers + public volatile Integer parallel_repair_percentage_in_group; + // the upper threshold of SSTables allowed to participate in a single repair session + // Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. + // This is to avoid penalizing good neighbors with an outlier. + public volatile Integer sstable_upper_threshold; + // the minimum time in hours between repairs of the same token range + // The minimum number of hours to run one repair cycle is 24 hours. The default is 24 hours. + // This means that if auto repair finishes one round on one cluster within 24 hours, it won’t start a new round. + // This is applicable for extremely tiny clusters, say 3 nodes. + public volatile DurationSpec.IntSecondsBound min_repair_interval; + // specifies a denylist of datacenters to repair + // This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. + public volatile Set ignore_dcs; + // Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false' + // It is the same as -pr in nodetool repair options. + public volatile Boolean repair_primary_token_range_only; + // configures whether to force immediate repair on new nodes + public volatile Boolean force_repair_new_node; + // the maximum time in seconds that a repair session can run for a single table + // Max time for repairing one table, if exceeded, skip the table. The default is 6 * 60 * 60, which is 6 hours. + // Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. + // Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. + // Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator telling them they are violating SLA even if I am a neighbor, and it would require a lot of back-and-forth manual interventions, etc. + // So, the idea here is to penalize the outliers instead of good candidates. This can easily be configured with a higher value if we want to disable the functionality. + // Please note the repair will still run in parallel on other nodes, this is to address outliers on a given node. + public volatile DurationSpec.IntSecondsBound table_max_repair_time; + // the default is 'true'. MVs are mutated at LOCAL_ONE consistency level in Cassandra. + // This flag determines whether the auto-repair framework needs to run anti-entropy, a.k.a, repair on the MV table or not. + public volatile Boolean mv_repair_enabled; + // the default is DefaultAutoRepairTokenSplitter.class.getName(). The class should implement IAutoRepairTokenRangeSplitter. + // The default implementation splits the tokens based on the token ranges owned by this node divided by the number of 'number_of_subranges' + public volatile String token_range_splitter; + + public String toString() + { + return "Options{" + + "enabled=" + enabled + + ", repair_by_keyspace=" + repair_by_keyspace + + ", number_of_subranges=" + number_of_subranges + + ", number_of_repair_threads=" + number_of_repair_threads + + ", parallel_repair_count_in_group=" + parallel_repair_count_in_group + + ", parallel_repair_percentage_in_group=" + parallel_repair_percentage_in_group + + ", sstable_upper_threshold=" + sstable_upper_threshold + + ", min_repair_interval=" + min_repair_interval + + ", ignore_dcs=" + ignore_dcs + + ", repair_primary_token_range_only=" + repair_primary_token_range_only + + ", force_repair_new_node=" + force_repair_new_node + + ", table_max_repair_time=" + table_max_repair_time + + ", mv_repair_enabled=" + mv_repair_enabled + + ", token_range_splitter=" + token_range_splitter + + '}'; + } + } + + @VisibleForTesting + protected T applyOverrides(RepairType repairType, Function optionSupplier) + { + ArrayList optsProviders = new ArrayList<>(); + if (repair_type_overrides != null) + { + optsProviders.add(repair_type_overrides.get(repairType)); + } + optsProviders.add(global_settings); + optsProviders.add(Options.defaultOptions); + + return optsProviders.stream() + .map(opt -> Optional.ofNullable(opt).map(optionSupplier).orElse(null)) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + } + + protected void ensureOverrides(RepairType repairType) + { + if (repair_type_overrides == null) + { + repair_type_overrides = new EnumMap<>(RepairType.class); + } + + if (repair_type_overrides.get(repairType) == null) + { + repair_type_overrides.put(repairType, new Options()); + } + } +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairKeyspace.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairKeyspace.java new file mode 100644 index 000000000000..14cba17f6095 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairKeyspace.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.concurrent.TimeUnit; + +import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.Tables; + +public class AutoRepairKeyspace +{ + private AutoRepairKeyspace() + { + } + + /** + * Generation is used as a timestamp for automatic table creation on startup. + * If you make any changes to the tables below, make sure to increment the + * generation and document your change here. + */ + public static final long GENERATION = 0; + + public static final String AUTO_REPAIR_HISTORY = "auto_repair_history"; + + public static final String AUTO_REPAIR_PRIORITY = "auto_repair_priority"; + + private static final TableMetadata AutoRepairHistory = + parse(AUTO_REPAIR_HISTORY, + "Auto repair history for each node", + "CREATE TABLE %s (" + + "host_id uuid," + + "repair_type text," + + "repair_turn text," + + "repair_start_ts timestamp," + + "repair_finish_ts timestamp," + + "delete_hosts set," + + "delete_hosts_update_time timestamp," + + "force_repair boolean," + + "PRIMARY KEY (repair_type, host_id))"); + + private static final TableMetadata AutoRepairPriority = + parse(AUTO_REPAIR_PRIORITY, + "Auto repair priority for each group", + "CREATE TABLE %s (" + + "repair_type text," + + "repair_priority set," + + "PRIMARY KEY (repair_type))"); + + private static TableMetadata parse(String name, String description, String schema) + { + return CreateTableStatement.parse(String.format(schema, name), SchemaConstants.DISTRIBUTED_KEYSPACE_NAME) + .id(TableId.forSystemTable(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, name)) + .comment(description) + .gcGraceSeconds((int) TimeUnit.DAYS.toSeconds(90)) + .build(); + } + + public static KeyspaceMetadata metadata() + { + Tables tables = Tables.of(AutoRepairHistory, AutoRepairPriority); + return KeyspaceMetadata.create(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, KeyspaceParams.simple(1), tables); + } +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java new file mode 100644 index 000000000000..b4a15ebb6dd0 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.view.TableViews; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.metrics.AutoRepairMetricsManager; +import org.apache.cassandra.metrics.AutoRepairMetrics; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils.AutoRepairHistory; +import org.apache.cassandra.repair.RepairParallelism; +import org.apache.cassandra.repair.RepairRunnable; +import org.apache.cassandra.repair.messages.RepairOption; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.concurrent.Condition; +import org.apache.cassandra.utils.progress.ProgressEvent; +import org.apache.cassandra.utils.progress.ProgressEventType; +import org.apache.cassandra.utils.progress.ProgressListener; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.text.SimpleDateFormat; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; + +// AutoRepairState represents the state of automated repair for a given repair type. +public abstract class AutoRepairState implements ProgressListener +{ + protected static final Logger logger = LoggerFactory.getLogger(AutoRepairState.class); + private final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); + @VisibleForTesting + protected static Supplier timeFunc = System::currentTimeMillis; + + @VisibleForTesting + protected final RepairType repairType; + @VisibleForTesting + protected int totalTablesConsideredForRepair = 0; + @VisibleForTesting + protected long lastRepairTimeInMs; + @VisibleForTesting + protected int nodeRepairTimeInSec = 0; + @VisibleForTesting + protected int clusterRepairTimeInSec = 0; + @VisibleForTesting + protected boolean repairInProgress = false; + @VisibleForTesting + protected int repairKeyspaceCount = 0; + @VisibleForTesting + protected int repairTableSuccessCount = 0; + @VisibleForTesting + protected int repairTableSkipCount = 0; + @VisibleForTesting + protected int repairTableFailureCount = 0; + @VisibleForTesting + protected int totalMVTablesConsideredForRepair = 0; + + @VisibleForTesting + protected int totalDisabledTablesRepairCount = 0; + @VisibleForTesting + protected AutoRepairHistory longestUnrepairedNode; + @VisibleForTesting + protected Condition condition = newOneTimeCondition(); + @VisibleForTesting + protected boolean success = true; + protected final AutoRepairMetrics metrics; + + protected AutoRepairState(RepairType repairType) + { + metrics = AutoRepairMetricsManager.getMetrics(repairType); + this.repairType = repairType; + } + + public abstract RepairRunnable getRepairRunnable(String keyspace, List tables, Set> ranges, boolean primaryRangeOnly); + + protected RepairRunnable getRepairRunnable(String keyspace, RepairOption options) + { + RepairRunnable task = new RepairRunnable(StorageService.instance, StorageService.nextRepairCommand.incrementAndGet(), + options, keyspace); + + task.addProgressListener(this); + + return task; + } + + @Override + public void progress(String tag, ProgressEvent event) + { + ProgressEventType type = event.getType(); + String message = String.format("[%s] %s", format.format(System.currentTimeMillis()), event.getMessage()); + if (type == ProgressEventType.ERROR) + { + logger.error("Repair failure for {} repair: {}", repairType.toString(), message); + success = false; + condition.signalAll(); + } + if (type == ProgressEventType.PROGRESS) + { + message = message + " (progress: " + (int) event.getProgressPercentage() + "%)"; + logger.debug("Repair progress for {} repair: {}", repairType.toString(), message); + } + if (type == ProgressEventType.COMPLETE) + { + success = true; + condition.signalAll(); + } + } + + public void waitForRepairToComplete() throws InterruptedException + { + //if for some reason we don't hear back on repair progress for sometime + if (!condition.await(12, TimeUnit.HOURS)) + { + success = false; + } + } + + public long getLastRepairTime() + { + return lastRepairTimeInMs; + } + + public void setTotalTablesConsideredForRepair(int count) + { + totalTablesConsideredForRepair = count; + } + + public int getTotalTablesConsideredForRepair() + { + return totalTablesConsideredForRepair; + } + + public void setLastRepairTime(long lastRepairTime) + { + lastRepairTimeInMs = lastRepairTime; + } + + public int getClusterRepairTimeInSec() + { + return clusterRepairTimeInSec; + } + + public int getNodeRepairTimeInSec() + { + return nodeRepairTimeInSec; + } + + public void setRepairInProgress(boolean repairInProgress) + { + this.repairInProgress = repairInProgress; + } + + public boolean isRepairInProgress() + { + return repairInProgress; + } + + public void setRepairSkippedTablesCount(int count) + { + repairTableSkipCount = count; + } + + public int getRepairSkippedTablesCount() + { + return repairTableSkipCount; + } + + public int getLongestUnrepairedSec() + { + if (longestUnrepairedNode == null) + { + return 0; + } + return (int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - longestUnrepairedNode.getLastRepairFinishTime()); + } + + public void setRepairFailedTablesCount(int count) + { + repairTableFailureCount = count; + } + + public int getRepairFailedTablesCount() + { + return repairTableFailureCount; + } + + public void setTotalMVTablesConsideredForRepair(int count) + { + totalMVTablesConsideredForRepair = count; + } + + public int getTotalMVTablesConsideredForRepair() + { + return totalMVTablesConsideredForRepair; + } + + public void setNodeRepairTimeInSec(int elapsed) + { + nodeRepairTimeInSec = elapsed; + } + + public void setClusterRepairTimeInSec(int seconds) + { + clusterRepairTimeInSec = seconds; + } + + public void setRepairKeyspaceCount(int count) + { + repairKeyspaceCount = count; + } + + public void setRepairTableSuccessCount(int count) + { + repairTableSuccessCount = count; + } + + public int getRepairKeyspaceCount() + { + return repairKeyspaceCount; + } + + public int getRepairTableSuccessCount() + { + return repairTableSuccessCount; + } + + public void setLongestUnrepairedNode(AutoRepairHistory longestUnrepairedNode) + { + this.longestUnrepairedNode = longestUnrepairedNode; + } + + public boolean isSuccess() + { + return success; + } + + public void recordTurn(AutoRepairUtils.RepairTurn turn) + { + metrics.recordTurn(turn); + } + + public void setTotalDisabledTablesRepairCount(int count) + { + totalDisabledTablesRepairCount = count; + } + + public int getTotalDisabledTablesRepairCount() + { + return totalDisabledTablesRepairCount; + } + + public void resetWaitCondition() + { + condition = newOneTimeCondition(); + } +} + +class IncrementalRepairState extends AutoRepairState +{ + public IncrementalRepairState() + { + super(RepairType.incremental); + } + + @Override + public RepairRunnable getRepairRunnable(String keyspace, List tables, Set> ranges, boolean primaryRangeOnly) + { + RepairOption option = new RepairOption(RepairParallelism.PARALLEL, primaryRangeOnly, true, false, + AutoRepairService.instance.getAutoRepairConfig().getRepairThreads(repairType), ranges, + !ranges.isEmpty(), false, false, PreviewKind.NONE, true, true, false, false); + + option.getColumnFamilies().addAll(filterOutUnsafeTables(keyspace, tables)); + + return getRepairRunnable(keyspace, option); + } + + @VisibleForTesting + protected List filterOutUnsafeTables(String keyspaceName, List tables) + { + Keyspace keyspace = Keyspace.open(keyspaceName); + + return tables.stream() + .filter(table -> { + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(table); + TableViews views = keyspace.viewManager.forTable(cfs.metadata().id); + if (views != null && !views.isEmpty()) + { + logger.debug("Skipping incremental repair for {}.{} as it has materialized views", keyspaceName, table); + return false; + } + + if (cfs.metadata().params != null && cfs.metadata().params.cdc) + { + logger.debug("Skipping incremental repair for {}.{} as it has CDC enabled", keyspaceName, table); + return false; + } + + return true; + }).collect(Collectors.toList()); + } +} + +class FullRepairState extends AutoRepairState +{ + public FullRepairState() + { + super(RepairType.full); + } + + @Override + public RepairRunnable getRepairRunnable(String keyspace, List tables, Set> ranges, boolean primaryRangeOnly) + { + RepairOption option = new RepairOption(RepairParallelism.PARALLEL, primaryRangeOnly, false, false, + AutoRepairService.instance.getAutoRepairConfig().getRepairThreads(repairType), ranges, + !ranges.isEmpty(), false, false, PreviewKind.NONE, true, true, false, false); + + option.getColumnFamilies().addAll(tables); + + return getRepairRunnable(keyspace, option); + } +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java new file mode 100644 index 000000000000..5cc95367192d --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -0,0 +1,806 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.repair.autorepair; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; +import com.google.common.collect.Lists; + +import org.apache.cassandra.locator.LocalStrategy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.cql3.statements.ModificationStatement; +import org.apache.cassandra.cql3.statements.SelectStatement; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.NetworkTopologyStrategy; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.ViewMetadata; +import org.apache.cassandra.serializers.SetSerializer; +import org.apache.cassandra.serializers.UUIDSerializer; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.QueryState; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.transport.messages.ResultMessage; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; + +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_DUE_TO_PRIORITY; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.NOT_MY_TURN; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_FORCE_REPAIR; + +/** + * This class serves as a utility class for AutoRepair. It contains various helper APIs + * to store/retrieve repair status, decide whose turn is next, etc. + */ +public class AutoRepairUtils +{ + private static final Logger logger = LoggerFactory.getLogger(AutoRepairUtils.class); + static final String COL_REPAIR_TYPE = "repair_type"; + static final String COL_HOST_ID = "host_id"; + static final String COL_REPAIR_START_TS = "repair_start_ts"; + static final String COL_REPAIR_FINISH_TS = "repair_finish_ts"; + static final String COL_REPAIR_PRIORITY = "repair_priority"; + static final String COL_DELETE_HOSTS = "delete_hosts"; // this set stores the host ids which think the row should be deleted + static final String COL_REPAIR_TURN = "repair_turn"; // this record the last repair turn. Normal turn or turn due to priority + static final String COL_DELETE_HOSTS_UPDATE_TIME = "delete_hosts_update_time"; // the time when delete hosts are upated + static final String COL_FORCE_REPAIR = "force_repair"; // if set to true, the node will do non-primary range rapair + + final static String SELECT_REPAIR_HISTORY = String.format( + "SELECT * FROM %s.%s WHERE %s = ?", SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, + AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE); + final static String SELECT_REPAIR_PRIORITY = String.format( + "SELECT * FROM %s.%s WHERE %s = ?", SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, + AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_TYPE); + final static String DEL_REPAIR_PRIORITY = String.format( + "DELETE %s[?] FROM %s.%s WHERE %s = ?", COL_REPAIR_PRIORITY, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, + AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_TYPE); + final static String ADD_PRIORITY_HOST = String.format( + "UPDATE %s.%s SET %s = %s + ? WHERE %s = ?", SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, + AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_PRIORITY, COL_REPAIR_PRIORITY, COL_REPAIR_TYPE); + + final static String INSERT_NEW_REPAIR_HISTORY = String.format( + "INSERT INTO %s.%s (%s, %s, %s, %s, %s, %s) values (?, ? ,?, ?, {}, ?) IF NOT EXISTS", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, + COL_HOST_ID, COL_REPAIR_START_TS, COL_REPAIR_FINISH_TS, COL_DELETE_HOSTS, COL_DELETE_HOSTS_UPDATE_TIME); + + final static String ADD_HOST_ID_TO_DELETE_HOSTS = String.format( + "UPDATE %s.%s SET %s = %s + ?, %s = ? WHERE %s = ? AND %s = ? IF EXISTS" + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_DELETE_HOSTS, + COL_DELETE_HOSTS, COL_DELETE_HOSTS_UPDATE_TIME, COL_REPAIR_TYPE, COL_HOST_ID); + + final static String DEL_AUTO_REPAIR_HISTORY = String.format( + "DELETE FROM %s.%s WHERE %s = ? AND %s = ?" + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, + COL_HOST_ID); + + final static String RECORD_START_REPAIR_HISTORY = String.format( + "UPDATE %s.%s SET %s= ?, repair_turn = ? WHERE %s = ? AND %s = ?" + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_START_TS, + COL_REPAIR_TYPE, COL_HOST_ID); + + final static String RECORD_FINISH_REPAIR_HISTORY = String.format( + + "UPDATE %s.%s SET %s= ?, %s=false WHERE %s = ? AND %s = ?" + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_FINISH_TS, + COL_FORCE_REPAIR, COL_REPAIR_TYPE, COL_HOST_ID); + + final static String CLEAR_DELETE_HOSTS = String.format( + "UPDATE %s.%s SET %s= {} WHERE %s = ? AND %s = ?" + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_DELETE_HOSTS, + COL_REPAIR_TYPE, COL_HOST_ID); + + final static String SET_FORCE_REPAIR = String.format( + "UPDATE %s.%s SET %s=true WHERE %s = ? AND %s = ?" + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_FORCE_REPAIR, + COL_REPAIR_TYPE, COL_HOST_ID); + + static ModificationStatement delStatementRepairHistory; + static SelectStatement selectStatementRepairHistory; + static ModificationStatement delStatementPriorityStatus; + static SelectStatement selectStatementRepairPriority; + static ModificationStatement addPriorityHost; + static ModificationStatement insertNewRepairHistoryStatement; + static ModificationStatement recordStartRepairHistoryStatement; + static ModificationStatement recordFinishRepairHistoryStatement; + static ModificationStatement addHostIDToDeleteHostsStatement; + static ModificationStatement clearDeleteHostsStatement; + static ModificationStatement setForceRepairStatement; + static ConsistencyLevel internalQueryCL; + + public enum RepairTurn + { + MY_TURN, + NOT_MY_TURN, + MY_TURN_DUE_TO_PRIORITY, + MY_TURN_FORCE_REPAIR + } + + public static void setup() + { + selectStatementRepairHistory = (SelectStatement) QueryProcessor.getStatement(SELECT_REPAIR_HISTORY, ClientState + .forInternalCalls()); + selectStatementRepairPriority = (SelectStatement) QueryProcessor.getStatement(SELECT_REPAIR_PRIORITY, ClientState + .forInternalCalls()); + delStatementPriorityStatus = (ModificationStatement) QueryProcessor.getStatement(DEL_REPAIR_PRIORITY, ClientState + .forInternalCalls()); + addPriorityHost = (ModificationStatement) QueryProcessor.getStatement(ADD_PRIORITY_HOST, ClientState + .forInternalCalls()); + insertNewRepairHistoryStatement = (ModificationStatement) QueryProcessor.getStatement(INSERT_NEW_REPAIR_HISTORY, ClientState + .forInternalCalls()); + recordStartRepairHistoryStatement = (ModificationStatement) QueryProcessor.getStatement(RECORD_START_REPAIR_HISTORY, ClientState + .forInternalCalls()); + recordFinishRepairHistoryStatement = (ModificationStatement) QueryProcessor.getStatement(RECORD_FINISH_REPAIR_HISTORY, ClientState + .forInternalCalls()); + addHostIDToDeleteHostsStatement = (ModificationStatement) QueryProcessor.getStatement(ADD_HOST_ID_TO_DELETE_HOSTS, ClientState + .forInternalCalls()); + setForceRepairStatement = (ModificationStatement) QueryProcessor.getStatement(SET_FORCE_REPAIR, ClientState + .forInternalCalls()); + clearDeleteHostsStatement = (ModificationStatement) QueryProcessor.getStatement(CLEAR_DELETE_HOSTS, ClientState + .forInternalCalls()); + delStatementRepairHistory = (ModificationStatement) QueryProcessor.getStatement(DEL_AUTO_REPAIR_HISTORY, ClientState + .forInternalCalls()); + Keyspace autoRepairKS = Schema.instance.getKeyspaceInstance(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME); + internalQueryCL = autoRepairKS.getReplicationStrategy().getClass() == NetworkTopologyStrategy.class ? + ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.ONE; + } + + public static class AutoRepairHistory + { + UUID hostId; + String repairTurn; + long lastRepairStartTime; + long lastRepairFinishTime; + Set deleteHosts; + long deleteHostsUpdateTime; + boolean forceRepair; + + public AutoRepairHistory(UUID hostId, String repairTurn, long lastRepairStartTime, long lastRepairFinishTime, + Set deleteHosts, long deleteHostsUpateTime, boolean forceRepair) + { + this.hostId = hostId; + this.repairTurn = repairTurn; + this.lastRepairStartTime = lastRepairStartTime; + this.lastRepairFinishTime = lastRepairFinishTime; + this.deleteHosts = deleteHosts; + if (this.deleteHosts == null) + { + this.deleteHosts = new HashSet<>(); + } + this.deleteHostsUpdateTime = deleteHostsUpateTime; + this.forceRepair = forceRepair; + } + + public String toString() + { + return MoreObjects.toStringHelper(this). + add("hostId", hostId). + add("repairTurn", repairTurn). + add("lastRepairStartTime", lastRepairStartTime). + add("lastRepairFinishTime", lastRepairFinishTime). + add("deleteHosts", deleteHosts). + toString(); + } + + public boolean isRepairRunning() + { + // if a repair history record has start time laster than finish time, it means the repair is running + return lastRepairStartTime > lastRepairFinishTime; + } + + public long getLastRepairFinishTime() + { + return lastRepairFinishTime; + } + } + + public static class CurrentRepairStatus + { + public Set hostIdsWithOnGoingRepair; // hosts that is running repair + public Set hostIdsWithOnGoingForceRepair; // hosts that is running repair because of force repair + Set priority; + List historiesWithoutOnGoingRepair; // hosts that is NOT running repair + + public CurrentRepairStatus(List repairHistories, Set priority) + { + hostIdsWithOnGoingRepair = new HashSet<>(); + hostIdsWithOnGoingForceRepair = new HashSet<>(); + historiesWithoutOnGoingRepair = new ArrayList<>(); + + for (AutoRepairHistory history : repairHistories) + { + if (history.isRepairRunning()) + { + if (history.forceRepair) + { + hostIdsWithOnGoingForceRepair.add(history.hostId); + } + else + { + hostIdsWithOnGoingRepair.add(history.hostId); + } + } + else + { + historiesWithoutOnGoingRepair.add(history); + } + } + this.priority = priority; + } + + public String toString() + { + return MoreObjects.toStringHelper(this). + add("hostIdsWithOnGoingRepair", hostIdsWithOnGoingRepair). + add("hostIdsWithOnGoingForceRepair", hostIdsWithOnGoingForceRepair). + add("historiesWithoutOnGoingRepair", historiesWithoutOnGoingRepair). + add("priority", priority). + toString(); + } + } + + @VisibleForTesting + public static List getAutoRepairHistoryByGroupID(RepairType repairType) + { + UntypedResultSet repairHistoryResult; + + ResultMessage.Rows repairStatusRows = selectStatementRepairHistory.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, Lists.newArrayList(ByteBufferUtil.bytes(repairType.toString()))), Dispatcher.RequestTime.forImmediateExecution()); + repairHistoryResult = UntypedResultSet.create(repairStatusRows.result); + + List repairHistories = new ArrayList<>(); + if (repairHistoryResult.size() > 0) + { + for (UntypedResultSet.Row row : repairHistoryResult) + { + UUID hostId = row.getUUID(COL_HOST_ID); + String repairTurn = null; + if (row.has(COL_REPAIR_TURN)) + repairTurn = row.getString(COL_REPAIR_TURN); + long lastRepairStartTime = row.getLong(COL_REPAIR_START_TS, 0); + long lastRepairFinishTime = row.getLong(COL_REPAIR_FINISH_TS, 0); + Set deleteHosts = row.getSet(COL_DELETE_HOSTS, UUIDType.instance); + long deleteHostsUpdateTime = row.getLong(COL_DELETE_HOSTS_UPDATE_TIME, 0); + Boolean forceRepair = row.has(COL_FORCE_REPAIR) ? row.getBoolean(COL_FORCE_REPAIR) : false; + repairHistories.add(new AutoRepairHistory(hostId, repairTurn, lastRepairStartTime, lastRepairFinishTime, + deleteHosts, deleteHostsUpdateTime, forceRepair)); + } + return repairHistories; + } + logger.info("No repair history found"); + return null; + } + + public static List getAutoRepairHistoryForLocalGroup(RepairType repairType) + { + return getAutoRepairHistoryByGroupID(repairType); + } + + // A host may add itself in delete hosts for some other hosts due to restart or some temp gossip issue. If a node's record + // delete_hosts is not growing for more than 2 hours, we consider it as a normal node so we clear the delete_hosts for that node + public static void clearDeleteHosts(RepairType repairType, UUID hostId) + { + clearDeleteHostsStatement.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, + Lists.newArrayList(ByteBufferUtil.bytes(repairType.toString()), + ByteBufferUtil.bytes(hostId))), Dispatcher.RequestTime.forImmediateExecution()); + } + + public static void setForceRepairNewNode(RepairType repairType) + { + // this function will be called when a node bootstrap finished + UUID hostId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); + // insert the data first + insertNewRepairHistory(repairType, System.currentTimeMillis(), System.currentTimeMillis()); + setForceRepair(repairType, hostId); + } + + public static void setForceRepair(RepairType repairType, Set hosts) + { + // this function is used by nodetool + for (InetAddressAndPort host : hosts) + { + UUID hostId = Gossiper.instance.getHostId(host); + setForceRepair(repairType, hostId); + } + } + + public static void setForceRepair(RepairType repairType, UUID hostId) + { + setForceRepairStatement.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, + Lists.newArrayList(ByteBufferUtil.bytes(repairType.toString()), + ByteBufferUtil.bytes(hostId))), + Dispatcher.RequestTime.forImmediateExecution()); + + logger.info("Set force repair repair type: {}, node: {}", repairType, hostId); + } + + public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType) + { + List autoRepairHistories = getAutoRepairHistoryForLocalGroup(repairType); + return getCurrentRepairStatus(repairType, autoRepairHistories); + } + + public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType, List autoRepairHistories) + { + if (autoRepairHistories != null) + { + CurrentRepairStatus status = new CurrentRepairStatus(autoRepairHistories, getPriorityHostIds(repairType)); + + return status; + } + return null; + } + + @VisibleForTesting + protected static TreeSet getHostIdsInCurrentRing(RepairType repairType, Set allNodesInRing) + { + TreeSet hostIdsInCurrentRing = new TreeSet<>(); + for (InetAddressAndPort node : allNodesInRing) + { + String nodeDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(node); + if (AutoRepairService.instance.getAutoRepairConfig().getIgnoreDCs(repairType).contains(nodeDC)) + { + logger.info("Ignore node {} because its datacenter is {}", node, nodeDC); + continue; + } + /** Check if endpoint state exists in gossip or not. If it + * does not then this maybe a ghost node so ignore it + */ + if (Gossiper.instance.isAlive(node)) + { + UUID hostId = Gossiper.instance.getHostId(node); + hostIdsInCurrentRing.add(hostId); + } + else + { + logger.info("Node is not present in Gossipe cache node {}, node data center {}", node, nodeDC); + } + } + return hostIdsInCurrentRing; + } + + public static TreeSet getHostIdsInCurrentRing(RepairType repairType) + { + Set allNodesInRing = StorageService.instance.getTokenMetadata().getAllEndpoints(); + return getHostIdsInCurrentRing(repairType, allNodesInRing); + } + + // This function will return the host ID for the node which has not been repaired for longest time + public static AutoRepairHistory getHostWithLongestUnrepairTime(RepairType repairType) + { + List autoRepairHistories = getAutoRepairHistoryForLocalGroup(repairType); + return getHostWithLongestUnrepairTime(autoRepairHistories); + } + + private static AutoRepairHistory getHostWithLongestUnrepairTime(List autoRepairHistories) + { + if (autoRepairHistories == null) + { + return null; + } + AutoRepairHistory rst = null; + long oldestTimestamp = Long.MAX_VALUE; + for (AutoRepairHistory autoRepairHistory : autoRepairHistories) + { + if (autoRepairHistory.lastRepairFinishTime < oldestTimestamp) + { + rst = autoRepairHistory; + oldestTimestamp = autoRepairHistory.lastRepairFinishTime; + } + } + return rst; + } + + public static int getMaxNumberOfNodeRunAutoRepairInGroup(RepairType repairType, int groupSize) + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + if (groupSize == 0) + { + return Math.max(config.getParallelRepairCountInGroup(repairType), 1); + } + // we will use the max number from config between auto_repair_parallel_repair_count_in_group and auto_repair_parallel_repair_percentage_in_group + int value = Math.max(groupSize * config.getParallelRepairPercentageInGroup(repairType) / 100, + config.getParallelRepairCountInGroup(repairType)); + // make sure at least one node getting repaired + return Math.max(1, value); + } + + @VisibleForTesting + public static RepairTurn myTurnToRunRepair(RepairType repairType, UUID myId) + { + try + { + Set allNodesInRing = StorageService.instance.getTokenMetadata().getAllEndpoints(); + logger.info("Total nodes in ring {}", allNodesInRing.size()); + TreeSet hostIdsInCurrentRing = getHostIdsInCurrentRing(repairType, allNodesInRing); + logger.info("Total nodes qualified for repair {}", hostIdsInCurrentRing.size()); + + List autoRepairHistories = getAutoRepairHistoryForLocalGroup(repairType); + Set autoRepairHistoryIds = new HashSet<>(); + + // 1. Remove any node that is not part of group based on goissip info + if (autoRepairHistories != null) + { + for (AutoRepairHistory nodeHistory : autoRepairHistories) + { + autoRepairHistoryIds.add(nodeHistory.hostId); + // clear delete_hosts if the node's delete hosts is not growing for more than two hours + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + if (nodeHistory.deleteHosts.size() > 0 + && config.getAutoRepairHistoryClearDeleteHostsBufferInterval().toSeconds() < TimeUnit.MILLISECONDS.toSeconds( + System.currentTimeMillis() - nodeHistory.deleteHostsUpdateTime + )) + { + clearDeleteHosts(repairType, nodeHistory.hostId); + logger.info("Delete hosts for {} for repair type {} has not been updated for more than {} seconds. Delete hosts has been cleared. Delete hosts before clear {}" + , nodeHistory.hostId, repairType, config.getAutoRepairHistoryClearDeleteHostsBufferInterval(), nodeHistory.deleteHosts); + } + else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) + { + if (nodeHistory.deleteHosts.size() > Math.max(2, hostIdsInCurrentRing.size() * 0.5)) + { + // More than half of the groups thinks the record should be deleted + logger.info("{} think {} is orphan node, will delete auto repair history for repair type {}.", nodeHistory.deleteHosts, nodeHistory.hostId, repairType); + deleteAutoRepairHistory(repairType, nodeHistory.hostId); + } + else + { + // I think this host should be deleted + logger.info("I({}) think {} is not part of ring, vote to delete it for repair type {}.", myId, nodeHistory.hostId, repairType); + addHostIdToDeleteHosts(repairType, myId, nodeHistory.hostId); + } + } + } + } + + // 2. Add node to auto repair history table if a node is in gossip info + for (UUID hostId : hostIdsInCurrentRing) + { + if (!autoRepairHistoryIds.contains(hostId)) + { + logger.info("{} for repair type {} doesn't exist in the auto repair history table, insert a new record.", repairType, hostId); + insertNewRepairHistory(repairType, hostId, System.currentTimeMillis(), System.currentTimeMillis()); + } + } + + //get current repair status + CurrentRepairStatus currentRepairStatus = getCurrentRepairStatus(repairType, autoRepairHistories); + if (currentRepairStatus != null) + { + logger.info("Latest repair status {}", currentRepairStatus); + //check if I am forced to run repair + for (AutoRepairHistory history : currentRepairStatus.historiesWithoutOnGoingRepair) + { + if (history.forceRepair && history.hostId.equals(myId)) + { + return MY_TURN_FORCE_REPAIR; + } + } + } + + int parallelRepairNumber = getMaxNumberOfNodeRunAutoRepairInGroup(repairType, + autoRepairHistories == null ? 0 : autoRepairHistories.size()); + logger.info("Will run repairs concurrently on {} node(s)", parallelRepairNumber); + + if (currentRepairStatus == null || parallelRepairNumber > currentRepairStatus.hostIdsWithOnGoingRepair.size()) + { + // more repairs can be run, I might be the new one + + if (autoRepairHistories != null) + { + logger.info("Auto repair history table has {} records", autoRepairHistories.size()); + } + else + { + // try to fetch again + autoRepairHistories = getAutoRepairHistoryForLocalGroup(repairType); + currentRepairStatus = getCurrentRepairStatus(repairType, autoRepairHistories); + if (autoRepairHistories == null || currentRepairStatus == null) + { + logger.error("No record found"); + return NOT_MY_TURN; + } + } + + // get the longest unrepaired node from the nodes which are not running repair + AutoRepairHistory defaultNodeToBeRepaired = getHostWithLongestUnrepairTime(currentRepairStatus.historiesWithoutOnGoingRepair); + //check who is next, which is helpful for debugging + logger.info("Next node to be repaired for repair type {} by default: {}", repairType, defaultNodeToBeRepaired); + UUID priorityHostId = null; + if (currentRepairStatus.priority != null) + { + for (UUID priorityID : currentRepairStatus.priority) + { + // remove ids doesn't belong to this ring + if (!hostIdsInCurrentRing.contains(priorityID)) + { + logger.info("{} is not part of the current ring, will be removed from priority list.", priorityID); + removePriorityStatus(repairType, priorityID); + } + else + { + priorityHostId = priorityID; + break; + } + } + } + + if (priorityHostId != null && !myId.equals(priorityHostId)) + { + logger.info("Priority list is not empty and I'm not the first node in the list, not my turn." + + "First node in priority list is {}", StorageService.instance.getTokenMetadata().getEndpointForHostId(priorityHostId)); + return NOT_MY_TURN; + } + + if (myId.equals(priorityHostId)) + { + //I have a priority for repair hence its my turn now + return MY_TURN_DUE_TO_PRIORITY; + } + + if (defaultNodeToBeRepaired.hostId.equals(myId)) + return MY_TURN; + } + else if (currentRepairStatus.hostIdsWithOnGoingForceRepair.contains(myId)) + { + return MY_TURN_FORCE_REPAIR; + } + // for some reason I was not done with the repair hence resume (maybe node restart in-between, etc.) + return currentRepairStatus.hostIdsWithOnGoingRepair.contains(myId) ? MY_TURN : NOT_MY_TURN; + } + catch (Exception e) + { + logger.error("Exception while deciding node's turn:", e); + } + return NOT_MY_TURN; + } + + static void deleteAutoRepairHistory(RepairType repairType, UUID hostId) + { + //delete the given hostId from current local group + delStatementRepairHistory.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, + Lists.newArrayList(ByteBufferUtil.bytes(repairType.toString()), + ByteBufferUtil.bytes(hostId))), Dispatcher.RequestTime.forImmediateExecution()); + } + + static void updateStartAutoRepairHistory(RepairType repairType, UUID myId, long timestamp, RepairTurn turn) + { + recordStartRepairHistoryStatement.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, + Lists.newArrayList(ByteBufferUtil.bytes(timestamp), + ByteBufferUtil.bytes(turn.name()), + ByteBufferUtil.bytes(repairType.toString()), + ByteBufferUtil.bytes(myId) + )), Dispatcher.RequestTime.forImmediateExecution()); + } + + static void updateFinishAutoRepairHistory(RepairType repairType, UUID myId, long timestamp) + { + recordFinishRepairHistoryStatement.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, + Lists.newArrayList(ByteBufferUtil.bytes(timestamp), + ByteBufferUtil.bytes(repairType.toString()), + ByteBufferUtil.bytes(myId) + )), Dispatcher.RequestTime.forImmediateExecution()); + // Do not remove beblow log, the log is used by dtest + logger.info("Auto repair finished for {}", myId); + } + + public static void insertNewRepairHistory(RepairType repairType, UUID hostId, long startTime, long finishTime) + { + try + { + Keyspace autoRepairKS = Schema.instance.getKeyspaceInstance(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME); + ConsistencyLevel cl = autoRepairKS.getReplicationStrategy().getClass() == NetworkTopologyStrategy.class ? + ConsistencyLevel.LOCAL_SERIAL : null; + + UntypedResultSet resultSet; + ResultMessage.Rows resultMessage = (ResultMessage.Rows) insertNewRepairHistoryStatement.execute( + QueryState.forInternalCalls(), QueryOptions.create(internalQueryCL, Lists.newArrayList( + ByteBufferUtil.bytes(repairType.toString()), + ByteBufferUtil.bytes(hostId), + ByteBufferUtil.bytes(startTime), + ByteBufferUtil.bytes(finishTime), + ByteBufferUtil.bytes(System.currentTimeMillis()) + ), false, -1, null, cl, ProtocolVersion.CURRENT, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME), + Dispatcher.RequestTime.forImmediateExecution()); + resultSet = UntypedResultSet.create(resultMessage.result); + boolean applied = resultSet.one().getBoolean(ModificationStatement.CAS_RESULT_COLUMN.toString()); + if (applied) + { + logger.info("Successfully inserted a new auto repair history record for host id: {}", hostId); + } + else + { + logger.info("Record exists, no need to insert again for host id: {}", hostId); + } + } + catch (Exception e) + { + logger.error("Exception in inserting new repair history:", e); + } + } + + public static void insertNewRepairHistory(RepairType repairType, long startTime, long finishTime) + { + UUID hostId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); + insertNewRepairHistory(repairType, hostId, startTime, finishTime); + } + + public static void addHostIdToDeleteHosts(RepairType repairType, UUID myID, UUID hostToBeDeleted) + { + SetSerializer serializer = SetSerializer.getInstance(UUIDSerializer.instance, UTF8Type.instance.comparatorSet); + addHostIDToDeleteHostsStatement.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, + Lists.newArrayList(serializer.serialize(new HashSet<>(Arrays.asList(myID))), + ByteBufferUtil.bytes(System.currentTimeMillis()), + ByteBufferUtil.bytes(repairType.toString()), + ByteBufferUtil.bytes(hostToBeDeleted) + )), Dispatcher.RequestTime.forImmediateExecution()); + } + + public static void addPriorityHosts(RepairType repairType, Set hosts) + { + Set hostIds = new HashSet<>(); + for (InetAddressAndPort host : hosts) + { + //find hostId from IP address + UUID hostId = StorageService.instance.getTokenMetadata().getHostId(host); + hostIds.add(hostId); + if (hostId != null) + { + logger.info("Add host {} to the priority list", hostId); + } + } + if (hostIds.size() > 0) + { + SetSerializer serializer = SetSerializer.getInstance(UUIDSerializer.instance, UTF8Type.instance.comparatorSet); + addPriorityHost.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, + Lists.newArrayList(serializer.serialize(hostIds), + ByteBufferUtil.bytes(repairType.toString()))), + Dispatcher.RequestTime.forImmediateExecution()); + } + } + + static void removePriorityStatus(RepairType repairType, UUID hostId) + { + logger.info("Remove host {} from priority list", hostId); + delStatementPriorityStatus.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, + Lists.newArrayList(ByteBufferUtil.bytes(hostId), + ByteBufferUtil.bytes(repairType.toString()))), + Dispatcher.RequestTime.forImmediateExecution()); + } + + public static Set getPriorityHostIds(RepairType repairType) + { + UntypedResultSet repairPriorityResult; + + ResultMessage.Rows repairPriorityRows = selectStatementRepairPriority.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, Lists.newArrayList(ByteBufferUtil.bytes(repairType.toString()))), Dispatcher.RequestTime.forImmediateExecution()); + repairPriorityResult = UntypedResultSet.create(repairPriorityRows.result); + + Set priorities = null; + if (repairPriorityResult.size() > 0) + { + // there should be only one row + UntypedResultSet.Row row = repairPriorityResult.one(); + priorities = row.getSet(COL_REPAIR_PRIORITY, UUIDType.instance); + } + if (priorities != null) + { + return priorities; + } + return Collections.emptySet(); + } + + public static Set getPriorityHosts(RepairType repairType) + { + Set hosts = new HashSet<>(); + for (UUID hostId : getPriorityHostIds(repairType)) + { + hosts.add(StorageService.instance.getTokenMetadata().getEndpointForHostId(hostId)); + } + return hosts; + } + + public static boolean checkNodeContainsKeyspaceReplica(Keyspace ks) + { + AbstractReplicationStrategy replicationStrategy = ks.getReplicationStrategy(); + boolean ksReplicaOnNode = true; + if (replicationStrategy instanceof NetworkTopologyStrategy) + { + Set datacenters = ((NetworkTopologyStrategy) replicationStrategy).getDatacenters(); + String localDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort()); + if (!datacenters.contains(localDC)) + { + ksReplicaOnNode = false; + } + } + if (replicationStrategy instanceof LocalStrategy) + { + ksReplicaOnNode = false; + } + return ksReplicaOnNode; + } + + + public static boolean tableMaxRepairTimeExceeded(RepairType repairType, long startTime) + { + long tableRepairTimeSoFar = TimeUnit.MILLISECONDS.toSeconds + (System.currentTimeMillis() - startTime); + return AutoRepairService.instance.getAutoRepairConfig().getAutoRepairTableMaxRepairTime(repairType).toSeconds() < + tableRepairTimeSoFar; + } + + public static boolean keyspaceMaxRepairTimeExceeded(RepairType repairType, long startTime, int numOfTablesToBeRepaired) + { + long keyspaceRepairTimeSoFar = TimeUnit.MILLISECONDS.toSeconds((System.currentTimeMillis() - startTime)); + return (long) AutoRepairService.instance.getAutoRepairConfig().getAutoRepairTableMaxRepairTime(repairType).toSeconds() * + numOfTablesToBeRepaired < keyspaceRepairTimeSoFar; + } + + public static List getAllMVs(RepairType repairType, Keyspace keyspace, TableMetadata tableMetadata) + { + List allMvs = new ArrayList<>(); + if (AutoRepairService.instance.getAutoRepairConfig().getMVRepairEnabled(repairType) && keyspace.getMetadata().views != null) + { + Iterator views = keyspace.getMetadata().views.forTable(tableMetadata.id).iterator(); + while (views.hasNext()) + { + String viewName = views.next().name(); + logger.info("Adding MV to the list {}.{}.{}", keyspace.getName(), tableMetadata.name, viewName); + allMvs.add(viewName); + } + } + return allMvs; + } +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java new file mode 100644 index 000000000000..7c254fd358d0 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.repair.autorepair; + + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.cassandra.service.AutoRepairService; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.Pair; + +public class DefaultAutoRepairTokenSplitter implements IAutoRepairTokenRangeSplitter +{ + private static final Logger logger = LoggerFactory.getLogger(DefaultAutoRepairTokenSplitter.class); + + + @Override + public List> getRange(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, String tableName) + { + List> range = new ArrayList<>(); + + Collection> tokens = StorageService.instance.getPrimaryRanges(keyspaceName); + if (!primaryRangeOnly) + { + // if we need to repair non-primary token ranges, then change the tokens accrodingly + tokens = StorageService.instance.getLocalReplicas(keyspaceName).ranges(); + } + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + int numberOfSubranges = config.getRepairSubRangeNum(repairType); + for (Range token : tokens) + { + Murmur3Partitioner.LongToken l = (Murmur3Partitioner.LongToken) (token.left); + Murmur3Partitioner.LongToken r = (Murmur3Partitioner.LongToken) (token.right); + Token parentStartToken = StorageService.instance.getTokenMetadata() + .partitioner.getTokenFactory().fromString("" + l.getTokenValue()); + Token parentEndToken = StorageService.instance.getTokenMetadata() + .partitioner.getTokenFactory().fromString("" + r.getTokenValue()); + logger.debug("Parent Token Left side {}, right side {}", parentStartToken.toString(), + parentEndToken.toString()); + + long left = (Long) l.getTokenValue(); + long right = (Long) r.getTokenValue(); + long repairTokenWidth = (right - left) / numberOfSubranges; + for (int i = 0; i < numberOfSubranges; i++) + { + long curLeft = left + (i * repairTokenWidth); + long curRight = curLeft + repairTokenWidth; + + if ((i + 1) == numberOfSubranges) + { + curRight = right; + } + + Token childStartToken = StorageService.instance.getTokenMetadata() + .partitioner.getTokenFactory().fromString("" + curLeft); + Token childEndToken = StorageService.instance.getTokenMetadata() + .partitioner.getTokenFactory().fromString("" + curRight); + logger.debug("Current Token Left side {}, right side {}", childStartToken + .toString(), childEndToken.toString()); + range.add(Pair.create(childStartToken, childEndToken)); + } + } + return range; + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java new file mode 100644 index 000000000000..2b69898a360d --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.repair.autorepair; + + +import java.util.List; + +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.utils.Pair; + +public interface IAutoRepairTokenRangeSplitter +{ + // split the token range you wish to repair into multiple subranges + // the autorepair framework will repair the list of returned subrange in a sequence + List> getRange(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, String tableName); +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java new file mode 100644 index 000000000000..50f482caf1b7 --- /dev/null +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.schema; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang3.StringUtils; + +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; + +import static java.lang.String.format; + +public final class AutoRepairParams +{ + public enum Option + { + ENABLED; + + @Override + public String toString() + { + return name().toLowerCase(); + } + } + + public static final Map> DEFAULT_OPTIONS = + ImmutableMap.of(AutoRepairConfig.RepairType.full, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true)), + AutoRepairConfig.RepairType.incremental, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true))); + + public final AutoRepairConfig.RepairType type; + + private Map> options = DEFAULT_OPTIONS; + + AutoRepairParams(AutoRepairConfig.RepairType type) + { + this.type = type; + } + + public static AutoRepairParams create(AutoRepairConfig.RepairType repairType, Map options) + { + Map> optionsMap = new HashMap<>(); + for (Map.Entry> entry : DEFAULT_OPTIONS.entrySet()) + { + optionsMap.put(entry.getKey(), new HashMap<>(entry.getValue())); + } + for (Map.Entry entry : options.entrySet()) + { + if (!Option.ENABLED.toString().equals(entry.getKey().toLowerCase())) + { + throw new ConfigurationException(format("Unknown property '%s'", entry.getKey())); + } + optionsMap.get(repairType).put(entry.getKey(), entry.getValue()); + } + AutoRepairParams repairParams = new AutoRepairParams(repairType); + repairParams.options = optionsMap; + return repairParams; + } + + public boolean repairEnabled() + { + String enabled = options.get(type).get(Option.ENABLED.toString()); + return enabled == null + ? Boolean.parseBoolean(DEFAULT_OPTIONS.get(type).get(Option.ENABLED.toString())) + : Boolean.parseBoolean(enabled); + } + + public void validate() + { + String enabled = options.get(type).get(Option.ENABLED.toString()); + if (enabled != null && !isValidBoolean(enabled)) + { + throw new ConfigurationException(format("Invalid value %s for '%s' repair sub-option - must be a boolean", + enabled, + Option.ENABLED)); + } + } + + public static boolean isValidBoolean(String value) + { + return StringUtils.equalsIgnoreCase(value, "true") || StringUtils.equalsIgnoreCase(value, "false"); + } + + public Map options() + { + return options.get(type); + } + + public static AutoRepairParams fromMap(AutoRepairConfig.RepairType repairType, Map map) + { + return create(repairType, map); + } + + public Map asMap() + { + return options.get(type); + } + + @Override + public String toString() + { + return MoreObjects.toStringHelper(this) + .add("options", options) + .toString(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof AutoRepairParams)) + return false; + + AutoRepairParams cp = (AutoRepairParams) o; + + return options.equals(cp.options); + } + + @Override + public int hashCode() + { + return Objects.hash(options); + } +} diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index 3f223dd8a1a2..5cd08901d93d 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -42,6 +42,7 @@ import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.rows.*; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; import org.apache.cassandra.schema.ColumnMetadata.ClusteringOrder; import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff; @@ -121,6 +122,8 @@ private SchemaKeyspace() + "additional_write_policy text," + "cdc boolean," + "read_repair text," + + "automated_repair_full frozen>," + + "automated_repair_incremental frozen>," + "PRIMARY KEY ((keyspace_name), table_name))"); private static final TableMetadata Columns = @@ -189,6 +192,8 @@ private SchemaKeyspace() + "additional_write_policy text," + "cdc boolean," + "read_repair text," + + "automated_repair_full frozen>," + + "automated_repair_incremental frozen>," + "PRIMARY KEY ((keyspace_name), view_name))"); private static final TableMetadata Indexes = @@ -552,7 +557,10 @@ private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBui .add("compaction", params.compaction.asMap()) .add("compression", params.compression.asMap()) .add("read_repair", params.readRepair.toString()) - .add("extensions", params.extensions); + .add("extensions", params.extensions) + .add("automated_repair_full", params.automatedRepair.get(AutoRepairConfig.RepairType.full).asMap()) + .add("automated_repair_incremental", params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()); + // Only add CDC-enabled flag to schema if it's enabled on the node. This is to work around RTE's post-8099 if a 3.8+ // node sends table schema to a < 3.8 versioned node with an unknown column. @@ -976,6 +984,8 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row) SpeculativeRetryPolicy.fromString("99PERCENTILE")) .cdc(row.has("cdc") && row.getBoolean("cdc")) .readRepair(getReadRepairStrategy(row)) + .automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.full, row.getFrozenTextMap("automated_repair_full"))) + .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.incremental, row.getFrozenTextMap("automated_repair_incremental"))) .build(); } diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 440729c4ef67..a61138c665f5 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -18,6 +18,7 @@ package org.apache.cassandra.schema; import java.nio.ByteBuffer; +import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -28,6 +29,7 @@ import org.apache.cassandra.cql3.Attributes; import org.apache.cassandra.cql3.CqlBuilder; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.service.reads.PercentileSpeculativeRetryPolicy; import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; @@ -57,7 +59,10 @@ public enum Option ADDITIONAL_WRITE_POLICY, CRC_CHECK_CHANCE, CDC, - READ_REPAIR; + READ_REPAIR, + AUTOMATED_REPAIR_FULL, + AUTOMATED_REPAIR_INCREMENTAL, + ; @Override public String toString() @@ -84,12 +89,14 @@ public String toString() public final boolean cdc; public final ReadRepairStrategy readRepair; + public final Map automatedRepair; + private TableParams(Builder builder) { comment = builder.comment; bloomFilterFpChance = builder.bloomFilterFpChance == null - ? builder.compaction.defaultBloomFilterFbChance() - : builder.bloomFilterFpChance; + ? builder.compaction.defaultBloomFilterFbChance() + : builder.bloomFilterFpChance; crcCheckChance = builder.crcCheckChance; gcGraceSeconds = builder.gcGraceSeconds; defaultTimeToLive = builder.defaultTimeToLive; @@ -105,6 +112,13 @@ private TableParams(Builder builder) extensions = builder.extensions; cdc = builder.cdc; readRepair = builder.readRepair; + automatedRepair = new HashMap<>() + { + { + put(AutoRepairConfig.RepairType.full, builder.automatedRepairFull); + put(AutoRepairConfig.RepairType.incremental, builder.automatedRepairIncremental); + } + }; } public static Builder builder() @@ -130,7 +144,10 @@ public static Builder builder(TableParams params) .additionalWritePolicy(params.additionalWritePolicy) .extensions(params.extensions) .cdc(params.cdc) - .readRepair(params.readRepair); + .readRepair(params.readRepair) + .automatedRepairFull(params.automatedRepair.get(AutoRepairConfig.RepairType.full)) + .automatedRepairIncremental(params.automatedRepair.get(AutoRepairConfig.RepairType.incremental)) + ; } public Builder unbuild() @@ -144,7 +161,7 @@ public void validate() compression.validate(); double minBloomFilterFpChanceValue = BloomCalculations.minSupportedBloomFilterFpChance(); - if (bloomFilterFpChance <= minBloomFilterFpChanceValue || bloomFilterFpChance > 1) + if (bloomFilterFpChance <= minBloomFilterFpChanceValue || bloomFilterFpChance > 1) { fail("%s must be larger than %s and less than or equal to 1.0 (got %s)", Option.BLOOM_FILTER_FP_CHANCE, @@ -185,6 +202,11 @@ public void validate() if (cdc && memtable.factory().writesShouldSkipCommitLog()) fail("CDC cannot work if writes skip the commit log. Check your memtable configuration."); + + for (Map.Entry entry : automatedRepair.entrySet()) + { + entry.getValue().validate(); + } } private static void fail(String format, Object... args) @@ -204,21 +226,22 @@ public boolean equals(Object o) TableParams p = (TableParams) o; return comment.equals(p.comment) - && bloomFilterFpChance == p.bloomFilterFpChance - && crcCheckChance == p.crcCheckChance - && gcGraceSeconds == p.gcGraceSeconds - && defaultTimeToLive == p.defaultTimeToLive - && memtableFlushPeriodInMs == p.memtableFlushPeriodInMs - && minIndexInterval == p.minIndexInterval - && maxIndexInterval == p.maxIndexInterval - && speculativeRetry.equals(p.speculativeRetry) - && caching.equals(p.caching) - && compaction.equals(p.compaction) - && compression.equals(p.compression) - && memtable.equals(p.memtable) - && extensions.equals(p.extensions) - && cdc == p.cdc - && readRepair == p.readRepair; + && bloomFilterFpChance == p.bloomFilterFpChance + && crcCheckChance == p.crcCheckChance + && gcGraceSeconds == p.gcGraceSeconds + && defaultTimeToLive == p.defaultTimeToLive + && memtableFlushPeriodInMs == p.memtableFlushPeriodInMs + && minIndexInterval == p.minIndexInterval + && maxIndexInterval == p.maxIndexInterval + && speculativeRetry.equals(p.speculativeRetry) + && caching.equals(p.caching) + && compaction.equals(p.compaction) + && compression.equals(p.compression) + && memtable.equals(p.memtable) + && extensions.equals(p.extensions) + && cdc == p.cdc + && readRepair == p.readRepair + && automatedRepair == p.automatedRepair; } @Override @@ -239,7 +262,8 @@ public int hashCode() memtable, extensions, cdc, - readRepair); + readRepair, + automatedRepair); } @Override @@ -262,6 +286,8 @@ public String toString() .add(Option.EXTENSIONS.toString(), extensions) .add(Option.CDC.toString(), cdc) .add(Option.READ_REPAIR.toString(), readRepair) + .add(Option.AUTOMATED_REPAIR_FULL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.full)) + .add(Option.AUTOMATED_REPAIR_INCREMENTAL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.incremental)) .toString(); } @@ -309,7 +335,11 @@ public void appendCqlTo(CqlBuilder builder, boolean isView) .newLine() .append("AND read_repair = ").appendWithSingleQuotes(readRepair.toString()) .newLine() - .append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString()); + .append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString()) + .newLine() + .append("AND automated_repair_full = ").append(automatedRepair.get(AutoRepairConfig.RepairType.full).asMap()) + .newLine() + .append("AND automated_repair_incremental = ").append(automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()); } public static final class Builder @@ -332,6 +362,9 @@ public static final class Builder private boolean cdc; private ReadRepairStrategy readRepair = ReadRepairStrategy.BLOCKING; + private AutoRepairParams automatedRepairFull = new AutoRepairParams(AutoRepairConfig.RepairType.full); + private AutoRepairParams automatedRepairIncremental = new AutoRepairParams(AutoRepairConfig.RepairType.incremental); + public Builder() { } @@ -442,5 +475,17 @@ public Builder extensions(Map val) extensions = ImmutableMap.copyOf(val); return this; } + + public Builder automatedRepairFull(AutoRepairParams val) + { + automatedRepairFull = val; + return this; + } + + public Builder automatedRepairIncremental(AutoRepairParams val) + { + automatedRepairIncremental = val; + return this; + } } } diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index eed52b78f8af..4d33e0493bd3 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -49,6 +49,7 @@ import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.EndpointsByRange; import org.apache.cassandra.locator.EndpointsForRange; +import org.apache.cassandra.service.disk.usage.DiskUsageMonitor; import org.apache.cassandra.utils.ExecutorUtils; import org.apache.cassandra.repair.state.CoordinatorState; import org.apache.cassandra.repair.state.ParticipateState; @@ -599,8 +600,29 @@ public static boolean verifyCompactionsPendingThreshold(TimeUUID parentRepairSes return true; } + public static boolean verifyDiskHeadroomThreshold(TimeUUID parentRepairSession, PreviewKind previewKind, boolean isIncremental) + { + if (!isIncremental) // disk headroom is required for anti-compaction which is only performed by incremental repair + return true; + + double diskUsage = DiskUsageMonitor.instance.getDiskUsage(); + double rejectRatio = ActiveRepairService.instance.getIncrementalRepairDiskHeadroomRejectRatio(); + + if (diskUsage + rejectRatio > 1) + { + logger.error("[{}] Rejecting incoming repair, disk usage ({}%) above threshold ({}%)", + previewKind.logPrefix(parentRepairSession), String.format("%.2f", diskUsage * 100), String.format("%.2f", (1 - rejectRatio) * 100)); + return false; + } + + return true; + } + public TimeUUID prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPort coordinator, Set endpoints, RepairOption options, boolean isForcedRepair, List columnFamilyStores) { + if (!verifyDiskHeadroomThreshold(parentRepairSession, options.getPreviewKind(), options.isIncremental())) + failRepair(parentRepairSession, "Rejecting incoming repair, disk usage above threshold"); // failRepair throws exception + if (!verifyCompactionsPendingThreshold(parentRepairSession, options.getPreviewKind())) failRepair(parentRepairSession, "Rejecting incoming repair, pending compactions above threshold"); // failRepair throws exception @@ -983,6 +1005,16 @@ public void setRepairPendingCompactionRejectThreshold(int value) DatabaseDescriptor.setRepairPendingCompactionRejectThreshold(value); } + public double getIncrementalRepairDiskHeadroomRejectRatio() + { + return DatabaseDescriptor.getIncrementalRepairDiskHeadroomRejectRatio(); + } + + public void setIncrementalRepairDiskHeadroomRejectRatio(double value) + { + DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(value); + } + /** * Remove any parent repair sessions matching predicate */ diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java new file mode 100644 index 000000000000..b07d770fb2de --- /dev/null +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils; +import org.apache.cassandra.utils.MBeanWrapper; + +import java.util.Set; + +import com.google.common.annotations.VisibleForTesting; + +public class AutoRepairService implements AutoRepairServiceMBean +{ + public static final String MBEAN_NAME = "org.apache.cassandra.db:type=AutoRepairService"; + + @VisibleForTesting + protected AutoRepairConfig config; + + public static final AutoRepairService instance = new AutoRepairService(); + + @VisibleForTesting + protected AutoRepairService() + { + } + + public static void setup() + { + instance.config = DatabaseDescriptor.getAutoRepairConfig(); + } + + static + { + MBeanWrapper.instance.registerMBean(instance, MBEAN_NAME); + } + + @Override + public AutoRepairConfig getAutoRepairConfig() + { + return config; + } + + @Override + public void setAutoRepairEnabled(RepairType repairType, boolean enabled) + { + config.setAutoRepairEnabled(repairType, enabled); + } + + @Override + public void setRepairThreads(RepairType repairType, int repairThreads) + { + config.setRepairThreads(repairType, repairThreads); + } + + @Override + public void setRepairPriorityForHosts(RepairType repairType, Set hosts) + { + AutoRepairUtils.addPriorityHosts(repairType, hosts); + } + + @Override + public Set getRepairHostPriority(RepairType repairType) { + return AutoRepairUtils.getPriorityHosts(repairType); + } + + @Override + public void setForceRepairForHosts(RepairType repairType, Set hosts) + { + AutoRepairUtils.setForceRepair(repairType, hosts); + } + + @Override + public void setRepairSubRangeNum(RepairType repairType, int repairSubRanges) + { + config.setRepairSubRangeNum(repairType, repairSubRanges); + } + + @Override + public void setRepairMinInterval(RepairType repairType, String minRepairInterval) + { + config.setRepairMinInterval(repairType, minRepairInterval); + } + + public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration) + { + config.setAutoRepairHistoryClearDeleteHostsBufferInterval(duration); + } + + @Override + public void setRepairSSTableCountHigherThreshold(RepairType repairType, int sstableHigherThreshold) + { + config.setRepairSSTableCountHigherThreshold(repairType, sstableHigherThreshold); + } + + @Override + public void setAutoRepairTableMaxRepairTime(RepairType repairType, String autoRepairTableMaxRepairTime) + { + config.setAutoRepairTableMaxRepairTime(repairType, autoRepairTableMaxRepairTime); + } + + @Override + public void setIgnoreDCs(RepairType repairType, Set ignoreDCs) + { + config.setIgnoreDCs(repairType, ignoreDCs); + } + + @Override + public void setPrimaryTokenRangeOnly(RepairType repairType, boolean primaryTokenRangeOnly) + { + config.setRepairPrimaryTokenRangeOnly(repairType, primaryTokenRangeOnly); + } + + @Override + public void setParallelRepairPercentageInGroup(RepairType repairType, int percentageInGroup) + { + config.setParallelRepairPercentageInGroup(repairType, percentageInGroup); + } + + @Override + public void setParallelRepairCountInGroup(RepairType repairType, int countInGroup) + { + config.setParallelRepairCountInGroup(repairType, countInGroup); + } + + public void setMVRepairEnabled(RepairType repairType, boolean enabled) + { + config.setMVRepairEnabled(repairType, enabled); + } +} diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java new file mode 100644 index 000000000000..5dda1b157517 --- /dev/null +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service; + +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; + +import java.util.Set; + +public interface AutoRepairServiceMBean +{ + /** + * Enable or disable auto-repair for a given repair type + */ + public void setAutoRepairEnabled(RepairType repairType, boolean enabled); + + public void setRepairThreads(RepairType repairType, int repairThreads); + + public void setRepairPriorityForHosts(RepairType repairType, Set host); + + public void setForceRepairForHosts(RepairType repairType, Set host); + + public Set getRepairHostPriority(RepairType repairType); + + public void setRepairSubRangeNum(RepairType repairType, int repairSubRangeNum); + + public void setRepairMinInterval(RepairType repairType, String minRepairInterval); + + public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration); + + public void setRepairSSTableCountHigherThreshold(RepairType repairType, int ssTableHigherThreshold); + + public void setAutoRepairTableMaxRepairTime(RepairType repairType, String autoRepairTableMaxRepairTime); + public void setIgnoreDCs(RepairType repairType, Set ignorDCs); + + public void setPrimaryTokenRangeOnly(RepairType repairType, boolean primaryTokenRangeOnly); + + public void setParallelRepairPercentageInGroup(RepairType repairType, int percentageInGroup); + public void setParallelRepairCountInGroup(RepairType repairType, int countInGroup); + + public void setMVRepairEnabled(RepairType repairType, boolean enabled); + + public AutoRepairConfig getAutoRepairConfig(); +} diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index c3820d1766ed..9928eac0eef7 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -84,6 +84,8 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.RateLimiter; import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.cassandra.repair.autorepair.AutoRepairKeyspace; +import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -393,7 +395,7 @@ public enum Mode { STARTING, NORMAL, JOINING, LEAVING, DECOMMISSIONED, MOVING, D /* Used for tracking drain progress */ private volatile int totalCFs, remainingCFs; - private static final AtomicInteger nextRepairCommand = new AtomicInteger(); + public static final AtomicInteger nextRepairCommand = new AtomicInteger(); private final List lifecycleSubscribers = new CopyOnWriteArrayList<>(); @@ -1309,6 +1311,8 @@ public void finishJoiningRing(boolean didBootstrap, Collection tokens) setTokens(tokens); assert tokenMetadata.sortedTokens().size() > 0; + + doAutoRepairSetup(); } @VisibleForTesting @@ -1331,6 +1335,17 @@ public void doAuthSetup(boolean setUpSchema) } } + private void doAutoRepairSetup() + { + if (DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + { + logger.info("Enable auto-repair scheduling"); + AutoRepair.instance.setup(); + } + logger.info("AutoRepair setup complete!"); + } + + public boolean isAuthSetupComplete() { return authSetupComplete; @@ -1349,6 +1364,10 @@ public void setUpDistributedSystemKeyspaces() Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(TraceKeyspace.metadata(), TraceKeyspace.GENERATION)); Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(SystemDistributedKeyspace.metadata(), SystemDistributedKeyspace.GENERATION)); Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(AuthKeyspace.metadata(), AuthKeyspace.GENERATION)); + if (DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + { + Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(AutoRepairKeyspace.metadata(), AutoRepairKeyspace.GENERATION)); + } } public boolean isJoined() diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 4a2f43d882ac..d4f6364d89f8 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -85,13 +85,17 @@ import org.apache.cassandra.hints.HintsServiceMBean; import org.apache.cassandra.locator.DynamicEndpointSnitchMBean; import org.apache.cassandra.locator.EndpointSnitchInfoMBean; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.metrics.StorageMetrics; import org.apache.cassandra.metrics.TableMetrics; import org.apache.cassandra.metrics.ThreadPoolMetrics; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.MessagingServiceMBean; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.service.ActiveRepairServiceMBean; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.service.AutoRepairServiceMBean; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.service.CacheServiceMBean; import org.apache.cassandra.service.GCInspector; @@ -154,6 +158,8 @@ public class NodeProbe implements AutoCloseable protected NetworkPermissionsCacheMBean npcProxy; protected PermissionsCacheMBean pcProxy; protected RolesCacheMBean rcProxy; + protected AutoRepairServiceMBean autoRepairProxy; + protected Output output; private boolean failed; @@ -278,6 +284,8 @@ protected void connect() throws IOException pcProxy = JMX.newMBeanProxy(mbeanServerConn, name, PermissionsCacheMBean.class); name = new ObjectName(AuthCache.MBEAN_NAME_BASE + RolesCache.CACHE_NAME); rcProxy = JMX.newMBeanProxy(mbeanServerConn, name, RolesCacheMBean.class); + name = new ObjectName(AutoRepairService.MBEAN_NAME); + autoRepairProxy = JMX.newMBeanProxy(mbeanServerConn, name, AutoRepairServiceMBean.class); } catch (MalformedObjectNameException e) { @@ -2137,6 +2145,82 @@ public int getDefaultKeyspaceReplicationFactor() { return ssProxy.getDefaultKeyspaceReplicationFactor(); } + + public AutoRepairConfig getAutoRepairConfig() { + return autoRepairProxy.getAutoRepairConfig(); + } + + public void setAutoRepairEnabled(AutoRepairConfig.RepairType repairType, boolean enabled) + { + autoRepairProxy.setAutoRepairEnabled(repairType, enabled); + } + + public void setRepairThreads(AutoRepairConfig.RepairType repairType, int repairThreads) + { + autoRepairProxy.setRepairThreads(repairType, repairThreads); + } + + public void setRepairPriorityForHosts(AutoRepairConfig.RepairType repairType, Set hosts) + { + autoRepairProxy.setRepairPriorityForHosts(repairType, hosts); + } + + public Set getRepairPriorityForHosts(AutoRepairConfig.RepairType repairType) + { + return autoRepairProxy.getRepairHostPriority(repairType); + } + + public void setForceRepairForHosts(AutoRepairConfig.RepairType repairType, Set hosts){ + autoRepairProxy.setForceRepairForHosts(repairType, hosts); + } + + public void setRepairSubRangeNum(AutoRepairConfig.RepairType repairType, int repairSubRanges) + { + autoRepairProxy.setRepairSubRangeNum(repairType, repairSubRanges); + } + + public void setRepairMinInterval(AutoRepairConfig.RepairType repairType, String minRepairInterval) + { + autoRepairProxy.setRepairMinInterval(repairType, minRepairInterval); + } + + public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration) + { + autoRepairProxy.setAutoRepairHistoryClearDeleteHostsBufferDuration(duration); + } + + public void setRepairSSTableCountHigherThreshold(AutoRepairConfig.RepairType repairType, int ssTableHigherThreshold) + { + autoRepairProxy.setRepairSSTableCountHigherThreshold(repairType, ssTableHigherThreshold); + } + + public void setAutoRepairTableMaxRepairTime(AutoRepairConfig.RepairType repairType, String autoRepairTableMaxRepairTime) + { + autoRepairProxy.setAutoRepairTableMaxRepairTime(repairType, autoRepairTableMaxRepairTime); + } + + public void setAutoRepairIgnoreDCs(AutoRepairConfig.RepairType repairType, Set ignoreDCs) + { + autoRepairProxy.setIgnoreDCs(repairType, ignoreDCs); + } + + public void setParallelRepairPercentageInGroup(AutoRepairConfig.RepairType repairType, int percentageInGroup) { + autoRepairProxy.setParallelRepairPercentageInGroup(repairType, percentageInGroup); + } + + public void setParallelRepairCountInGroup(AutoRepairConfig.RepairType repairType, int countInGroup) { + autoRepairProxy.setParallelRepairCountInGroup(repairType, countInGroup); + } + + public void setPrimaryTokenRangeOnly(AutoRepairConfig.RepairType repairType, boolean primaryTokenRangeOnly) + { + autoRepairProxy.setPrimaryTokenRangeOnly(repairType, primaryTokenRangeOnly); + } + + public void setMVRepairEnabled(AutoRepairConfig.RepairType repairType, boolean enabled) + { + autoRepairProxy.setMVRepairEnabled(repairType, enabled); + } } class ColumnFamilyStoreMBeanIterator implements Iterator> diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index 8d87c88906b1..dd3291b6d2ac 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -131,6 +131,7 @@ public int execute(String... args) GcStats.class, GetAuditLog.class, GetAuthCacheConfig.class, + GetAutoRepairConfig.class, GetBatchlogReplayTrottle.class, GetColumnIndexSize.class, GetCompactionThreshold.class, @@ -189,6 +190,7 @@ public int execute(String... args) Ring.class, Scrub.class, SetAuthCacheConfig.class, + SetAutoRepairConfig.class, SetBatchlogReplayThrottle.class, SetCacheCapacity.class, SetCacheKeysToSave.class, diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java new file mode 100644 index 000000000000..49ccf0f2aa24 --- /dev/null +++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools.nodetool; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import io.airlift.airline.Command; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.NodeTool.NodeToolCmd; + +import java.io.PrintStream; + +@Command(name = "getautorepairconfig", description = "Print autorepair configurations") +public class GetAutoRepairConfig extends NodeToolCmd +{ + @VisibleForTesting + protected static PrintStream out = System.out; + + @Override + public void execute(NodeProbe probe) + { + AutoRepairConfig config = probe.getAutoRepairConfig(); + if (config == null || !config.isAutoRepairSchedulingEnabled()) + { + out.println("Auto-repair is not enabled"); + return; + } + + StringBuilder sb = new StringBuilder(); + sb.append("repair scheduler configuration:"); + sb.append("\n\trepair eligibility check interval: " + config.getRepairCheckInterval()); + sb.append("\n\tTTL for repair history for dead nodes: " + config.getAutoRepairHistoryClearDeleteHostsBufferInterval()); + for (RepairType repairType : RepairType.values()) + { + sb.append(formatRepairTypeConfig(probe, repairType, config)); + } + + out.println(sb); + } + + private String formatRepairTypeConfig(NodeProbe probe, RepairType repairType, AutoRepairConfig config) + { + StringBuilder sb = new StringBuilder(); + sb.append("\nconfiguration for repair type: " + repairType); + sb.append("\n\tenabled: " + config.isAutoRepairEnabled(repairType)); + sb.append("\n\tminimum repair interval: " + config.getRepairMinInterval(repairType)); + sb.append("\n\trepair threads: " + config.getRepairThreads(repairType)); + sb.append("\n\tnumber of repair subranges: " + config.getRepairSubRangeNum(repairType)); + sb.append("\n\tpriority hosts: " + Joiner.on(',').skipNulls().join(probe.getRepairPriorityForHosts(repairType))); + sb.append("\n\tsstable count higher threshold: " + config.getRepairSSTableCountHigherThreshold(repairType)); + sb.append("\n\ttable max repair time in sec: " + config.getAutoRepairTableMaxRepairTime(repairType)); + sb.append("\n\tignore datacenters: " + Joiner.on(',').skipNulls().join(config.getIgnoreDCs(repairType))); + sb.append("\n\trepair primary token-range: " + config.getRepairPrimaryTokenRangeOnly(repairType)); + sb.append("\n\tnumber of parallel repairs within group: " + config.getParallelRepairCountInGroup(repairType)); + sb.append("\n\tpercentage of parallel repairs within group: " + config.getParallelRepairPercentageInGroup(repairType)); + sb.append("\n\tmv repair enabled: " + config.getMVRepairEnabled(repairType)); + + return sb.toString(); + } +} diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java new file mode 100644 index 000000000000..bd00809dcf22 --- /dev/null +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools.nodetool; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Splitter; +import io.airlift.airline.Arguments; +import io.airlift.airline.Command; +import io.airlift.airline.Option; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.NodeTool.NodeToolCmd; + +import java.io.PrintStream; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkArgument; + +@Command(name = "setautorepairconfig", description = "sets the autorepair configuration") +public class SetAutoRepairConfig extends NodeToolCmd +{ + @VisibleForTesting + @Arguments(title = " ", usage = " ", + description = "autorepair param and value.\nPossible autorepair parameters are as following: " + + "[number_of_repair_threads|number_of_subranges|min_repair_interval|sstable_upper_threshold" + + "|enabled|table_max_repair_time|priority_hosts|forcerepair_hosts|ignore_dcs" + + "|history_clear_delete_hosts_buffer_interval|repair_primary_token_range_only|parallel_repair_count_in_group|parallel_repair_percentage_in_group|mv_repair_enabled]", + required = true) + protected List args = new ArrayList<>(); + + @VisibleForTesting + @Option(title = "repair type", name = { "-t", "--repair-type" }, description = "Repair type") + protected RepairType repairType; + + @VisibleForTesting + protected PrintStream out = System.out; + + @Override + public void execute(NodeProbe probe) + { + checkArgument(repairType != null, "--repair-type is required."); + checkArgument(args.size() == 2, "setautorepairconfig requires param-type, and value args."); + String paramType = args.get(0); + String paramVal = args.get(1); + + if (!probe.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + { + out.println("Auto-repair is not enabled"); + return; + } + + if (paramType.equals("history_clear_delete_hosts_buffer_interval")) + { + probe.setAutoRepairHistoryClearDeleteHostsBufferDuration(paramVal); + return; + } + + // options below require --repair-type option + checkArgument(repairType != null, "--repair-type is required for this parameter."); + Set hosts; + switch (paramType) + { + case "enabled": + probe.setAutoRepairEnabled(repairType, Boolean.parseBoolean(paramVal)); + break; + case "number_of_repair_threads": + probe.setRepairThreads(repairType, Integer.parseInt(paramVal)); + break; + case "number_of_subranges": + probe.setRepairSubRangeNum(repairType, Integer.parseInt(paramVal)); + break; + case "min_repair_interval": + probe.setRepairMinInterval(repairType, paramVal); + break; + case "sstable_upper_threshold": + probe.setRepairSSTableCountHigherThreshold(repairType, Integer.parseInt(paramVal)); + break; + case "table_max_repair_time": + probe.setAutoRepairTableMaxRepairTime(repairType, paramVal); + break; + case "priority_hosts": + hosts = validateLocalGroupHosts(probe, repairType, paramVal); + if (!hosts.isEmpty()) + { + probe.setRepairPriorityForHosts(repairType, hosts); + } + break; + case "forcerepair_hosts": + hosts = validateLocalGroupHosts(probe, repairType, paramVal); + if (!hosts.isEmpty()) + { + probe.setForceRepairForHosts(repairType, hosts); + } + break; + case "ignore_dcs": + Set ignoreDCs = new HashSet<>(); + for (String dc : Splitter.on(',').split(paramVal)) + { + ignoreDCs.add(dc); + } + probe.setAutoRepairIgnoreDCs(repairType, ignoreDCs); + break; + case "repair_primary_token_range_only": + probe.setPrimaryTokenRangeOnly(repairType, Boolean.parseBoolean(paramVal)); + break; + case "parallel_repair_count_in_group": + probe.setParallelRepairCountInGroup(repairType, Integer.parseInt(paramVal)); + break; + case "parallel_repair_percentage_in_group": + probe.setParallelRepairPercentageInGroup(repairType, Integer.parseInt(paramVal)); + break; + case "mv_repair_enabled": + probe.setMVRepairEnabled(repairType, Boolean.parseBoolean(paramVal)); + break; + default: + throw new IllegalArgumentException("Unknown parameter: " + paramType); + } + } + + private Set validateLocalGroupHosts(NodeProbe probe, RepairType repairType, String paramVal) { + Set hosts = new HashSet<>(); + for (String host : Splitter.on(',').split(paramVal)) + { + try + { + hosts.add(InetAddressAndPort.getByName(host)); + } + catch (UnknownHostException e) + { + out.println("invalid ip address: " + host); + } + } + + return hosts; + } +} diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index 3d881ddf3f59..d3764778da9e 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -53,6 +53,7 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileInputStreamPlus; import org.apache.cassandra.io.util.FileOutputStreamPlus; +import org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter; import org.apache.cassandra.utils.concurrent.*; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -685,6 +686,13 @@ public static ISslContextFactory newSslContextFactory(String className, Map !Modifier.isStatic(f.getModifiers())) - .filter(isDurationSpec.or(isDataRateSpec).or(isDataStorageSpec)).count(), 0); + .filter(f -> !Modifier.isStatic(f.getModifiers())) + .filter(isDurationSpec.or(isDataRateSpec).or(isDataStorageSpec)).count(), 0); } @Test @@ -65,12 +68,12 @@ public void updateInPlace() { Config config = new Config(); Map map = ImmutableMap.builder().put("storage_port", 123) - .put("commitlog_sync", Config.CommitLogSync.batch) - .put("seed_provider.class_name", "org.apache.cassandra.locator.SimpleSeedProvider") - .put("client_encryption_options.cipher_suites", Collections.singletonList("FakeCipher")) - .put("client_encryption_options.optional", false) - .put("client_encryption_options.enabled", true) - .build(); + .put("commitlog_sync", Config.CommitLogSync.batch) + .put("seed_provider.class_name", "org.apache.cassandra.locator.SimpleSeedProvider") + .put("client_encryption_options.cipher_suites", Collections.singletonList("FakeCipher")) + .put("client_encryption_options.optional", false) + .put("client_encryption_options.enabled", true) + .build(); Config updated = YamlConfigurationLoader.updateFromMap(map, true, config); assert updated == config : "Config pointers do not match"; assertThat(config.storage_port).isEqualTo(123); @@ -225,6 +228,12 @@ public void fromMapTest() Map encryptionOptions = ImmutableMap.of("cipher_suites", Collections.singletonList("FakeCipher"), "optional", false, "enabled", true); + Map autoRepairConfig = ImmutableMap.of("enabled", true, + "global_settings", ImmutableMap.of("repair_dc_groups", + ImmutableSet.of("all the groups")), + "repair_type_overrides", ImmutableMap.of( + "full", ImmutableMap.of("repair_dc_groups", + ImmutableSet.of("none of the groups")))); Map map = new ImmutableMap.Builder() .put("storage_port", storagePort) .put("commitlog_sync", commitLogSync) @@ -233,6 +242,7 @@ public void fromMapTest() .put("internode_socket_send_buffer_size", "5B") .put("internode_socket_receive_buffer_size", "5B") .put("commitlog_sync_group_window_in_ms", "42") + .put("auto_repair", autoRepairConfig) .build(); Config config = YamlConfigurationLoader.fromMap(map, Config.class); @@ -243,6 +253,9 @@ public void fromMapTest() assertEquals(true, config.client_encryption_options.enabled); // Check a nested object assertEquals(new DataStorageSpec.IntBytesBound("5B"), config.internode_socket_send_buffer_size); // Check names backward compatibility (CASSANDRA-17141 and CASSANDRA-15234) assertEquals(new DataStorageSpec.IntBytesBound("5B"), config.internode_socket_receive_buffer_size); // Check names backward compatibility (CASSANDRA-17141 and CASSANDRA-15234) + assertEquals(true, config.auto_repair.enabled); + assertEquals(6 * 60 * 60L, config.auto_repair.getAutoRepairTableMaxRepairTime(AutoRepairConfig.RepairType.incremental)); + config.auto_repair.setMVRepairEnabled(AutoRepairConfig.RepairType.incremental, false); } @Test @@ -384,4 +397,4 @@ private static Config load(String path) } return new YamlConfigurationLoader().loadConfig(url); } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java new file mode 100644 index 000000000000..dea0c9a827c3 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -0,0 +1,453 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.EnumMap; +import java.util.Objects; +import java.util.Set; + +import com.google.common.collect.ImmutableSet; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.Options; +import org.apache.cassandra.utils.FBUtilities; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +@RunWith(Parameterized.class) +public class AutoRepairConfigTest extends CQLTester +{ + private AutoRepairConfig config; + + private Set testSet = ImmutableSet.of("dc1"); + + @Parameterized.Parameter + public AutoRepairConfig.RepairType repairType; + + @Parameterized.Parameters + public static Object[] repairTypes() + { + return AutoRepairConfig.RepairType.values(); + } + + @Before + public void setUp() + { + config = new AutoRepairConfig(true); + config.repair_type_overrides = null; + } + + @Test + public void autoRepairConfigDefaultsAreNotNull() + { + AutoRepairConfig config = new AutoRepairConfig(); + assertNotNull(config.global_settings); + } + + @Test + public void autoRepairConfigRepairTypesAreNotNull() + { + AutoRepairConfig config = new AutoRepairConfig(); + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + assertNotNull(config.repair_type_overrides.get(repairType)); + } + } + + @Test + public void testIsAutoRepairEnabledReturnsTrueWhenRepairIsEnabled() + { + config.global_settings.enabled = true; + + assertTrue(config.isAutoRepairEnabled(repairType)); + } + + @Test + public void testIsAutoRepairEnabledReturnsTrueWhenRepairIsDisabledGlobally() + { + config = new AutoRepairConfig(false); + config.global_settings.enabled = true; + assertFalse(config.isAutoRepairEnabled(repairType)); + } + + + @Test + public void testIsAutoRepairEnabledReturnsTrueWhenRepairIsDisabledForRepairType() + { + config.global_settings.enabled = true; + config.repair_type_overrides = new EnumMap<>(AutoRepairConfig.RepairType.class); + config.repair_type_overrides.put(repairType, new Options()); + config.repair_type_overrides.get(repairType).enabled = false; + assertFalse(config.isAutoRepairEnabled(repairType)); + } + + @Test + public void testSetAutoRepairEnabledNoMVOrCDC() + { + DatabaseDescriptor.setCDCEnabled(false); + DatabaseDescriptor.setMaterializedViewsEnabled(false); + config.setAutoRepairEnabled(repairType, true); + + assertTrue(config.repair_type_overrides.get(repairType).enabled); + } + + @Test + public void testSetAutoRepairEnabledWithMV() + { + DatabaseDescriptor.setCDCEnabled(false); + DatabaseDescriptor.setMaterializedViewsEnabled(true); + + try + { + config.setAutoRepairEnabled(repairType, true); + + if (repairType == AutoRepairConfig.RepairType.incremental) + { + assertFalse(config.repair_type_overrides.get(repairType).enabled); // IR should not be allowed with MV + assertNotEquals(AutoRepairConfig.RepairType.incremental, repairType); // should receive exception + } + else + { + assertTrue(config.repair_type_overrides.get(repairType).enabled); + } + } + catch (ConfigurationException e) + { + // should throw only if repairType is incremental + assertEquals(AutoRepairConfig.RepairType.incremental, repairType); + } + } + + @Test + public void testSetAutoRepairEnabledWithCDC() + { + DatabaseDescriptor.setCDCEnabled(true); + DatabaseDescriptor.setMaterializedViewsEnabled(false); + + try + { + config.setAutoRepairEnabled(repairType, true); + + + if (repairType == AutoRepairConfig.RepairType.incremental) + { + assertFalse(config.repair_type_overrides.get(repairType).enabled); // IR should not be allowed with CDC + assertNotEquals(AutoRepairConfig.RepairType.incremental, repairType); // should receive exception + } + else + { + assertTrue(config.repair_type_overrides.get(repairType).enabled); + } + } + catch (ConfigurationException e) + { + // should throw only if repairType is incremental + assertEquals(AutoRepairConfig.RepairType.incremental, repairType); + } + } + + + @Test + public void testSetRepairByKeyspace() + { + config.setRepairByKeyspace(repairType, true); + + assertTrue(config.repair_type_overrides.get(repairType).repair_by_keyspace); + } + + @Test + public void testGetRepairByKeyspace() + { + config.global_settings.repair_by_keyspace = true; + + boolean result = config.getRepairByKeyspace(repairType); + + assertTrue(result); + } + + @Test + public void testSetRepairThreads() + { + config.setRepairThreads(repairType, 5); + + assert config.repair_type_overrides.get(repairType).number_of_repair_threads == 5; + } + + @Test + public void testGetRepairThreads() + { + config.global_settings.number_of_repair_threads = 5; + + int result = config.getRepairThreads(repairType); + + assertEquals(5, result); + } + + @Test + public void testGetRepairSubRangeNum() + { + config.global_settings.number_of_subranges = 5; + + int result = config.getRepairSubRangeNum(repairType); + + assertEquals(5, result); + } + + @Test + public void testSetRepairSubRangeNum() + { + config.setRepairSubRangeNum(repairType, 5); + + assert config.repair_type_overrides.get(repairType).number_of_subranges == 5; + } + + @Test + public void testGetRepairMinFrequencyInHours() + { + config.global_settings.min_repair_interval = new DurationSpec.IntSecondsBound("5s"); + + DurationSpec.IntSecondsBound result = config.getRepairMinInterval(repairType); + + assertEquals(5, result.toSeconds()); + } + + @Test + public void testSetRepairMinFrequencyInHours() + { + config.setRepairMinInterval(repairType, "5s"); + + assert config.repair_type_overrides.get(repairType).min_repair_interval.toSeconds() == 5; + } + + @Test + public void testGetAutoRepairHistoryClearDeleteHostsBufferInSec() + { + config.history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound("5s"); + + int result = config.getAutoRepairHistoryClearDeleteHostsBufferInterval().toSeconds(); + + assertEquals(5, result); + } + + @Test + public void testSetAutoRepairHistoryClearDeleteHostsBufferInSec() + { + config.setAutoRepairHistoryClearDeleteHostsBufferInterval("5s"); + + assert Objects.equals(config.history_clear_delete_hosts_buffer_interval, new DurationSpec.IntSecondsBound("5s")); + } + + @Test + public void testGetRepairSSTableCountHigherThreshold() + { + config.global_settings.sstable_upper_threshold = 5; + + int result = config.getRepairSSTableCountHigherThreshold(repairType); + + assertEquals(5, result); + } + + @Test + public void testSetRepairSSTableCountHigherThreshold() + { + config.setRepairSSTableCountHigherThreshold(repairType, 5); + + assert config.repair_type_overrides.get(repairType).sstable_upper_threshold == 5; + } + + @Test + public void testGetAutoRepairTableMaxRepairTimeInSec() + { + config.global_settings.table_max_repair_time = new DurationSpec.IntSecondsBound("5s"); + + DurationSpec.IntSecondsBound result = config.getAutoRepairTableMaxRepairTime(repairType); + + assertEquals(5, result.toSeconds()); + } + + @Test + public void testSetAutoRepairTableMaxRepairTimeInSec() + { + config.setAutoRepairTableMaxRepairTime(repairType, "5s"); + + assert config.repair_type_overrides.get(repairType).table_max_repair_time.toSeconds() == 5; + } + + @Test + public void testGetIgnoreDCs() + { + config.global_settings.ignore_dcs = testSet; + + Set result = config.getIgnoreDCs(repairType); + + assertEquals(testSet, result); + } + + @Test + public void testSetIgnoreDCs() + { + config.setIgnoreDCs(repairType, testSet); + + assertEquals(config.repair_type_overrides.get(repairType).ignore_dcs, testSet); + } + + @Test + public void testGetRepairPrimaryTokenRangeOnly() + { + config.global_settings.repair_primary_token_range_only = true; + + boolean result = config.getRepairPrimaryTokenRangeOnly(repairType); + + assertTrue(result); + } + + @Test + public void testSetRepairPrimaryTokenRangeOnly() + { + config.setRepairPrimaryTokenRangeOnly(repairType, true); + + assertTrue(config.repair_type_overrides.get(repairType).repair_primary_token_range_only); + } + + @Test + public void testGetParallelRepairPercentageInGroup() + { + config.global_settings.parallel_repair_percentage_in_group = 5; + + int result = config.getParallelRepairPercentageInGroup(repairType); + + assertEquals(5, result); + } + + @Test + public void testSetParallelRepairPercentageInGroup() + { + config.setParallelRepairPercentageInGroup(repairType, 5); + + assert config.repair_type_overrides.get(repairType).parallel_repair_percentage_in_group == 5; + } + + @Test + public void testGetParallelRepairCountInGroup() + { + config.global_settings.parallel_repair_count_in_group = 5; + + int result = config.getParallelRepairCountInGroup(repairType); + + assertEquals(5, result); + } + + @Test + public void testSetParallelRepairCountInGroup() + { + config.setParallelRepairCountInGroup(repairType, 5); + + assert config.repair_type_overrides.get(repairType).parallel_repair_count_in_group == 5; + } + + @Test + public void testGetMVRepairEnabled() + { + config.global_settings.mv_repair_enabled = true; + + boolean result = config.getMVRepairEnabled(repairType); + + assertTrue(result); + } + + @Test + public void testSetMVRepairEnabled() + { + config.setMVRepairEnabled(repairType, true); + + assertTrue(config.repair_type_overrides.get(repairType).mv_repair_enabled); + } + + @Test + public void testSetForceRepairNewNode() + { + config.setForceRepairNewNode(repairType, true); + + assertTrue(config.repair_type_overrides.get(repairType).force_repair_new_node); + } + + @Test + public void testGetForceRepairNewNode() + { + config.global_settings.force_repair_new_node = true; + + boolean result = config.getForceRepairNewNode(repairType); + + assertTrue(result); + } + + @Test + public void testIsAutoRepairSchedulingEnabledDefault() + { + config = new AutoRepairConfig(); + + boolean result = config.isAutoRepairSchedulingEnabled(); + + assertFalse(result); + } + + @Test + public void testIsAutoRepairSchedulingEnabledTrue() + { + boolean result = config.isAutoRepairSchedulingEnabled(); + + assertTrue(result); + } + + @Test + public void testGetDefaultOptionsMVRepairIsEnabledByDefault() + { + Options defaultOptions = Options.getDefaultOptions(); + + assertTrue(defaultOptions.mv_repair_enabled); + } + + @Test + public void testGetDefaultOptionsTokenRangeSplitter() + { + Options defaultOptions = Options.getDefaultOptions(); + + assertEquals(DefaultAutoRepairTokenSplitter.class.getName(),defaultOptions.token_range_splitter); + assertEquals(DefaultAutoRepairTokenSplitter.class.getName(), FBUtilities.newAutoRepairTokenRangeSplitter(defaultOptions.token_range_splitter).getClass().getName()); + } + + @Test(expected = ConfigurationException.class) + public void testInvalidTokenRangeSplitter() + { + assertEquals(DefaultAutoRepairTokenSplitter.class.getName(), FBUtilities.newAutoRepairTokenRangeSplitter("invalid-class").getClass().getName()); + } + +} diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java new file mode 100644 index 000000000000..579b996a12f5 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.Optional; +import java.util.Set; + +import com.google.common.collect.ImmutableSet; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.TableMetadata; + +import static org.apache.cassandra.Util.setAutoRepairEnabled; + +public class AutoRepairKeyspaceTest +{ + private static final Set tables = ImmutableSet.of( + AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + AutoRepairKeyspace.AUTO_REPAIR_PRIORITY + ); + + @BeforeClass + public static void setupDatabaseDescriptor() + { + DatabaseDescriptor.daemonInitialization(); + } + + + @Test + public void testMetadataCanParseSchemas() throws Exception + { + setAutoRepairEnabled(true); + KeyspaceMetadata keyspaceMetadata = AutoRepairKeyspace.metadata(); + + assert keyspaceMetadata.tables.size() == tables.size() : "Expected " + tables.size() + " tables, got " + keyspaceMetadata.tables.size(); + + for (String table : tables) + { + Optional tableMetadata = keyspaceMetadata.tables.get(table); + + assert tableMetadata.isPresent() : "Table " + table + " not found in metadata"; + } + } +} diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java new file mode 100644 index 000000000000..7a522a3458ae --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -0,0 +1,581 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; + +import org.apache.cassandra.cql3.statements.schema.TableAttributes; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.repair.RepairRunnable; +import org.apache.cassandra.schema.AutoRepairParams; +import org.apache.cassandra.schema.TableParams; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.Pair; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import junit.framework.Assert; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.concurrent.ScheduledExecutorPlus; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.metrics.AutoRepairMetricsManager; +import org.apache.cassandra.metrics.AutoRepairMetrics; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.progress.ProgressEvent; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import static org.apache.cassandra.Util.setAutoRepairEnabled; +import static org.apache.cassandra.Util.token; +import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; +import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.NOT_MY_TURN; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(Parameterized.class) +public class AutoRepairParameterizedTest extends CQLTester +{ + private static final String KEYSPACE = "ks"; + private static final String TABLE = "tbl"; + private static final String TABLE_DISABLED_AUTO_REPAIR = "tbl_disabled_auto_repair"; + private static final String MV = "mv"; + private static TableMetadata cfm; + private static TableMetadata cfmDisabledAutoRepair; + private static Keyspace keyspace; + private static int timeFuncCalls; + @Mock + ScheduledExecutorPlus mockExecutor; + @Mock + ProgressEvent progressEvent; + @Mock + AutoRepairState autoRepairState; + @Mock + RepairRunnable repairRunnable; + private static AutoRepairConfig defaultConfig; + + + @Parameterized.Parameter() + public AutoRepairConfig.RepairType repairType; + + @Parameterized.Parameters(name = "repairType={0}") + public static Collection repairTypes() + { + return Arrays.asList(AutoRepairConfig.RepairType.values()); + } + + @BeforeClass + public static void setupClass() throws Exception + { + SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); + setAutoRepairEnabled(true); + requireNetwork(); + AutoRepairUtils.setup(); + + + cfm = TableMetadata.builder(KEYSPACE, TABLE) + .addPartitionKeyColumn("k", UTF8Type.instance) + .addStaticColumn("s", UTF8Type.instance) + .addClusteringColumn("i", IntegerType.instance) + .addRegularColumn("v", UTF8Type.instance) + .params(TableParams.builder().automatedRepairFull(AutoRepairParams.create(AutoRepairConfig.RepairType.full, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))). + automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.incremental, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))).build()) + .build(); + + cfmDisabledAutoRepair = TableMetadata.builder(KEYSPACE, TABLE_DISABLED_AUTO_REPAIR) + .addPartitionKeyColumn("k", UTF8Type.instance) + .addStaticColumn("s", UTF8Type.instance) + .addClusteringColumn("i", IntegerType.instance) + .addRegularColumn("v", UTF8Type.instance) + .params(TableParams.builder().automatedRepairFull(AutoRepairParams.create(AutoRepairConfig.RepairType.full, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))). + automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.incremental, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))).build()) + .build(); + + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), cfm, cfmDisabledAutoRepair); + cfm = Schema.instance.getTableMetadata(KEYSPACE, TABLE); + cfmDisabledAutoRepair = Schema.instance.getTableMetadata(KEYSPACE, TABLE_DISABLED_AUTO_REPAIR); + keyspace = Keyspace.open(KEYSPACE); + DatabaseDescriptor.setMaterializedViewsEnabled(true); + QueryProcessor.executeInternal(String.format("CREATE MATERIALIZED VIEW %s.%s AS SELECT i, k from %s.%s " + + "WHERE k IS NOT null AND i IS NOT null PRIMARY KEY (i, k)", KEYSPACE, MV, KEYSPACE, TABLE)); + + defaultConfig = new AutoRepairConfig(true); + DatabaseDescriptor.setMaterializedViewsEnabled(false); + DatabaseDescriptor.setCDCEnabled(false); + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + defaultConfig.setAutoRepairEnabled(repairType, true); + defaultConfig.setMVRepairEnabled(repairType, false); + } + } + + @Before + public void setup() + { + MockitoAnnotations.initMocks(this); + + Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).truncateBlocking(); + Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).disableAutoCompaction(); + + Keyspace.open(KEYSPACE).getColumnFamilyStore(MV).truncateBlocking(); + Keyspace.open(KEYSPACE).getColumnFamilyStore(MV).disableAutoCompaction(); + + Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(AutoRepairKeyspace.AUTO_REPAIR_PRIORITY).truncateBlocking(); + Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(AutoRepairKeyspace.AUTO_REPAIR_HISTORY).truncateBlocking(); + + + AutoRepair.instance = new AutoRepair(); + executeCQL(); + + timeFuncCalls = 0; + AutoRepair.timeFunc = System::currentTimeMillis; + resetCounters(); + resetConfig(); + } + + + private void resetCounters() + { + AutoRepairMetrics metrics = AutoRepairMetricsManager.getMetrics(repairType); + Metrics.removeMatching((name, metric) -> name.startsWith("repairTurn")); + metrics.repairTurnMyTurn = Metrics.counter(String.format("repairTurnMyTurn-%s", repairType)); + metrics.repairTurnMyTurnForceRepair = Metrics.counter(String.format("repairTurnMyTurnForceRepair-%s", repairType)); + metrics.repairTurnMyTurnDueToPriority = Metrics.counter(String.format("repairTurnMyTurnDueToPriority-%s", repairType)); + } + + private void resetConfig() + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.repair_type_overrides = defaultConfig.repair_type_overrides; + config.global_settings = defaultConfig.global_settings; + config.history_clear_delete_hosts_buffer_interval = defaultConfig.history_clear_delete_hosts_buffer_interval; + config.setRepairSubRangeNum(repairType, 1); + } + + private void executeCQL() + { + QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, s) VALUES ('k', 's')"); + QueryProcessor.executeInternal("SELECT s FROM ks.tbl WHERE k='k'"); + Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME) + .getColumnFamilyStore(AutoRepairKeyspace.AUTO_REPAIR_PRIORITY) + .forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + } + + + @Test + public void testRepairAsync() + { + AutoRepair.instance.repairExecutors.put(repairType, mockExecutor); + + AutoRepair.instance.repairAsync(repairType, 60); + + verify(mockExecutor, Mockito.times(1)).submit(any(Runnable.class)); + } + + @Test + public void testRepairTurn() + { + UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); + Assert.assertTrue("Expected my turn for the repair", AutoRepairUtils.myTurnToRunRepair(repairType, myId) != NOT_MY_TURN); + } + + @Test + public void testRepair() + { + AutoRepairService.instance.getAutoRepairConfig().setRepairMinInterval(repairType, "0s"); + AutoRepair.instance.repair(repairType, 0); + assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); + assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); + long lastRepairTime = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + //if repair was done then lastRepairTime should be non-zero + Assert.assertTrue(String.format("Expected lastRepairTime > 0, actual value lastRepairTime %d", + lastRepairTime), lastRepairTime > 0); + } + + @Test + public void testTooFrequentRepairs() + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + //in the first round let repair run + config.setRepairMinInterval(repairType, "0s"); + AutoRepair.instance.repair(repairType, 0); + long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + int consideredTables = AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair(); + Assert.assertNotSame(String.format("Expected total repaired tables > 0, actual value %s ", consideredTables), + consideredTables, 0); + + //if repair was done in last 24 hours then it should not trigger another repair + config.setRepairMinInterval(repairType, "24h"); + AutoRepair.instance.repair(repairType, 0); + long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + Assert.assertEquals(String.format("Expected repair time to be same, actual value lastRepairTime1 %d, " + + "lastRepairTime2 %d", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); + consideredTables = AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair(); + Assert.assertEquals("Expected total repaired tables = 0, actual value: " + consideredTables, + consideredTables, 0); + assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); + assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); + } + + @Test + public void testNonFrequentRepairs() + { + Integer prevMetricsCount = AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue(); + AutoRepairState state = AutoRepair.instance.repairStates.get(repairType); + long prevCount = state.getTotalMVTablesConsideredForRepair(); + AutoRepairService.instance.getAutoRepairConfig().setRepairMinInterval(repairType, "0s"); + AutoRepair.instance.repair(repairType, 0); + long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + Assert.assertTrue(String.format("Expected lastRepairTime1 > 0, actual value lastRepairTime1 %d", + lastRepairTime1), lastRepairTime1 > 0); + UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); + Assert.assertTrue("Expected my turn for the repair", + AutoRepairUtils.myTurnToRunRepair(repairType, myId) != NOT_MY_TURN); + AutoRepair.instance.repair(repairType, 0); + long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + Assert.assertNotSame(String.format("Expected repair time to be same, actual value lastRepairTime1 %d, " + + "lastRepairTime2 ", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); + assertEquals(prevCount, state.getTotalMVTablesConsideredForRepair()); + assertEquals(prevMetricsCount, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue()); + } + + @Test + public void testGetPriorityHosts() + { + Integer prevMetricsCount = AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue(); + AutoRepairState state = AutoRepair.instance.repairStates.get(repairType); + long prevCount = state.getTotalMVTablesConsideredForRepair(); + AutoRepairService.instance.getAutoRepairConfig().setRepairMinInterval(repairType, "0s"); + Assert.assertSame(String.format("Priority host count is not same, actual value %d, expected value %d", + AutoRepairUtils.getPriorityHosts(repairType).size(), 0), AutoRepairUtils.getPriorityHosts(repairType).size(), 0); + UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); + Assert.assertTrue("Expected my turn for the repair", AutoRepairUtils.myTurnToRunRepair(repairType, myId) != + NOT_MY_TURN); + AutoRepair.instance.repair(repairType, 0); + AutoRepairUtils.addPriorityHosts(repairType, Sets.newHashSet(FBUtilities.getBroadcastAddressAndPort())); + AutoRepair.instance.repair(repairType, 0); + Assert.assertSame(String.format("Priority host count is not same actual value %d, expected value %d", + AutoRepairUtils.getPriorityHosts(repairType).size(), 0), AutoRepairUtils.getPriorityHosts(repairType).size(), 0); + assertEquals(prevCount, state.getTotalMVTablesConsideredForRepair()); + assertEquals(prevMetricsCount, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue()); + } + + @Test + public void testCheckAutoRepairStartStop() throws Throwable + { + Integer prevMetricsCount = AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue(); + AutoRepairState state = AutoRepair.instance.repairStates.get(repairType); + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + long prevCount = state.getTotalMVTablesConsideredForRepair(); + config.setRepairMinInterval(repairType, "0s"); + config.setAutoRepairEnabled(repairType, false); + long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + AutoRepair.instance.repair(repairType, 0); + long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + //Since repair has not happened, both the last repair times should be same + Assert.assertEquals(String.format("Expected lastRepairTime1 %d, and lastRepairTime2 %d to be same", + lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); + + config.setAutoRepairEnabled(repairType, true); + AutoRepair.instance.repair(repairType, 0); + //since repair is done now, so lastRepairTime1/lastRepairTime2 and lastRepairTime3 should not be same + long lastRepairTime3 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + Assert.assertNotSame(String.format("Expected lastRepairTime1 %d, and lastRepairTime3 %d to be not same", + lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime3); + assertEquals(prevCount, state.getTotalMVTablesConsideredForRepair()); + assertEquals(prevMetricsCount, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue()); + } + + @Test + public void testRepairPrimaryRangesByDefault() + { + Assert.assertTrue("Expected primary range repair only", + AutoRepairService.instance.getAutoRepairConfig().getRepairPrimaryTokenRangeOnly(repairType)); + } + + @Test + public void testGetAllMVs() + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setMVRepairEnabled(repairType, false); + assertFalse(config.getMVRepairEnabled(repairType)); + assertEquals(0, AutoRepairUtils.getAllMVs(repairType, keyspace, cfm).size()); + + config.setMVRepairEnabled(repairType, true); + + assertTrue(config.getMVRepairEnabled(repairType)); + assertEquals(Arrays.asList(MV), AutoRepairUtils.getAllMVs(repairType, keyspace, cfm)); + config.setMVRepairEnabled(repairType, false); + } + + + @Test + public void testMVRepair() + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setMVRepairEnabled(repairType, true); + AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(0); + AutoRepair.instance.repair(repairType, 0); + assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); + assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); + + config.setMVRepairEnabled(repairType, false); + AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(0); + AutoRepair.instance.repair(repairType, 0); + assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); + assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); + + config.setMVRepairEnabled(repairType, true); + AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(0); + AutoRepair.instance.repair(repairType, 0); + assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); + assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); + + config.setMVRepairEnabled(repairType, false); + } + + @Test + public void testSkipRepairSSTableCountHigherThreshold() + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + AutoRepairState state = AutoRepair.instance.repairStates.get(repairType); + ColumnFamilyStore cfsBaseTable = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE); + ColumnFamilyStore cfsMVTable = Keyspace.open(KEYSPACE).getColumnFamilyStore(MV); + Set preBaseTable = cfsBaseTable.getLiveSSTables(); + Set preMVTable = cfsBaseTable.getLiveSSTables(); + + for (int i = 0; i < 10; i++) + { + QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, i, v) VALUES('k1', %d, 'v1')", KEYSPACE, TABLE, i)); + cfsBaseTable.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + cfsMVTable.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + } + + Set postBaseTable = cfsBaseTable.getLiveSSTables(); + Set diffBaseTable = new HashSet<>(postBaseTable); + diffBaseTable.removeAll(preBaseTable); + assert diffBaseTable.size() == 10; + + Set postMVTable = cfsBaseTable.getLiveSSTables(); + Set diffMVTable = new HashSet<>(postMVTable); + diffMVTable.removeAll(preMVTable); + assert diffMVTable.size() == 10; + + int beforeCount = config.getRepairSSTableCountHigherThreshold(repairType); + config.setMVRepairEnabled(repairType, true); + config.setRepairSSTableCountHigherThreshold(repairType, 9); + assertEquals(0, state.getRepairSkippedTablesCount()); + assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + state.setLastRepairTime(0); + AutoRepair.instance.repair(repairType, 0); + assertEquals(1, state.getTotalMVTablesConsideredForRepair()); + assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); + // skipping one time for the base table and another time for MV table + assertEquals(2, state.getRepairSkippedTablesCount()); + assertEquals(2, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + + // set it to higher value, and this time, the tables should not be skipped + config.setRepairSSTableCountHigherThreshold(repairType, 11); + config.setRepairSSTableCountHigherThreshold(repairType, beforeCount); + state.setLastRepairTime(0); + AutoRepair.instance.repair(repairType, 0); + assertEquals(1, state.getTotalMVTablesConsideredForRepair()); + assertEquals(0, state.getRepairSkippedTablesCount()); + assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); + assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + + config.setMVRepairEnabled(repairType, false); + config.setRepairSSTableCountHigherThreshold(repairType, beforeCount); + } + + @Test + public void testGetRepairState() + { + assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getRepairKeyspaceCount()); + + AutoRepairState state = AutoRepair.instance.getRepairState(repairType); + state.setRepairKeyspaceCount(100); + + assertEquals(100L, AutoRepair.instance.getRepairState(repairType).getRepairKeyspaceCount()); + } + + @Test + public void testMetrics() + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setMVRepairEnabled(repairType, true); + config.setRepairMinInterval(repairType, "0s"); + config.setAutoRepairTableMaxRepairTime(repairType, "0s"); + AutoRepair.timeFunc = () -> { + timeFuncCalls++; + return timeFuncCalls * 1000L; + }; + AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(1000L); + + AutoRepair.instance.repair(repairType, 0); + + assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); + assertTrue(AutoRepairMetricsManager.getMetrics(repairType).nodeRepairTimeInSec.getValue() > 0); + assertTrue(AutoRepairMetricsManager.getMetrics(repairType).clusterRepairTimeInSec.getValue() > 0); + assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).repairTurnMyTurn.getCount()); + assertTrue(AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue() > 0); + assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).longestUnrepairedSec.getValue().intValue()); + + config.setAutoRepairTableMaxRepairTime(repairType, String.valueOf(Integer.MAX_VALUE-1) + 's'); + AutoRepair.instance.repairStates.put(repairType, autoRepairState); + when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())) + .thenReturn(repairRunnable); + when(autoRepairState.getRepairFailedTablesCount()).thenReturn(10); + when(autoRepairState.getLongestUnrepairedSec()).thenReturn(10); + + AutoRepair.instance.repair(repairType, 0); + assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + assertTrue(AutoRepairMetricsManager.getMetrics(repairType).failedTablesCount.getValue() > 0); + assertTrue(AutoRepairMetricsManager.getMetrics(repairType).longestUnrepairedSec.getValue() > 0); + } + + @Test + public void testRepairWaitsForRepairToFinishBeforeSchedullingNewSession() throws Exception + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setMVRepairEnabled(repairType, false); + when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())) + .thenReturn(repairRunnable); + AutoRepair.instance.repairStates.put(repairType, autoRepairState); + when(autoRepairState.getLastRepairTime()).thenReturn((long) 0); + AtomicInteger resetWaitConditionCalls = new AtomicInteger(); + AtomicInteger waitForRepairCompletedCalls = new AtomicInteger(); + doAnswer(invocation -> { + resetWaitConditionCalls.getAndIncrement(); + assertEquals("waitForRepairToComplete was called before resetWaitCondition", + resetWaitConditionCalls.get(), waitForRepairCompletedCalls.get() + 1); + return null; + }).when(autoRepairState).resetWaitCondition(); + doAnswer(invocation -> { + waitForRepairCompletedCalls.getAndIncrement(); + assertEquals("resetWaitCondition was not called before waitForRepairToComplete", + resetWaitConditionCalls.get(), waitForRepairCompletedCalls.get()); + return null; + }).when(autoRepairState).waitForRepairToComplete(); + + AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType, 0); + } + + @Test + public void testDisabledAutoRepairForATableThroughTableLevelConfiguration() + { + Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); + Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).repairEnabled()); + Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); + Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).repairEnabled()); + + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setRepairMinInterval(repairType, "0s"); + int disabledTablesRepairCountBefore = AutoRepair.instance.repairStates.get(repairType).getTotalDisabledTablesRepairCount(); + AutoRepair.instance.repair(repairType, 0); + int consideredTables = AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair(); + Assert.assertNotSame(String.format("Expected total repaired tables > 0, actual value %s ", consideredTables), + consideredTables, 0); + int disabledTablesRepairCountAfter = AutoRepair.instance.repairStates.get(repairType).getTotalDisabledTablesRepairCount(); + Assert.assertTrue(String.format("A table %s should be skipped from auto repair, expected value: %d, actual value %d ", TABLE_DISABLED_AUTO_REPAIR, disabledTablesRepairCountBefore + 1, disabledTablesRepairCountAfter), + disabledTablesRepairCountBefore < disabledTablesRepairCountAfter); + } + + @Test + public void testTokenRangesNoSplit() + { + Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); + assertEquals(1, tokens.size()); + List> expectedToken = new ArrayList<>(); + expectedToken.addAll(tokens); + + List> ranges = new DefaultAutoRepairTokenSplitter().getRange(repairType, true, KEYSPACE, TABLE); + assertEquals(1, ranges.size()); + assertEquals(expectedToken.get(0).left, ranges.get(0).left); + assertEquals(expectedToken.get(0).right, ranges.get(0).right); + } + + @Test + public void testTokenRangesSplit() + { + Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); + assertEquals(1, tokens.size()); + List> expectedToken = new ArrayList<>(); + expectedToken.addAll(tokens); + + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setRepairSubRangeNum(repairType, 4); + List> ranges = new DefaultAutoRepairTokenSplitter().getRange(repairType, true, KEYSPACE, TABLE); + assertEquals(4, ranges.size()); + } + + @Test + public void testTableAttribute() + { + assertTrue(TableAttributes.validKeywords().contains("automated_repair_full")); + assertTrue(TableAttributes.validKeywords().contains("automated_repair_incremental")); + } + + @Test + public void testDefaultAutomatedRepair() + { + Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); + Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).repairEnabled()); + Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); + Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).repairEnabled()); + } +} diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java new file mode 100644 index 000000000000..a0e5bdc45294 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import org.junit.Test; + +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class AutoRepairStateFactoryTest +{ + @Test + public void testGetRepairState() { + AutoRepairState state = RepairType.getAutoRepairState(RepairType.full); + + assert state instanceof FullRepairState; + + state = RepairType.getAutoRepairState(RepairType.incremental); + + assert state instanceof IncrementalRepairState; + } + + @Test + public void testGetRepairStateSupportsAllRepairTypes() { + for (RepairType repairType : RepairType.values()) { + try { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + assertNotNull(state); + } catch (IllegalArgumentException e) { + assertNull(e); + } + } + } +} diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java new file mode 100644 index 000000000000..972b6339954f --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.Arrays; +import java.util.Collection; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils.AutoRepairHistory; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.utils.concurrent.Condition; +import org.apache.cassandra.utils.progress.ProgressEvent; +import org.apache.cassandra.utils.progress.ProgressEventType; +import org.mockito.Mock; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + +@RunWith(Parameterized.class) +public class AutoRepairStateTest extends CQLTester +{ + private static final String testTable = "test"; + + @Parameterized.Parameter + public RepairType repairType; + + @Mock + ProgressEvent progressEvent; + + @Parameterized.Parameters + public static Collection repairTypes() + { + return Arrays.asList(RepairType.values()); + } + + @Before + public void setUp() { + initMocks(this); + createTable(String.format("CREATE TABLE IF NOT EXISTS %s.%s (pk int PRIMARY KEY, v int)", KEYSPACE, testTable)); + } + + @Test + public void testGetRepairRunnable() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairService.setup(); + + Runnable runnable = state.getRepairRunnable(KEYSPACE, ImmutableList.of(testTable), ImmutableSet.of(), false); + + assertNotNull(runnable); + } + + @Test + public void testProgressError() throws InterruptedException + { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + when(progressEvent.getType()).thenReturn(ProgressEventType.ERROR); + + state.progress("test", progressEvent); + + assertFalse(state.success); + assertTrue(state.condition.await(0, TimeUnit.MILLISECONDS)); + } + + @Test + public void testProgress_progress() throws InterruptedException + { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + when(progressEvent.getType()).thenReturn(ProgressEventType.PROGRESS); + + state.progress("test", progressEvent); + + assertTrue(state.success); + assertFalse(state.condition.await(0, TimeUnit.MILLISECONDS)); + } + + + @Test + public void testProgress_complete() throws InterruptedException + { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + when(progressEvent.getType()).thenReturn(ProgressEventType.COMPLETE); + + state.progress("test", progressEvent); + + assertTrue(state.success); + assertTrue(state.condition.await(1, TimeUnit.MILLISECONDS)); + } + + @Test + public void testWaitForRepairToComplete() throws Exception + { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.condition.signalAll(); + Condition finishedCondition = Condition.newOneTimeCondition(); + Callable waitForRepairToComplete = () -> { + state.waitForRepairToComplete(); + finishedCondition.signalAll(); + return null; + }; + + waitForRepairToComplete.call(); + + assertTrue(finishedCondition.await(1, TimeUnit.MILLISECONDS)); + } + + @Test + public void testGetLastRepairTime() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.lastRepairTimeInMs = 1; + + assertEquals(1, state.getLastRepairTime()); + } + + @Test + public void testSetTotalTablesConsideredForRepair() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + + state.setTotalTablesConsideredForRepair(1); + + assertEquals(1, state.totalTablesConsideredForRepair); + } + + @Test + public void testGetTotalTablesConsideredForRepair() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.totalTablesConsideredForRepair = 1; + + assertEquals(1, state.getTotalTablesConsideredForRepair()); + } + + @Test + public void testSetLastRepairTimeInMs() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + + state.setLastRepairTime(1); + + assertEquals(1, state.lastRepairTimeInMs); + } + + @Test + public void testGetClusterRepairTimeInSec() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.clusterRepairTimeInSec = 1; + + assertEquals(1, state.getClusterRepairTimeInSec()); + } + + @Test + public void testGetNodeRepairTimeInSec() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.nodeRepairTimeInSec = 1; + + assertEquals(1, state.getNodeRepairTimeInSec()); + } + + @Test + public void testSetRepairInProgress() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + + state.setRepairInProgress(true); + + assertTrue(state.repairInProgress); + } + + @Test + public void testIsRepairInProgress() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.repairInProgress = true; + + assertTrue(state.isRepairInProgress()); + } + + @Test + public void testSetRepairSkippedTablesCount() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + + state.setRepairSkippedTablesCount(1); + + assertEquals(1, state.repairTableSkipCount); + } + + @Test + public void testGetRepairSkippedTablesCount() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.repairTableSkipCount = 1; + + assertEquals(1, state.getRepairSkippedTablesCount()); + } + + @Test + public void testGetLongestUnrepairedSecNull() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.longestUnrepairedNode = null; + + try + { + assertEquals(0, state.getLongestUnrepairedSec()); + } catch (Exception e) { + assertNull(e); + } + } + + @Test + public void testGetLongestUnrepairedSec() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.longestUnrepairedNode = new AutoRepairHistory(UUID.randomUUID(), "", 0, 1000, + null, 0, false); + AutoRepairState.timeFunc = () -> 2000L; + + try + { + assertEquals(1, state.getLongestUnrepairedSec()); + } catch (Exception e) { + assertNull(e); + } + } + + @Test + public void testSetRepairFailedTablesCount() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + + state.setRepairFailedTablesCount(1); + + assertEquals(1, state.repairTableFailureCount); + } + + @Test + public void testGetRepairFailedTablesCount() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.repairTableFailureCount = 1; + + assertEquals(1, state.getRepairFailedTablesCount()); + } + + @Test + public void testSetTotalMVTablesConsideredForRepair() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + + state.setTotalMVTablesConsideredForRepair(1); + + assertEquals(1, state.totalMVTablesConsideredForRepair); + } + + @Test + public void testGetTotalMVTablesConsideredForRepair() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.totalMVTablesConsideredForRepair = 1; + + assertEquals(1, state.getTotalMVTablesConsideredForRepair()); + } + + @Test + public void testSetNodeRepairTimeInSec() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + + state.setNodeRepairTimeInSec(1); + + assertEquals(1, state.nodeRepairTimeInSec); + } + + @Test + public void testSetClusterRepairTimeInSec() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + + state.setClusterRepairTimeInSec(1); + + assertEquals(1, state.clusterRepairTimeInSec); + } + + @Test + public void testSetRepairKeyspaceCount() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + + state.setRepairKeyspaceCount(1); + + assertEquals(1, state.repairKeyspaceCount); + } + + @Test + public void testSetRepairTableSuccessCount() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + + state.setRepairTableSuccessCount(1); + + assertEquals(1, state.repairTableSuccessCount); + } + + @Test + public void testGetRepairKeyspaceCount() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.repairKeyspaceCount = 1; + + assertEquals(1, state.getRepairKeyspaceCount()); + } + + @Test + public void testGetRepairTableSuccessCount() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.repairTableSuccessCount = 1; + + assertEquals(1, state.getRepairTableSuccessCount()); + } + + @Test + public void testSetLongestUnrepairedNode() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairHistory history = new AutoRepairHistory(UUID.randomUUID(), "", 0, 0, null, 0, false); + + state.setLongestUnrepairedNode(history); + + assertEquals(history, state.longestUnrepairedNode); + } + + @Test + public void isSuccess() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.success = true; + + assertTrue(state.isSuccess()); + + state.success = false; + + assertFalse(state.isSuccess()); + } + + @Test + public void testWaitForRepairToCompleteDoesNotSetSuccessWhenProgressReceivesError() throws InterruptedException + { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + when(progressEvent.getType()).thenReturn(ProgressEventType.ERROR); + + state.progress("test", progressEvent); + assertFalse(state.success); + + state.waitForRepairToComplete(); + assertFalse(state.success); + } + + @Test + public void testResetWaitCondition() + { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.condition.signalAll(); + assertTrue(state.condition.isSignalled()); + + state.resetWaitCondition(); + + assertFalse(state.condition.isSignalled()); + } +} diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java new file mode 100644 index 000000000000..b16e0ea5b491 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.BeforeClass; +import org.junit.Test; + +import junit.framework.Assert; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.schema.SchemaTestUtil; + +import static org.apache.cassandra.Util.setAutoRepairEnabled; +import static org.junit.Assert.assertEquals; + +public class AutoRepairTest extends CQLTester +{ + @BeforeClass + public static void setupClass() throws Exception + { + setAutoRepairEnabled(true); + requireNetwork(); + } + + @Test + public void testSetup() + { + AutoRepair instance = new AutoRepair(); + + instance.setup(); + + assertEquals(RepairType.values().length, instance.repairExecutors.size()); + for (RepairType repairType : instance.repairExecutors.keySet()) + { + assertEquals(String.format("Expected 1 task in queue for %s", repairType), + 1, instance.repairExecutors.get(repairType).getPendingTaskCount() + + instance.repairExecutors.get(repairType).getActiveTaskCount()); + } + } + + @Test(expected = ConfigurationException.class) + public void testSetupFailsWhenIREnabledWithCDC() + { + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.incremental, true); + DatabaseDescriptor.setCDCEnabled(true); + + AutoRepair instance = new AutoRepair(); + instance.setup(); + } + + @Test(expected = ConfigurationException.class) + public void testSetupFailsWhenIREnabledWithMV() + { + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.incremental, true); + DatabaseDescriptor.setMaterializedViewsEnabled(true); + + AutoRepair instance = new AutoRepair(); + instance.setup(); + } + + @Test + public void testCheckNTSreplicationNodeInsideOutsideDC() + { + String ksname1 = "ks_nts1"; + String ksname2 = "ks_nts2"; + Map configOptions1 = new HashMap<>(); + configOptions1.put("datacenter1", "3"); + configOptions1.put(ReplicationParams.CLASS, "NetworkTopologyStrategy"); + SchemaTestUtil.addOrUpdateKeyspace(KeyspaceMetadata.create(ksname1, KeyspaceParams.create(false, configOptions1)), false); + Map configOptions2 = new HashMap<>(); + configOptions2.put("datacenter2", "3"); + configOptions2.put(ReplicationParams.CLASS, "NetworkTopologyStrategy"); + SchemaTestUtil.addOrUpdateKeyspace(KeyspaceMetadata.create(ksname2, KeyspaceParams.create(false, configOptions2)), false); + + for (Keyspace ks : Keyspace.all()) + { + if (ks.getName().equals(ksname1)) + { + // case 1 : + // node reside in "datacenter1" + // keyspace has replica in "datacenter1" + Assert.assertTrue(AutoRepairUtils.checkNodeContainsKeyspaceReplica(ks)); + } + else if (ks.getName().equals(ksname2)) + { + // case 2 : + // node reside in "datacenter1" + // keyspace has replica in "datacenter2" + Assert.assertFalse(AutoRepairUtils.checkNodeContainsKeyspaceReplica(ks)); + } + } + } +} diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java new file mode 100644 index 000000000000..20c6545a67d0 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -0,0 +1,476 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils.AutoRepairHistory; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils.CurrentRepairStatus; + +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.FBUtilities; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.apache.cassandra.Util.setAutoRepairEnabled; +import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.COL_DELETE_HOSTS; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.COL_FORCE_REPAIR; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.COL_REPAIR_FINISH_TS; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.COL_REPAIR_PRIORITY; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.COL_REPAIR_START_TS; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.COL_REPAIR_TURN; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +public class AutoRepairUtilsTest extends CQLTester +{ + static RepairType repairType = RepairType.incremental; + static UUID hostId; + + static InetAddressAndPort localEndpoint; + + @Mock + static IEndpointSnitch snitchMock; + + static IEndpointSnitch defaultSnitch; + + + @BeforeClass + public static void setupClass() throws Exception + { + SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); + setAutoRepairEnabled(true); + requireNetwork(); + defaultSnitch = DatabaseDescriptor.getEndpointSnitch(); + localEndpoint = FBUtilities.getBroadcastAddressAndPort(); + hostId = Gossiper.instance.getHostId(localEndpoint); + AutoRepairUtils.setup(); + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace("ks", KeyspaceParams.create(false, + ImmutableMap.of("class", "NetworkTopologyStrategy", "datacenter1", "1")), + TableMetadata.builder("ks", "tbl") + .addPartitionKeyColumn("k", UTF8Type.instance) + .build()); + } + + @Before + public void setup() + { + MockitoAnnotations.initMocks(this); + DatabaseDescriptor.setEndpointSnitch(defaultSnitch); + QueryProcessor.executeInternal(String.format( + "TRUNCATE %s.%s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY)); + QueryProcessor.executeInternal(String.format( + "TRUNCATE %s.%s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY)); + } + + @Test + public void testSetForceRepair() + { + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id, force_repair) VALUES ('%s', %s, false)", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + + AutoRepairUtils.setForceRepair(repairType, ImmutableSet.of(localEndpoint)); + + UntypedResultSet result = QueryProcessor.executeInternal(String.format( + "SELECT force_repair FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + assertNotNull(result); + assertEquals(1, result.size()); + assertTrue(result.one().getBoolean(COL_FORCE_REPAIR)); + } + + @Test + public void testSetForceRepairNewNode() + { + AutoRepairUtils.setForceRepairNewNode(repairType); + + UntypedResultSet result = QueryProcessor.executeInternal(String.format( + "SELECT force_repair FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + assertNotNull(result); + assertEquals(1, result.size()); + assertTrue(result.one().getBoolean(COL_FORCE_REPAIR)); + } + + + @Test + public void testClearDeleteHosts() + { + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id, delete_hosts, delete_hosts_update_time) VALUES ('%s', %s, { %s }, toTimestamp(now()))", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId, hostId)); + + AutoRepairUtils.clearDeleteHosts(repairType, hostId); + + UntypedResultSet result = QueryProcessor.executeInternal(String.format( + "SELECT delete_hosts FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + assertNotNull(result); + assertEquals(1, result.size()); + Set deleteHosts = result.one().getSet(COL_DELETE_HOSTS, UUIDType.instance); + assertNull(deleteHosts); + } + + @Test + public void testGetAutoRepairHistoryForLocalGroup() + { + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id, force_repair) VALUES ('%s', %s, false)", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + + List history = AutoRepairUtils.getAutoRepairHistoryForLocalGroup(repairType); + + assertNotNull(history); + assertEquals(1, history.size()); + assertEquals(hostId, history.get(0).hostId); + } + + @Test + public void testGetAutoRepairHistoryForLocalGroup_empty_history() + { + List history = AutoRepairUtils.getAutoRepairHistoryForLocalGroup(repairType); + + assertNull(history); + } + + @Test + public void testGetCurrentRepairStatus() + { + UUID forceRepair = UUID.randomUUID(); + UUID regularRepair = UUID.randomUUID(); + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id, force_repair, repair_start_ts) VALUES ('%s', %s, true, toTimestamp(now()))", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), forceRepair)); + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id, repair_start_ts) VALUES ('%s', %s, toTimestamp(now()))", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), regularRepair)); + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, repair_priority) VALUES ('%s', { %s })", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, + repairType.toString(), regularRepair)); + + CurrentRepairStatus status = AutoRepairUtils.getCurrentRepairStatus(repairType); + + assertNotNull(status); + assertEquals(1, status.historiesWithoutOnGoingRepair.size()); + assertEquals(hostId, status.historiesWithoutOnGoingRepair.get(0).hostId); + assertEquals(1, status.hostIdsWithOnGoingRepair.size()); + assertTrue(status.hostIdsWithOnGoingRepair.contains(regularRepair)); + assertEquals(1, status.hostIdsWithOnGoingForceRepair.size()); + assertTrue(status.hostIdsWithOnGoingForceRepair.contains(forceRepair)); + assertEquals(1, status.priority.size()); + assertTrue(status.priority.contains(regularRepair)); + } + + @Test + public void testGetHostIdsInCurrentRing() + { + TreeSet hosts = AutoRepairUtils.getHostIdsInCurrentRing(repairType); + + assertNotNull(hosts); + assertEquals(1, hosts.size()); + assertTrue(hosts.contains(hostId)); + } + + @Test + public void testGetHostIdsInCurrentRing_multiple_nodes() + { + InetAddressAndPort ignoredEndpoint = localEndpoint.withPort(localEndpoint.getPort() + 1); + InetAddressAndPort deadEndpoint = localEndpoint.withPort(localEndpoint.getPort() + 2); + DatabaseDescriptor.getAutoRepairConfig().setIgnoreDCs(repairType, ImmutableSet.of("dc2")); + DatabaseDescriptor.setEndpointSnitch(snitchMock); + when(snitchMock.getDatacenter(localEndpoint)).thenReturn("dc1"); + when(snitchMock.getDatacenter(ignoredEndpoint)).thenReturn("dc2"); + when(snitchMock.getDatacenter(deadEndpoint)).thenReturn("dc1"); + + TreeSet hosts = AutoRepairUtils.getHostIdsInCurrentRing(repairType, ImmutableSet.of(localEndpoint, ignoredEndpoint, deadEndpoint)); + + assertNotNull(hosts); + assertEquals(1, hosts.size()); + assertTrue(hosts.contains(hostId)); + } + + @Test + public void testGetHostWithLongestUnrepairTime() + { + UUID otherHostId = UUID.randomUUID(); + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id, repair_finish_ts) VALUES ('%s', %s, toTimestamp(now()))", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), otherHostId)); + + AutoRepairHistory history = AutoRepairUtils.getHostWithLongestUnrepairTime(repairType); + + assertEquals(hostId, history.hostId); + } + + @Test + public void testGetMaxNumberOfNodeRunAutoRepairInGroup_0_group_size() + { + DatabaseDescriptor.getAutoRepairConfig().setParallelRepairCountInGroup(repairType, 2); + + int count = AutoRepairUtils.getMaxNumberOfNodeRunAutoRepairInGroup(repairType, 0); + + assertEquals(2, count); + } + + + @Test + public void testGetMaxNumberOfNodeRunAutoRepairInGroup_percentage() + { + DatabaseDescriptor.getAutoRepairConfig().setParallelRepairCountInGroup(repairType, 2); + DatabaseDescriptor.getAutoRepairConfig().setParallelRepairPercentageInGroup(repairType, 50); + + + int count = AutoRepairUtils.getMaxNumberOfNodeRunAutoRepairInGroup(repairType, 10); + + assertEquals(5, count); + } + + @Test + public void testDeleteAutoRepairHistory() + { + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + + AutoRepairUtils.deleteAutoRepairHistory(repairType, hostId); + + UntypedResultSet result = QueryProcessor.executeInternal(String.format( + "SELECT * FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + assertNotNull(result); + assertEquals(0, result.size()); + } + + @Test + public void testUpdateStartAutoRepairHistory() + { + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + + AutoRepairUtils.updateStartAutoRepairHistory(repairType, hostId, 123, AutoRepairUtils.RepairTurn.MY_TURN); + + UntypedResultSet result = QueryProcessor.executeInternal(String.format( + "SELECT repair_start_ts, repair_turn FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + assertNotNull(result); + assertEquals(1, result.size()); + UntypedResultSet.Row row = result.one(); + assertEquals(123, row.getLong(COL_REPAIR_START_TS, 0)); + assertEquals(AutoRepairUtils.RepairTurn.MY_TURN.toString(), row.getString(COL_REPAIR_TURN)); + } + + @Test + public void testUpdateFinishAutoRepairHistory() + { + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + + AutoRepairUtils.updateFinishAutoRepairHistory(repairType, hostId, 123); + + UntypedResultSet result = QueryProcessor.executeInternal(String.format( + "SELECT repair_finish_ts FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), hostId)); + assertNotNull(result); + assertEquals(1, result.size()); + assertEquals(123, result.one().getLong(COL_REPAIR_FINISH_TS, 0)); + } + + @Test + public void testAddHostIdToDeleteHosts() + { + UUID otherHostId = UUID.randomUUID(); + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), otherHostId)); + + AutoRepairUtils.addHostIdToDeleteHosts(repairType, hostId, otherHostId); + + UntypedResultSet result = QueryProcessor.executeInternal(String.format( + "SELECT * FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + repairType.toString(), otherHostId)); + assertNotNull(result); + assertEquals(1, result.size()); + Set deleteHosts = result.one().getSet(COL_DELETE_HOSTS, UUIDType.instance); + assertNotNull(deleteHosts); + assertEquals(1, deleteHosts.size()); + assertTrue(deleteHosts.contains(hostId)); + } + + @Test + public void testAddPriorityHost() + { + AutoRepairUtils.addPriorityHosts(repairType, ImmutableSet.of(localEndpoint)); + + UntypedResultSet result = QueryProcessor.executeInternal(String.format( + "SELECT * FROM %s.%s WHERE repair_type = '%s'", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, + repairType.toString())); + assertNotNull(result); + assertEquals(1, result.size()); + Set repairPriority = result.one().getSet(COL_REPAIR_PRIORITY, UUIDType.instance); + assertNotNull(repairPriority); + assertEquals(1, repairPriority.size()); + assertTrue(repairPriority.contains(hostId)); + } + + @Test + public void testRemovePriorityStatus() + { + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, repair_priority) VALUES ('%s', { %s })", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, + repairType.toString(), hostId)); + + AutoRepairUtils.removePriorityStatus(repairType, hostId); + + UntypedResultSet result = QueryProcessor.executeInternal(String.format( + "SELECT * FROM %s.%s WHERE repair_type = '%s'", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, + repairType.toString())); + assertNotNull(result); + assertEquals(1, result.size()); + Set repairPriority = result.one().getSet(COL_REPAIR_PRIORITY, UUIDType.instance); + assertNull(repairPriority); + } + + @Test + public void testGetPriorityHosts() + { + QueryProcessor.executeInternal(String.format( + "INSERT INTO %s.%s (repair_type, repair_priority) VALUES ('%s', { %s })", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, + repairType.toString(), hostId)); + + Set hosts = AutoRepairUtils.getPriorityHosts(repairType); + + assertNotNull(hosts); + assertEquals(1, hosts.size()); + assertTrue(hosts.contains(localEndpoint)); + } + + @Test + public void testCheckNodeContainsKeyspaceReplica() + { + Keyspace ks = Keyspace.open("ks"); + + assertTrue(AutoRepairUtils.checkNodeContainsKeyspaceReplica(ks)); + } + + @Test + public void testTableMaxRepairTimeExceeded() + { + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairTableMaxRepairTime(repairType, "0s"); + + assertTrue(AutoRepairUtils.tableMaxRepairTimeExceeded(repairType, 0)); + } + + @Test + public void testKeyspaceMaxRepairTimeExceeded() + { + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairTableMaxRepairTime(repairType, "0s"); + + assertTrue(AutoRepairUtils.keyspaceMaxRepairTimeExceeded(repairType, 0, 1)); + } + + @Test + public void testGetLastRepairFinishTime() + { + AutoRepairHistory history = new AutoRepairHistory(UUID.randomUUID(), "", 0, 0, null, 0, false); + + assertEquals(0, history.getLastRepairFinishTime()); + + history.lastRepairFinishTime = 100; + + assertEquals(100, history.getLastRepairFinishTime()); + } + + @Test + public void testMyTurnToRunRepairShouldReturnMyTurnWhenRepairOngoing() + { + UUID myID = UUID.randomUUID(); + UUID otherID = UUID.randomUUID(); + DatabaseDescriptor.getAutoRepairConfig().setParallelRepairCountInGroup(repairType, 5); + long currentMillis = System.currentTimeMillis(); + // finish time less than start time means that repair is ongoing + AutoRepairUtils.insertNewRepairHistory(repairType, myID, currentMillis, currentMillis - 100); + // finish time is larger than start time means that repair for other node is finished + AutoRepairUtils.insertNewRepairHistory(repairType, otherID, currentMillis, currentMillis + 100); + + assertEquals(AutoRepairUtils.RepairTurn.MY_TURN, AutoRepairUtils.myTurnToRunRepair(repairType, myID)); + } +} diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java index 9c1660a9c39f..2f1b3c3a2288 100644 --- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java @@ -36,6 +36,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import org.apache.cassandra.service.disk.usage.DiskUsageMonitor; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Condition; import org.junit.Assert; @@ -66,6 +67,7 @@ import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Refs; +import org.mockito.Mock; import static org.apache.cassandra.repair.messages.RepairOption.DATACENTERS_KEY; import static org.apache.cassandra.repair.messages.RepairOption.FORCE_REPAIR_KEY; @@ -74,11 +76,14 @@ import static org.apache.cassandra.repair.messages.RepairOption.RANGES_KEY; import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; import static org.apache.cassandra.service.ActiveRepairService.getRepairedAt; +import static org.apache.cassandra.service.ActiveRepairService.instance; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; public class ActiveRepairServiceTest { @@ -90,7 +95,8 @@ public class ActiveRepairServiceTest public String cfname; public ColumnFamilyStore store; public InetAddressAndPort LOCAL, REMOTE; - + @Mock + public DiskUsageMonitor diskUsageMonitor; private boolean initialized; @BeforeClass @@ -121,6 +127,7 @@ public void prepare() throws Exception StorageService.instance.setTokens(Collections.singleton(tmd.partitioner.getRandomToken())); tmd.updateNormalToken(tmd.partitioner.getMinimumToken(), REMOTE); assert tmd.isMember(REMOTE); + initMocks(this); } @Test @@ -469,6 +476,41 @@ public void testQueueWhenPoolFullStrategy() throws InterruptedException } } + @Test + public void testVerifyDiskHeadroomThresholdFullRepair() + { + Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, false)); + } + + @Test + public void testVerifyDiskHeadroomThresholdDiskFull() + { + DiskUsageMonitor.instance = diskUsageMonitor; + when(diskUsageMonitor.getDiskUsage()).thenReturn(1.0); + DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(1.0); + + Assert.assertFalse(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, true)); + } + + @Test + public void testVerifyDiskHeadroomThresholdSufficientDisk() + { + DiskUsageMonitor.instance = diskUsageMonitor; + when(diskUsageMonitor.getDiskUsage()).thenReturn(0.0); + DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(0.0); + + Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, true)); + } + + @Test(expected = RuntimeException.class) + public void testPrepareForRepairThrowsExceptionForInsufficientDisk() + { + DiskUsageMonitor.instance = diskUsageMonitor; + when(diskUsageMonitor.getDiskUsage()).thenReturn(1.5); + + instance.prepareForRepair(TimeUUID.maxAtUnixMillis(0), null, null, opts(INCREMENTAL_KEY, b2s(true)), false, null); + } + private static class Task implements Runnable { private final Condition blocked; From 96993e52597717314f1f8761300928565dba0266 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 23 Jul 2024 14:28:07 -0700 Subject: [PATCH 002/199] cr --- .../cassandra/repair/autorepair/AutoRepair.java | 15 ++++++++++----- .../repair/autorepair/AutoRepairState.java | 3 ++- .../repair/autorepair/AutoRepairUtils.java | 15 ++++++++------- .../apache/cassandra/schema/AutoRepairParams.java | 11 +++++++---- .../org/apache/cassandra/schema/TableParams.java | 4 ++-- .../autorepair/AutoRepairParameterizedTest.java | 3 +-- .../repair/autorepair/AutoRepairTest.java | 3 +-- 7 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 9fbc208f95ab..abdf4c932300 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -39,7 +38,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.concurrent.NamedThreadFactory; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; @@ -53,6 +51,7 @@ import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn; +import org.apache.cassandra.utils.concurrent.Future; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN; @@ -73,6 +72,9 @@ public class AutoRepair @VisibleForTesting protected final Map repairExecutors; + + protected final Map repairRunnableExecutors; + @VisibleForTesting protected final Map repairStates; @@ -84,10 +86,12 @@ protected AutoRepair() { AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); repairExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class); + repairRunnableExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class); repairStates = new EnumMap<>(AutoRepairConfig.RepairType.class); for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) { repairExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-Repair-" + repairType, Thread.NORM_PRIORITY)); + repairRunnableExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-RepairRunnable-" + repairType, Thread.NORM_PRIORITY)); repairStates.put(repairType, AutoRepairConfig.RepairType.getAutoRepairState(repairType)); tokenRangeSplitters.put(repairType, FBUtilities.newAutoRepairTokenRangeSplitter(config.getTokenRangeSplitter(repairType))); } @@ -297,7 +301,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : ImmutableList.of(tableName), ranges, primaryRangeOnly); repairState.resetWaitCondition(); - new Thread(NamedThreadFactory.createAnonymousThread(new FutureTask<>(task, null))).start(); + Future f = repairRunnableExecutors.get(repairType).submit(task); try { repairState.waitForRepairToComplete(); @@ -317,10 +321,11 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) else { repairSuccess = false; + boolean cancellationStatus = f.cancel(true); //in future we can add retry, etc. logger.info("Repair failed for range {}-{} for {}.{} total subranges: {}," + - "processed subranges: {}", childStartToken, childEndToken, - keyspaceName, config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, totalSubRanges, totalProcessedSubRanges); + "processed subranges: {}, cancellationStatus: {}", childStartToken, childEndToken, + keyspaceName, config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, totalSubRanges, totalProcessedSubRanges, cancellationStatus); } ranges.clear(); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index b4a15ebb6dd0..7f6e84bcb665 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -50,6 +50,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; // AutoRepairState represents the state of automated repair for a given repair type. @@ -115,7 +116,7 @@ protected RepairRunnable getRepairRunnable(String keyspace, RepairOption options public void progress(String tag, ProgressEvent event) { ProgressEventType type = event.getType(); - String message = String.format("[%s] %s", format.format(System.currentTimeMillis()), event.getMessage()); + String message = String.format("[%s] %s", format.format(currentTimeMillis()), event.getMessage()); if (type == ProgressEventType.ERROR) { logger.error("Repair failure for {} repair: {}", repairType.toString(), message); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index 5cc95367192d..e318849fee1e 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -72,6 +72,7 @@ import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_DUE_TO_PRIORITY; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.NOT_MY_TURN; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_FORCE_REPAIR; +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; /** * This class serves as a utility class for AutoRepair. It contains various helper APIs @@ -335,7 +336,7 @@ public static void setForceRepairNewNode(RepairType repairType) // this function will be called when a node bootstrap finished UUID hostId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); // insert the data first - insertNewRepairHistory(repairType, System.currentTimeMillis(), System.currentTimeMillis()); + insertNewRepairHistory(repairType, currentTimeMillis(), currentTimeMillis()); setForceRepair(repairType, hostId); } @@ -474,7 +475,7 @@ public static RepairTurn myTurnToRunRepair(RepairType repairType, UUID myId) AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); if (nodeHistory.deleteHosts.size() > 0 && config.getAutoRepairHistoryClearDeleteHostsBufferInterval().toSeconds() < TimeUnit.MILLISECONDS.toSeconds( - System.currentTimeMillis() - nodeHistory.deleteHostsUpdateTime + currentTimeMillis() - nodeHistory.deleteHostsUpdateTime )) { clearDeleteHosts(repairType, nodeHistory.hostId); @@ -505,7 +506,7 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) if (!autoRepairHistoryIds.contains(hostId)) { logger.info("{} for repair type {} doesn't exist in the auto repair history table, insert a new record.", repairType, hostId); - insertNewRepairHistory(repairType, hostId, System.currentTimeMillis(), System.currentTimeMillis()); + insertNewRepairHistory(repairType, hostId, currentTimeMillis(), currentTimeMillis()); } } @@ -648,7 +649,7 @@ public static void insertNewRepairHistory(RepairType repairType, UUID hostId, lo ByteBufferUtil.bytes(hostId), ByteBufferUtil.bytes(startTime), ByteBufferUtil.bytes(finishTime), - ByteBufferUtil.bytes(System.currentTimeMillis()) + ByteBufferUtil.bytes(currentTimeMillis()) ), false, -1, null, cl, ProtocolVersion.CURRENT, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME), Dispatcher.RequestTime.forImmediateExecution()); resultSet = UntypedResultSet.create(resultMessage.result); @@ -680,7 +681,7 @@ public static void addHostIdToDeleteHosts(RepairType repairType, UUID myID, UUID addHostIDToDeleteHostsStatement.execute(QueryState.forInternalCalls(), QueryOptions.forInternalCalls(internalQueryCL, Lists.newArrayList(serializer.serialize(new HashSet<>(Arrays.asList(myID))), - ByteBufferUtil.bytes(System.currentTimeMillis()), + ByteBufferUtil.bytes(currentTimeMillis()), ByteBufferUtil.bytes(repairType.toString()), ByteBufferUtil.bytes(hostToBeDeleted) )), Dispatcher.RequestTime.forImmediateExecution()); @@ -776,14 +777,14 @@ public static boolean checkNodeContainsKeyspaceReplica(Keyspace ks) public static boolean tableMaxRepairTimeExceeded(RepairType repairType, long startTime) { long tableRepairTimeSoFar = TimeUnit.MILLISECONDS.toSeconds - (System.currentTimeMillis() - startTime); + (currentTimeMillis() - startTime); return AutoRepairService.instance.getAutoRepairConfig().getAutoRepairTableMaxRepairTime(repairType).toSeconds() < tableRepairTimeSoFar; } public static boolean keyspaceMaxRepairTimeExceeded(RepairType repairType, long startTime, int numOfTablesToBeRepaired) { - long keyspaceRepairTimeSoFar = TimeUnit.MILLISECONDS.toSeconds((System.currentTimeMillis() - startTime)); + long keyspaceRepairTimeSoFar = TimeUnit.MILLISECONDS.toSeconds((currentTimeMillis() - startTime)); return (long) AutoRepairService.instance.getAutoRepairConfig().getAutoRepairTableMaxRepairTime(repairType).toSeconds() * numOfTablesToBeRepaired < keyspaceRepairTimeSoFar; } diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index 50f482caf1b7..ae97e562635f 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -63,13 +63,16 @@ public static AutoRepairParams create(AutoRepairConfig.RepairType repairType, Ma { optionsMap.put(entry.getKey(), new HashMap<>(entry.getValue())); } - for (Map.Entry entry : options.entrySet()) + if (options != null) { - if (!Option.ENABLED.toString().equals(entry.getKey().toLowerCase())) + for (Map.Entry entry : options.entrySet()) { - throw new ConfigurationException(format("Unknown property '%s'", entry.getKey())); + if (!Option.ENABLED.toString().equals(entry.getKey().toLowerCase())) + { + throw new ConfigurationException(format("Unknown property '%s'", entry.getKey())); + } + optionsMap.get(repairType).put(entry.getKey(), entry.getValue()); } - optionsMap.get(repairType).put(entry.getKey(), entry.getValue()); } AutoRepairParams repairParams = new AutoRepairParams(repairType); repairParams.options = optionsMap; diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index a61138c665f5..b8b10c768494 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -18,7 +18,7 @@ package org.apache.cassandra.schema; import java.nio.ByteBuffer; -import java.util.HashMap; +import java.util.EnumMap; import java.util.Map; import java.util.Map.Entry; @@ -112,7 +112,7 @@ private TableParams(Builder builder) extensions = builder.extensions; cdc = builder.cdc; readRepair = builder.readRepair; - automatedRepair = new HashMap<>() + automatedRepair = new EnumMap(AutoRepairConfig.RepairType.class) { { put(AutoRepairConfig.RepairType.full, builder.automatedRepairFull); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 7a522a3458ae..bbf3714e315b 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -39,13 +39,13 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.Pair; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import junit.framework.Assert; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; @@ -71,7 +71,6 @@ import org.mockito.MockitoAnnotations; import static org.apache.cassandra.Util.setAutoRepairEnabled; -import static org.apache.cassandra.Util.token; import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.NOT_MY_TURN; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index b16e0ea5b491..c1837aa6a2cc 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -24,8 +24,7 @@ import org.junit.BeforeClass; import org.junit.Test; -import junit.framework.Assert; - +import org.junit.Assert; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.Keyspace; From de00b62195d75ccb72de001859f6d11358e4c52b Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sat, 27 Jul 2024 09:43:23 -0700 Subject: [PATCH 003/199] chop the repair API --- .../repair/autorepair/AutoRepair.java | 150 ++++++++++-------- 1 file changed, 83 insertions(+), 67 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index abdf4c932300..bb3265e00d22 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -139,10 +139,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) logger.debug("Auto-repair is disabled for repair type {}", repairType); return; } - - AutoRepairState repairState = repairStates.get(repairType); - try { String localDC = DatabaseDescriptor.getLocalDataCenter(); @@ -168,19 +165,9 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) boolean primaryRangeOnly = config.getRepairPrimaryTokenRangeOnly(repairType) && turn != MY_TURN_FORCE_REPAIR; repairState.setTotalTablesConsideredForRepair(0); - if (repairState.getLastRepairTime() != 0) + if (tooSoonToRunRepair(repairType, repairState, config)) { - /** check if it is too soon to run repair. one of the reason we - * should not run frequent repair is because repair triggers - * memtable flush - */ - long timeElapsedSinceLastRepair = TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - repairState.getLastRepairTime()); - if (timeElapsedSinceLastRepair < config.getRepairMinInterval(repairType).toSeconds()) - { - logger.info("Too soon to run repair, last repair was done {} seconds ago", - timeElapsedSinceLastRepair); - return; - } + return; } long startTime = timeFunc.get(); @@ -196,34 +183,16 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) repairState.setTotalMVTablesConsideredForRepair(0); for (Keyspace keyspace : Keyspace.all()) { - Tables tables = keyspace.getMetadata().tables; - Iterator iter = tables.iterator(); - String keyspaceName = keyspace.getName(); if (!AutoRepairUtils.checkNodeContainsKeyspaceReplica(keyspace)) { continue; } repairState.setRepairKeyspaceCount(repairState.getRepairKeyspaceCount() + 1); - List tablesToBeRepaired = new ArrayList<>(); - while (iter.hasNext()) - { - repairState.setTotalTablesConsideredForRepair(repairState.getTotalTablesConsideredForRepair() + 1); - TableMetadata tableMetadata = iter.next(); - String tableName = tableMetadata.name; - tablesToBeRepaired.add(tableName); - - // See if we should repair MVs as well that are associated with this given table - List mvs = AutoRepairUtils.getAllMVs(repairType, keyspace, tableMetadata); - if (!mvs.isEmpty()) - { - tablesToBeRepaired.addAll(mvs); - repairState.setTotalMVTablesConsideredForRepair(repairState.getTotalMVTablesConsideredForRepair() + mvs.size()); - } - } - + List tablesToBeRepaired = retrieveTablesToBeRepaired(keyspace, repairType, repairState); for (String tableName : tablesToBeRepaired) { + String keyspaceName = keyspace.getName(); try { ColumnFamilyStore columnFamilyStore = keyspace.getColumnFamilyStore(tableName); @@ -355,38 +324,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) } } } - - //if it was due to priority then remove it now - if (turn == MY_TURN_DUE_TO_PRIORITY) - { - logger.info("Remove current host from priority list"); - AutoRepairUtils.removePriorityStatus(repairType, myId); - } - - repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - startTime)); - long timeInHours = TimeUnit.SECONDS.toHours(repairState.getNodeRepairTimeInSec()); - logger.info("Local {} repair time {} hour(s), stats: repairKeyspaceCount {}, " + - "repairTableSuccessCount {}, repairTableFailureCount {}, " + - "repairTableSkipCount {}", repairType, timeInHours, repairState.getRepairKeyspaceCount(), - repairState.getRepairTableSuccessCount(), repairState.getRepairFailedTablesCount(), - repairState.getRepairSkippedTablesCount()); - if (repairState.getLastRepairTime() != 0) - { - repairState.setClusterRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - - repairState.getLastRepairTime())); - logger.info("Cluster repair time for repair type {}: {} day(s)", repairType, - TimeUnit.SECONDS.toDays(repairState.getClusterRepairTimeInSec())); - } - repairState.setLastRepairTime(timeFunc.get()); - if (timeInHours == 0 && millisToWait > 0) - { - //If repair finished quickly, happens for an empty instance, in such case - //wait for a minute so that the JMX metrics can detect the repairInProgress - logger.info("Wait for {} milliseconds for repair type {}.", millisToWait, repairType); - Thread.sleep(millisToWait); - } - repairState.setRepairInProgress(false); - AutoRepairUtils.updateFinishAutoRepairHistory(repairType, myId, timeFunc.get()); + cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, millisToWait); } else { @@ -399,6 +337,84 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) } } + private boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoRepairState repairState, AutoRepairConfig config) + { + if (repairState.getLastRepairTime() != 0) + { + /** check if it is too soon to run repair. one of the reason we + * should not run frequent repair is that repair triggers + * memtable flush + */ + long timeElapsedSinceLastRepair = TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - repairState.getLastRepairTime()); + if (timeElapsedSinceLastRepair < config.getRepairMinInterval(repairType).toSeconds()) + { + logger.info("Too soon to run repair, last repair was done {} seconds ago", + timeElapsedSinceLastRepair); + return true; + } + } + return false; + } + + private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairConfig.RepairType repairType, AutoRepairState repairState) + { + Tables tables = keyspace.getMetadata().tables; + List tablesToBeRepaired = new ArrayList<>(); + Iterator iter = tables.iterator(); + while (iter.hasNext()) + { + repairState.setTotalTablesConsideredForRepair(repairState.getTotalTablesConsideredForRepair() + 1); + TableMetadata tableMetadata = iter.next(); + String tableName = tableMetadata.name; + tablesToBeRepaired.add(tableName); + + // See if we should repair MVs as well that are associated with this given table + List mvs = AutoRepairUtils.getAllMVs(repairType, keyspace, tableMetadata); + if (!mvs.isEmpty()) + { + tablesToBeRepaired.addAll(mvs); + repairState.setTotalMVTablesConsideredForRepair(repairState.getTotalMVTablesConsideredForRepair() + mvs.size()); + } + } + return tablesToBeRepaired; + } + + private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, UUID myId, + long startTime, long millisToWait) throws InterruptedException + { + //if it was due to priority then remove it now + if (turn == MY_TURN_DUE_TO_PRIORITY) + { + logger.info("Remove current host from priority list"); + AutoRepairUtils.removePriorityStatus(repairType, myId); + } + + repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - startTime)); + long timeInHours = TimeUnit.SECONDS.toHours(repairState.getNodeRepairTimeInSec()); + logger.info("Local {} repair time {} hour(s), stats: repairKeyspaceCount {}, " + + "repairTableSuccessCount {}, repairTableFailureCount {}, " + + "repairTableSkipCount {}", repairType, timeInHours, repairState.getRepairKeyspaceCount(), + repairState.getRepairTableSuccessCount(), repairState.getRepairFailedTablesCount(), + repairState.getRepairSkippedTablesCount()); + if (repairState.getLastRepairTime() != 0) + { + repairState.setClusterRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - + repairState.getLastRepairTime())); + logger.info("Cluster repair time for repair type {}: {} day(s)", repairType, + TimeUnit.SECONDS.toDays(repairState.getClusterRepairTimeInSec())); + } + repairState.setLastRepairTime(timeFunc.get()); + if (timeInHours == 0 && millisToWait > 0) + { + //If repair finished quickly, happens for an empty instance, in such case + //wait for a minute so that the JMX metrics can detect the repairInProgress + logger.info("Wait for {} milliseconds for repair type {}.", millisToWait, repairType); + Thread.sleep(millisToWait); + } + repairState.setRepairInProgress(false); + AutoRepairUtils.updateFinishAutoRepairHistory(repairType, myId, timeFunc.get()); + } + public AutoRepairState getRepairState(AutoRepairConfig.RepairType repairType) { return repairStates.get(repairType); From 8916d312985d16f6c960bdf7ff8977cc94947556 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 16 Aug 2024 14:54:06 -0700 Subject: [PATCH 004/199] Add a test case that ensures LocalStrategy.class is ignored from being repaired --- .../cassandra/repair/autorepair/AutoRepairUtilsTest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index 20c6545a67d0..16515862e347 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -60,6 +60,7 @@ import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.COL_REPAIR_START_TS; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.COL_REPAIR_TURN; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -77,7 +78,6 @@ public class AutoRepairUtilsTest extends CQLTester static IEndpointSnitch defaultSnitch; - @BeforeClass public static void setupClass() throws Exception { @@ -473,4 +473,11 @@ public void testMyTurnToRunRepairShouldReturnMyTurnWhenRepairOngoing() assertEquals(AutoRepairUtils.RepairTurn.MY_TURN, AutoRepairUtils.myTurnToRunRepair(repairType, myID)); } + + @Test + public void testLocalStrategyAndNetworkKeyspace() + { + assertFalse(AutoRepairUtils.checkNodeContainsKeyspaceReplica(Keyspace.open("system"))); + assertTrue(AutoRepairUtils.checkNodeContainsKeyspaceReplica(Keyspace.open(KEYSPACE))); + } } From 02e5ea084c09bc9af4781c77dbc681a8699f9439 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 16 Aug 2024 15:14:20 -0700 Subject: [PATCH 005/199] Shuffle keyspaces and tables when running auto-repair --- .../repair/autorepair/AutoRepair.java | 14 +++++++++- .../AutoRepairParameterizedTest.java | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index bb3265e00d22..c8fcb0eb2bc2 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -27,6 +27,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import java.util.function.Supplier; import com.google.common.annotations.VisibleForTesting; @@ -78,6 +79,9 @@ public class AutoRepair @VisibleForTesting protected final Map repairStates; + @VisibleForTesting + protected static Consumer> shuffleFunc = java.util.Collections::shuffle; + protected final Map tokenRangeSplitters = new EnumMap<>(AutoRepairConfig.RepairType.class); @@ -181,7 +185,14 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) repairState.setRepairSkippedTablesCount(0); repairState.setRepairInProgress(true); repairState.setTotalMVTablesConsideredForRepair(0); - for (Keyspace keyspace : Keyspace.all()) + + List keyspaces = new ArrayList<>(); + Keyspace.all().forEach(keyspaces::add); + // Auto-repair is likely to be run on multiple nodes independently, we want to avoid running multiple repair + // sessions on overlapping datasets at the same time. Shuffling keyspaces reduces the likelihood of this happening. + shuffleFunc.accept(keyspaces); + + for (Keyspace keyspace : keyspaces) { if (!AutoRepairUtils.checkNodeContainsKeyspaceReplica(keyspace)) { @@ -190,6 +201,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) repairState.setRepairKeyspaceCount(repairState.getRepairKeyspaceCount() + 1); List tablesToBeRepaired = retrieveTablesToBeRepaired(keyspace, repairType, repairState); + shuffleFunc.accept(tablesToBeRepaired); for (String tableName : tablesToBeRepaired) { String keyspaceName = keyspace.getName(); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index bbf3714e315b..2b7e1bd6afbc 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -182,6 +182,8 @@ public void setup() AutoRepair.timeFunc = System::currentTimeMillis; resetCounters(); resetConfig(); + + AutoRepair.shuffleFunc = java.util.Collections::shuffle; } @@ -577,4 +579,28 @@ public void testDefaultAutomatedRepair() Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).repairEnabled()); } + + @Test + public void testRepairShufflesKeyspacesAndTables() + { + AtomicInteger shuffleKeyspacesCall = new AtomicInteger(); + AtomicInteger shuffleTablesCall = new AtomicInteger(); + AutoRepair.shuffleFunc = (List list) -> { + assertTrue(list.get(0) instanceof Keyspace || list.get(0) instanceof String); + if (list.get(0) instanceof Keyspace) + { + shuffleKeyspacesCall.getAndIncrement(); + assertFalse(list.isEmpty()); + } + else if (list.get(0) instanceof String) + { + shuffleTablesCall.getAndIncrement(); + } + }; + + AutoRepair.instance.repair(repairType, 0); + + assertEquals(1, shuffleKeyspacesCall.get()); + assertEquals(4, shuffleTablesCall.get()); + } } From d2ecbdeb8b30422a2a81874393d32bc87fbc2760 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sat, 17 Aug 2024 10:06:52 -0700 Subject: [PATCH 006/199] Prevent node restart from scheduling a premature auto-repair cycle --- .../repair/autorepair/AutoRepair.java | 33 ++++++------ .../repair/autorepair/AutoRepairUtils.java | 28 ++++++++++ .../AutoRepairParameterizedTest.java | 51 ++++++++++++------- .../autorepair/AutoRepairUtilsTest.java | 20 ++++++++ 4 files changed, 100 insertions(+), 32 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index c8fcb0eb2bc2..c7a2a1210709 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -168,8 +168,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) // When doing force repair, we want to repair without -pr. boolean primaryRangeOnly = config.getRepairPrimaryTokenRangeOnly(repairType) && turn != MY_TURN_FORCE_REPAIR; - repairState.setTotalTablesConsideredForRepair(0); - if (tooSoonToRunRepair(repairType, repairState, config)) + if (tooSoonToRunRepair(repairType, repairState, config, myId)) { return; } @@ -184,6 +183,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) repairState.setRepairFailedTablesCount(0); repairState.setRepairSkippedTablesCount(0); repairState.setRepairInProgress(true); + repairState.setTotalTablesConsideredForRepair(0); repairState.setTotalMVTablesConsideredForRepair(0); List keyspaces = new ArrayList<>(); @@ -349,21 +349,24 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) } } - private boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoRepairState repairState, AutoRepairConfig config) + private boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoRepairState repairState, AutoRepairConfig config, UUID myId) { - if (repairState.getLastRepairTime() != 0) + if (repairState.getLastRepairTime() == 0) { - /** check if it is too soon to run repair. one of the reason we - * should not run frequent repair is that repair triggers - * memtable flush - */ - long timeElapsedSinceLastRepair = TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - repairState.getLastRepairTime()); - if (timeElapsedSinceLastRepair < config.getRepairMinInterval(repairType).toSeconds()) - { - logger.info("Too soon to run repair, last repair was done {} seconds ago", - timeElapsedSinceLastRepair); - return true; - } + // the node has either just boooted or has not run repair before, + // we should check for the node's repair history in the DB + repairState.setLastRepairTime(AutoRepairUtils.getLastRepairTimeForNode(repairType, myId)); + } + /** check if it is too soon to run repair. one of the reason we + * should not run frequent repair is that repair triggers + * memtable flush + */ + long timeElapsedSinceLastRepair = TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - repairState.getLastRepairTime()); + if (timeElapsedSinceLastRepair < config.getRepairMinInterval(repairType).toSeconds()) + { + logger.info("Too soon to run repair, last repair was done {} seconds ago", + timeElapsedSinceLastRepair); + return true; } return false; } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index e318849fee1e..003d5cf6412e 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -140,10 +140,15 @@ public class AutoRepairUtils , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_FORCE_REPAIR, COL_REPAIR_TYPE, COL_HOST_ID); + final static String SELECT_LAST_REPAIR_TIME_FOR_NODE = String.format( + "SELECT %s FROM %s.%s WHERE %s = ? AND %s = ?", COL_REPAIR_FINISH_TS, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, + AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, COL_HOST_ID); + static ModificationStatement delStatementRepairHistory; static SelectStatement selectStatementRepairHistory; static ModificationStatement delStatementPriorityStatus; static SelectStatement selectStatementRepairPriority; + static SelectStatement selectLastRepairTimeForNode; static ModificationStatement addPriorityHost; static ModificationStatement insertNewRepairHistoryStatement; static ModificationStatement recordStartRepairHistoryStatement; @@ -167,6 +172,10 @@ public static void setup() .forInternalCalls()); selectStatementRepairPriority = (SelectStatement) QueryProcessor.getStatement(SELECT_REPAIR_PRIORITY, ClientState .forInternalCalls()); + + selectLastRepairTimeForNode = (SelectStatement) QueryProcessor.getStatement(SELECT_LAST_REPAIR_TIME_FOR_NODE, ClientState + .forInternalCalls()); + delStatementPriorityStatus = (ModificationStatement) QueryProcessor.getStatement(DEL_REPAIR_PRIORITY, ClientState .forInternalCalls()); addPriorityHost = (ModificationStatement) QueryProcessor.getStatement(ADD_PRIORITY_HOST, ClientState @@ -367,6 +376,25 @@ public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType) return getCurrentRepairStatus(repairType, autoRepairHistories); } + public static long getLastRepairTimeForNode(RepairType repairType, UUID hostId) + { + ResultMessage.Rows rows = selectLastRepairTimeForNode.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, + Lists.newArrayList( + ByteBufferUtil.bytes(repairType.toString()), + ByteBufferUtil.bytes(hostId))), + Dispatcher.RequestTime.forImmediateExecution()); + + UntypedResultSet repairTime = UntypedResultSet.create(rows.result); + + if (repairTime.isEmpty()) + { + return 0; + } + + return repairTime.one().getLong(COL_REPAIR_FINISH_TS); + } + public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType, List autoRepairHistories) { if (autoRepairHistories != null) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 2b7e1bd6afbc..2d9333799a17 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -150,14 +150,8 @@ public static void setupClass() throws Exception QueryProcessor.executeInternal(String.format("CREATE MATERIALIZED VIEW %s.%s AS SELECT i, k from %s.%s " + "WHERE k IS NOT null AND i IS NOT null PRIMARY KEY (i, k)", KEYSPACE, MV, KEYSPACE, TABLE)); - defaultConfig = new AutoRepairConfig(true); DatabaseDescriptor.setMaterializedViewsEnabled(false); DatabaseDescriptor.setCDCEnabled(false); - for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) - { - defaultConfig.setAutoRepairEnabled(repairType, true); - defaultConfig.setMVRepairEnabled(repairType, false); - } } @Before @@ -198,6 +192,15 @@ private void resetCounters() private void resetConfig() { + // prepare a fresh default config + defaultConfig = new AutoRepairConfig(true); + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + defaultConfig.setAutoRepairEnabled(repairType, true); + defaultConfig.setMVRepairEnabled(repairType, false); + } + + // reset the AutoRepairService config to default AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.repair_type_overrides = defaultConfig.repair_type_overrides; config.global_settings = defaultConfig.global_settings; @@ -263,9 +266,6 @@ public void testTooFrequentRepairs() long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); Assert.assertEquals(String.format("Expected repair time to be same, actual value lastRepairTime1 %d, " + "lastRepairTime2 %d", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); - consideredTables = AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair(); - Assert.assertEquals("Expected total repaired tables = 0, actual value: " + consideredTables, - consideredTables, 0); assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); } @@ -367,24 +367,23 @@ public void testMVRepair() { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setMVRepairEnabled(repairType, true); - AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(0); + config.setRepairMinInterval(repairType, "0s"); + AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); AutoRepair.instance.repair(repairType, 0); assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); config.setMVRepairEnabled(repairType, false); - AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(0); + AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); AutoRepair.instance.repair(repairType, 0); assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); config.setMVRepairEnabled(repairType, true); - AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(0); + AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); AutoRepair.instance.repair(repairType, 0); assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); - - config.setMVRepairEnabled(repairType, false); } @Test @@ -396,6 +395,7 @@ public void testSkipRepairSSTableCountHigherThreshold() ColumnFamilyStore cfsMVTable = Keyspace.open(KEYSPACE).getColumnFamilyStore(MV); Set preBaseTable = cfsBaseTable.getLiveSSTables(); Set preMVTable = cfsBaseTable.getLiveSSTables(); + config.setRepairMinInterval(repairType, "0s"); for (int i = 0; i < 10; i++) { @@ -436,9 +436,6 @@ public void testSkipRepairSSTableCountHigherThreshold() assertEquals(0, state.getRepairSkippedTablesCount()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); - - config.setMVRepairEnabled(repairType, false); - config.setRepairSSTableCountHigherThreshold(repairType, beforeCount); } @Test @@ -598,9 +595,29 @@ else if (list.get(0) instanceof String) } }; + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setRepairMinInterval(repairType, "0s"); AutoRepair.instance.repair(repairType, 0); assertEquals(1, shuffleKeyspacesCall.get()); assertEquals(4, shuffleTablesCall.get()); } + + @Test + public void testRepairTakesLastRepairTimeFromDB() + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setMVRepairEnabled(repairType, true); + long lastRepairTime = System.currentTimeMillis() - 1000; + AutoRepairUtils.insertNewRepairHistory(repairType, 0, lastRepairTime); + AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(0); + config.setRepairMinInterval(repairType, "1h"); + + AutoRepair.instance.repair(repairType, 0); + + // repair scheduler should not attempt to run repair as last repair time in DB is current time - 1s + assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair()); + // repair scheduler should load the repair time from the DB + assertEquals(lastRepairTime, AutoRepair.instance.repairStates.get(repairType).getLastRepairTime()); + } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index 16515862e347..6597c7159931 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -480,4 +480,24 @@ public void testLocalStrategyAndNetworkKeyspace() assertFalse(AutoRepairUtils.checkNodeContainsKeyspaceReplica(Keyspace.open("system"))); assertTrue(AutoRepairUtils.checkNodeContainsKeyspaceReplica(Keyspace.open(KEYSPACE))); } + + @Test + public void testGetLastRepairTimeForNode() + { + UUID myID = UUID.randomUUID(); + UUID otherID = UUID.randomUUID(); + long currentMillis = System.currentTimeMillis(); + AutoRepairUtils.insertNewRepairHistory(repairType, myID, currentMillis, currentMillis - 100); + AutoRepairUtils.insertNewRepairHistory(repairType, otherID, currentMillis, currentMillis + 100); + + assertEquals(currentMillis - 100, AutoRepairUtils.getLastRepairTimeForNode(repairType, myID)); + } + + @Test + public void testGetLastRepairTimeForNodeWhenHistoryIsEmpty() + { + UUID myID = UUID.randomUUID(); + + assertEquals(0, AutoRepairUtils.getLastRepairTimeForNode(repairType, myID)); + } } From c003060141d6c78ca9863021e2f35e437a63d10b Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sat, 17 Aug 2024 10:26:17 -0700 Subject: [PATCH 007/199] Implement SucceededTokenRangesCount and FailedTokenRangesCount metrics for auto-repair --- .../cassandra/metrics/AutoRepairMetrics.java | 15 ++++- .../repair/autorepair/AutoRepair.java | 28 ++++----- .../repair/autorepair/AutoRepairState.java | 46 +++++++-------- .../AutoRepairParameterizedTest.java | 6 +- .../autorepair/AutoRepairStateTest.java | 59 +++++++++---------- 5 files changed, 79 insertions(+), 75 deletions(-) diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java index cd46c462fb66..426034ffcd4b 100644 --- a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java @@ -36,7 +36,8 @@ public class AutoRepairMetrics public Gauge clusterRepairTimeInSec; public Gauge skippedTablesCount; public Gauge longestUnrepairedSec; - public Gauge failedTablesCount; + public Gauge succeededTokenRangesCount; + public Gauge failedTokenRangesCount; public Counter repairTurnMyTurn; public Counter repairTurnMyTurnDueToPriority; public Counter repairTurnMyTurnForceRepair; @@ -87,11 +88,19 @@ public Integer getValue() } }); - failedTablesCount = Metrics.register(factory.createMetricName("FailedTablesCount"), new Gauge() + succeededTokenRangesCount = Metrics.register(factory.createMetricName("SucceededTokenRangesCount"), new Gauge() { public Integer getValue() { - return AutoRepair.instance.getRepairState(repairType).getRepairFailedTablesCount(); + return AutoRepair.instance.getRepairState(repairType).getSucceededTokenRangesCount(); + } + }); + + failedTokenRangesCount = Metrics.register(factory.createMetricName("FailedTokenRangesCount"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).getFailedTokenRangesCount(); } }); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index c7a2a1210709..0e6d70d1e9d0 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -179,13 +179,14 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) AutoRepairUtils.updateStartAutoRepairHistory(repairType, myId, timeFunc.get(), turn); repairState.setRepairKeyspaceCount(0); - repairState.setRepairTableSuccessCount(0); - repairState.setRepairFailedTablesCount(0); repairState.setRepairSkippedTablesCount(0); repairState.setRepairInProgress(true); repairState.setTotalTablesConsideredForRepair(0); repairState.setTotalMVTablesConsideredForRepair(0); + int failedTokenRanges = 0; + int succeededTokenRanges = 0; + List keyspaces = new ArrayList<>(); Keyspace.all().forEach(keyspaces::add); // Auto-repair is likely to be run on multiple nodes independently, we want to avoid running multiple repair @@ -234,7 +235,6 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) logger.info("Repair table {}.{}", keyspaceName, tableName); } long tableStartTime = timeFunc.get(); - boolean repairSuccess = true; Set> ranges = new HashSet<>(); List> subRangesToBeRepaired = tokenRangeSplitters.get(repairType).getRange(repairType, primaryRangeOnly, keyspaceName, tableName); int totalSubRanges = subRangesToBeRepaired.size(); @@ -298,28 +298,20 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) logger.info("Repair completed for range {}-{} for {}.{}, total subranges: {}," + "processed subranges: {}", childStartToken, childEndToken, keyspaceName, config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, totalSubRanges, totalProcessedSubRanges); + succeededTokenRanges += ranges.size(); } else { - repairSuccess = false; boolean cancellationStatus = f.cancel(true); //in future we can add retry, etc. logger.info("Repair failed for range {}-{} for {}.{} total subranges: {}," + "processed subranges: {}, cancellationStatus: {}", childStartToken, childEndToken, keyspaceName, config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, totalSubRanges, totalProcessedSubRanges, cancellationStatus); + failedTokenRanges += ranges.size(); } ranges.clear(); } } - int touchedTables = config.getRepairByKeyspace(repairType) ? tablesToBeRepaired.size() : 1; - if (repairSuccess) - { - repairState.setRepairTableSuccessCount(repairState.getRepairTableSuccessCount() + touchedTables); - } - else - { - repairState.setRepairFailedTablesCount(repairState.getRepairFailedTablesCount() + touchedTables); - } if (config.getRepairByKeyspace(repairType)) { logger.info("Repair completed for keyspace {}, tables: {}", keyspaceName, tablesToBeRepaired); @@ -336,7 +328,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) } } } - cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, millisToWait); + cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, millisToWait, failedTokenRanges, succeededTokenRanges); } else { @@ -395,7 +387,7 @@ private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairCon } private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, UUID myId, - long startTime, long millisToWait) throws InterruptedException + long startTime, long millisToWait, int failedTokenRanges, int succeededTokenRanges) throws InterruptedException { //if it was due to priority then remove it now if (turn == MY_TURN_DUE_TO_PRIORITY) @@ -404,12 +396,14 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType AutoRepairUtils.removePriorityStatus(repairType, myId); } + repairState.setFailedTokenRangesCount(failedTokenRanges); + repairState.setSucceededTokenRangesCount(succeededTokenRanges); repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - startTime)); long timeInHours = TimeUnit.SECONDS.toHours(repairState.getNodeRepairTimeInSec()); logger.info("Local {} repair time {} hour(s), stats: repairKeyspaceCount {}, " + - "repairTableSuccessCount {}, repairTableFailureCount {}, " + + "repairTokenRangesSuccessCount {}, repairTokenRangesFailureCount {}, " + "repairTableSkipCount {}", repairType, timeInHours, repairState.getRepairKeyspaceCount(), - repairState.getRepairTableSuccessCount(), repairState.getRepairFailedTablesCount(), + repairState.getSucceededTokenRangesCount(), repairState.getFailedTokenRangesCount(), repairState.getRepairSkippedTablesCount()); if (repairState.getLastRepairTime() != 0) { diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index 7f6e84bcb665..87fdace2693c 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -76,16 +76,16 @@ public abstract class AutoRepairState implements ProgressListener @VisibleForTesting protected int repairKeyspaceCount = 0; @VisibleForTesting - protected int repairTableSuccessCount = 0; - @VisibleForTesting protected int repairTableSkipCount = 0; @VisibleForTesting - protected int repairTableFailureCount = 0; - @VisibleForTesting protected int totalMVTablesConsideredForRepair = 0; - @VisibleForTesting protected int totalDisabledTablesRepairCount = 0; + + @VisibleForTesting + protected int failedTokenRangesCount = 0; + @VisibleForTesting + protected int succeededTokenRangesCount = 0; @VisibleForTesting protected AutoRepairHistory longestUnrepairedNode; @VisibleForTesting @@ -203,16 +203,6 @@ public int getLongestUnrepairedSec() return (int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - longestUnrepairedNode.getLastRepairFinishTime()); } - public void setRepairFailedTablesCount(int count) - { - repairTableFailureCount = count; - } - - public int getRepairFailedTablesCount() - { - return repairTableFailureCount; - } - public void setTotalMVTablesConsideredForRepair(int count) { totalMVTablesConsideredForRepair = count; @@ -238,24 +228,34 @@ public void setRepairKeyspaceCount(int count) repairKeyspaceCount = count; } - public void setRepairTableSuccessCount(int count) + public int getRepairKeyspaceCount() + { + return repairKeyspaceCount; + } + + public void setLongestUnrepairedNode(AutoRepairHistory longestUnrepairedNode) { - repairTableSuccessCount = count; + this.longestUnrepairedNode = longestUnrepairedNode; } - public int getRepairKeyspaceCount() + public void setFailedTokenRangesCount(int count) { - return repairKeyspaceCount; + failedTokenRangesCount = count; } - public int getRepairTableSuccessCount() + public int getFailedTokenRangesCount() { - return repairTableSuccessCount; + return failedTokenRangesCount; } - public void setLongestUnrepairedNode(AutoRepairHistory longestUnrepairedNode) + public void setSucceededTokenRangesCount(int count) { - this.longestUnrepairedNode = longestUnrepairedNode; + succeededTokenRangesCount = count; + } + + public int getSucceededTokenRangesCount() + { + return succeededTokenRangesCount; } public boolean isSuccess() diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 2d9333799a17..bd261760be16 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -475,12 +475,14 @@ public void testMetrics() AutoRepair.instance.repairStates.put(repairType, autoRepairState); when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())) .thenReturn(repairRunnable); - when(autoRepairState.getRepairFailedTablesCount()).thenReturn(10); + when(autoRepairState.getFailedTokenRangesCount()).thenReturn(10); + when(autoRepairState.getSucceededTokenRangesCount()).thenReturn(11); when(autoRepairState.getLongestUnrepairedSec()).thenReturn(10); AutoRepair.instance.repair(repairType, 0); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); - assertTrue(AutoRepairMetricsManager.getMetrics(repairType).failedTablesCount.getValue() > 0); + assertTrue(AutoRepairMetricsManager.getMetrics(repairType).failedTokenRangesCount.getValue() > 0); + assertTrue(AutoRepairMetricsManager.getMetrics(repairType).succeededTokenRangesCount.getValue() > 0); assertTrue(AutoRepairMetricsManager.getMetrics(repairType).longestUnrepairedSec.getValue() > 0); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java index 972b6339954f..19aa0a8b439c 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -247,23 +247,6 @@ public void testGetLongestUnrepairedSec() { } } - @Test - public void testSetRepairFailedTablesCount() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); - - state.setRepairFailedTablesCount(1); - - assertEquals(1, state.repairTableFailureCount); - } - - @Test - public void testGetRepairFailedTablesCount() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); - state.repairTableFailureCount = 1; - - assertEquals(1, state.getRepairFailedTablesCount()); - } - @Test public void testSetTotalMVTablesConsideredForRepair() { AutoRepairState state = RepairType.getAutoRepairState(repairType); @@ -307,40 +290,56 @@ public void testSetRepairKeyspaceCount() { assertEquals(1, state.repairKeyspaceCount); } + @Test + public void testGetRepairKeyspaceCount() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.repairKeyspaceCount = 1; + + assertEquals(1, state.getRepairKeyspaceCount()); + } @Test - public void testSetRepairTableSuccessCount() { + public void testSetLongestUnrepairedNode() { AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairHistory history = new AutoRepairHistory(UUID.randomUUID(), "", 0, 0, null, 0, false); - state.setRepairTableSuccessCount(1); + state.setLongestUnrepairedNode(history); - assertEquals(1, state.repairTableSuccessCount); + assertEquals(history, state.longestUnrepairedNode); } @Test - public void testGetRepairKeyspaceCount() { + public void testSetSucceededTokenRangesCount() { AutoRepairState state = RepairType.getAutoRepairState(repairType); - state.repairKeyspaceCount = 1; - assertEquals(1, state.getRepairKeyspaceCount()); + state.setSucceededTokenRangesCount(1); + + assertEquals(1, state.succeededTokenRangesCount); } @Test - public void testGetRepairTableSuccessCount() { + public void testGetSucceededTokenRangesCount() { AutoRepairState state = RepairType.getAutoRepairState(repairType); - state.repairTableSuccessCount = 1; + state.succeededTokenRangesCount = 1; - assertEquals(1, state.getRepairTableSuccessCount()); + assertEquals(1, state.getSucceededTokenRangesCount()); } @Test - public void testSetLongestUnrepairedNode() { + public void testSetFailedTokenRangesCount() { AutoRepairState state = RepairType.getAutoRepairState(repairType); - AutoRepairHistory history = new AutoRepairHistory(UUID.randomUUID(), "", 0, 0, null, 0, false); - state.setLongestUnrepairedNode(history); + state.setFailedTokenRangesCount(1); - assertEquals(history, state.longestUnrepairedNode); + assertEquals(1, state.failedTokenRangesCount); + } + + @Test + public void testGetFailedTokenRangesCount() { + AutoRepairState state = RepairType.getAutoRepairState(repairType); + state.failedTokenRangesCount = 1; + + assertEquals(1, state.getFailedTokenRangesCount()); } @Test From 9aafd7ed0d626408052aab12fa4e734b8e3b750f Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sat, 17 Aug 2024 10:37:51 -0700 Subject: [PATCH 008/199] Fix NPE in RepairRunnable.fail --- .../cassandra/repair/RepairRunnable.java | 9 ++- .../cassandra/repair/RepairRunnableTest.java | 61 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 test/unit/org/apache/cassandra/repair/RepairRunnableTest.java diff --git a/src/java/org/apache/cassandra/repair/RepairRunnable.java b/src/java/org/apache/cassandra/repair/RepairRunnable.java index 95a628c330b0..a5c46a918c0c 100644 --- a/src/java/org/apache/cassandra/repair/RepairRunnable.java +++ b/src/java/org/apache/cassandra/repair/RepairRunnable.java @@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -102,7 +103,9 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo private final String tag; private final List listeners = new ArrayList<>(); - private final AtomicReference firstError = new AtomicReference<>(null); + + @VisibleForTesting + protected final AtomicReference firstError = new AtomicReference<>(null); final Scheduler validationScheduler; private TraceState traceState; @@ -195,12 +198,12 @@ private void success(String msg) complete(null); } - private void fail(String reason) + protected void fail(String reason) { if (reason == null) { Throwable error = firstError.get(); - reason = error != null ? error.getMessage() : "Some repair failed"; + reason = (error != null && error.getMessage() != null) ? error.getMessage() : "Some repair failed"; } state.phase.fail(reason); ParticipateState p = ActiveRepairService.instance.participate(state.id); diff --git a/test/unit/org/apache/cassandra/repair/RepairRunnableTest.java b/test/unit/org/apache/cassandra/repair/RepairRunnableTest.java new file mode 100644 index 000000000000..2fece69a247b --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/RepairRunnableTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import org.junit.Before; +import org.junit.Test; + +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.repair.messages.RepairOption; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.streaming.PreviewKind; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.mockito.Mockito.when; + + +public class RepairRunnableTest extends CQLTester +{ + @Mock + StorageService storageService; + + @Mock + RepairOption repairOption; + + @Mock + Throwable throwable; + + @Before + public void setUp() + { + MockitoAnnotations.initMocks(this); + } + + @Test + public void testFailNPE() + { + RepairRunnable repairRunnable = new RepairRunnable(storageService, 0, repairOption, KEYSPACE); + repairRunnable.firstError.set(throwable); + when(throwable.getMessage()).thenReturn(null); + when(repairOption.getPreviewKind()).thenReturn(PreviewKind.NONE); + + repairRunnable.fail(null); + } +} From 5f73781811f5fb6901495cc06b2dfc067f0d68ca Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sat, 17 Aug 2024 10:52:47 -0700 Subject: [PATCH 009/199] Add intial_scheduler_delay option to AutoRepairConfig --- .../repair/autorepair/AutoRepair.java | 5 +---- .../repair/autorepair/AutoRepairConfig.java | 15 +++++++++++++++ .../tools/nodetool/GetAutoRepairConfig.java | 1 + .../autorepair/AutoRepairConfigTest.java | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 0e6d70d1e9d0..2968a1c40428 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -61,9 +61,6 @@ public class AutoRepair { - // Initial delay for repair session to start after setup - final static long INITIAL_REPAIR_DELAY_SEC = 30; - private static final Logger logger = LoggerFactory.getLogger(AutoRepair.class); @VisibleForTesting @@ -113,7 +110,7 @@ public void setup() { repairExecutors.get(repairType).scheduleWithFixedDelay( () -> repair(repairType, 60000), - INITIAL_REPAIR_DELAY_SEC, + config.getInitialSchedulerDelay(repairType).toSeconds(), config.getRepairCheckInterval().toSeconds(), TimeUnit.SECONDS); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index bd20c1d3c548..8e1c321dd1c0 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -254,6 +254,17 @@ public String getTokenRangeSplitter(RepairType repairType) return applyOverrides(repairType, opt -> opt.token_range_splitter); } + public void setInitialSchedulerDelay(RepairType repairType, String initialSchedulerDelay) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).intial_scheduler_delay = new DurationSpec.IntSecondsBound(initialSchedulerDelay); + } + + public DurationSpec.IntSecondsBound getInitialSchedulerDelay(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.intial_scheduler_delay); + } + // Options configures auto-repair behavior for a given repair type. // All fields can be modified dynamically. public static class Options implements Serializable @@ -288,6 +299,7 @@ protected static Options getDefaultOptions() opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h"); opts.mv_repair_enabled = true; opts.token_range_splitter = DefaultAutoRepairTokenSplitter.class.getName(); + opts.intial_scheduler_delay = new DurationSpec.IntSecondsBound("15m"); // 15 minutes return opts; } @@ -353,6 +365,8 @@ protected static Options getDefaultOptions() // the default is DefaultAutoRepairTokenSplitter.class.getName(). The class should implement IAutoRepairTokenRangeSplitter. // The default implementation splits the tokens based on the token ranges owned by this node divided by the number of 'number_of_subranges' public volatile String token_range_splitter; + // the minimum delay in seconds after a node starts before the scheduler starts running repair + public volatile DurationSpec.IntSecondsBound intial_scheduler_delay; public String toString() { @@ -371,6 +385,7 @@ public String toString() ", table_max_repair_time=" + table_max_repair_time + ", mv_repair_enabled=" + mv_repair_enabled + ", token_range_splitter=" + token_range_splitter + + ", intial_scheduler_delay=" + intial_scheduler_delay + '}'; } } diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java index 49ccf0f2aa24..feaf4c98ae17 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java @@ -71,6 +71,7 @@ private String formatRepairTypeConfig(NodeProbe probe, RepairType repairType, Au sb.append("\n\tnumber of parallel repairs within group: " + config.getParallelRepairCountInGroup(repairType)); sb.append("\n\tpercentage of parallel repairs within group: " + config.getParallelRepairPercentageInGroup(repairType)); sb.append("\n\tmv repair enabled: " + config.getMVRepairEnabled(repairType)); + sb.append("\n\tinitial scheduler delay: " + config.getInitialSchedulerDelay(repairType)); return sb.toString(); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index dea0c9a827c3..aa3db7835d54 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -450,4 +450,22 @@ public void testInvalidTokenRangeSplitter() assertEquals(DefaultAutoRepairTokenSplitter.class.getName(), FBUtilities.newAutoRepairTokenRangeSplitter("invalid-class").getClass().getName()); } + @Test + public void testSetInitialSchedulerDelay() + { + config.setInitialSchedulerDelay(repairType, "5s"); + + assert config.repair_type_overrides.get(repairType).intial_scheduler_delay.toSeconds() == 5; + } + + @Test + public void testGetInitialSchedulerDelay() + { + config.global_settings.intial_scheduler_delay = new DurationSpec.IntSecondsBound("5s"); + + int result = config.getInitialSchedulerDelay(repairType).toSeconds(); + + assertEquals(5, result); + } + } From ac0039b7998330a963f2733cf12da569c907a1dc Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 18 Aug 2024 10:19:20 -0700 Subject: [PATCH 010/199] Implement nodetool sstablerepairedset --- .../cassandra/service/StorageService.java | 33 ++++ .../service/StorageServiceMBean.java | 7 + .../org/apache/cassandra/tools/NodeProbe.java | 10 ++ .../org/apache/cassandra/tools/NodeTool.java | 1 + .../tools/nodetool/SSTableRepairedSet.java | 112 ++++++++++++ .../service/StorageServiceServerTest.java | 167 +++++++++++++++--- .../cassandra/tools/JMXStandardsTest.java | 2 + .../nodetool/SSTableRepairedSetTest.java | 114 ++++++++++++ 8 files changed, 424 insertions(+), 22 deletions(-) create mode 100644 src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java create mode 100644 test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 9928eac0eef7..f35861ae50f8 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -84,6 +84,8 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.RateLimiter; import com.google.common.util.concurrent.Uninterruptibles; + +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.repair.autorepair.AutoRepairKeyspace; import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.commons.lang3.StringUtils; @@ -7064,4 +7066,35 @@ public void setEnforceNativeDeadlineForHints(boolean value) DatabaseDescriptor.setEnforceNativeDeadlineForHints(value); } + public List getTablesForKeyspace(String keyspace) { + return Keyspace.open(keyspace).getColumnFamilyStores().stream().map(cfs -> cfs.name).collect(Collectors.toList()); + } + + public List mutateSSTableRepairedState(boolean repaired, boolean preview, String keyspace, List tableNames) throws InvalidRequestException + { + Map tables = Keyspace.open(keyspace).getColumnFamilyStores() + .stream().collect(Collectors.toMap(c -> c.name, c -> c)); + for (String tableName : tableNames) { + if (!tables.containsKey(tableName)) + throw new InvalidRequestException("Table " + tableName + " does not exist in keyspace " + keyspace); + } + + // only select SSTables that are unrepaired when repaired is true and vice versa + Predicate predicate = sst -> repaired != sst.isRepaired(); + + // mutate SSTables + long repairedAt = !repaired ? 0 : currentTimeMillis(); + List sstablesTouched = new ArrayList<>(); + for (String tableName : tableNames) { + ColumnFamilyStore table = tables.get(tableName); + Set result = table.runWithCompactionsDisabled(() -> { + Set sstables = table.getLiveSSTables().stream().filter(predicate).collect(Collectors.toSet()); + if (!preview) + table.getCompactionStrategyManager().mutateRepaired(sstables, repairedAt, null, false); + return sstables; + }, predicate, true, false, true); + sstablesTouched.addAll(result.stream().map(sst -> sst.descriptor.baseFilename()).collect(Collectors.toList())); + } + return sstablesTouched; + } } diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 4f5eeb131bdd..475ed2281e10 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -36,6 +36,7 @@ import org.apache.cassandra.db.ColumnFamilyStoreMBean; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.utils.BreaksJMX; +import org.apache.cassandra.exceptions.InvalidRequestException; public interface StorageServiceMBean extends NotificationEmitter { @@ -1069,4 +1070,10 @@ public void enableAuditLog(String loggerName, String includedKeyspaces, String e boolean getEnforceNativeDeadlineForHints(); void setEnforceNativeDeadlineForHints(boolean value); + + /** Gets the names of all tables for the given keyspace */ + public List getTablesForKeyspace(String keyspace); + + /** Mutates the repaired state of all SSTables for the given SSTables */ + public List mutateSSTableRepairedState(boolean repaired, boolean preview, String keyspace, List tables) throws InvalidRequestException; } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index d4f6364d89f8..331916f26431 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -75,6 +75,7 @@ import org.apache.cassandra.db.ColumnFamilyStoreMBean; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.CompactionManagerMBean; +import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.fql.FullQueryLoggerOptions; import org.apache.cassandra.fql.FullQueryLoggerOptionsCompositeData; import org.apache.cassandra.gms.FailureDetector; @@ -2221,6 +2222,15 @@ public void setMVRepairEnabled(AutoRepairConfig.RepairType repairType, boolean e { autoRepairProxy.setMVRepairEnabled(repairType, enabled); } + + public List mutateSSTableRepairedState(boolean repair, boolean preview, String keyspace, List tables) throws InvalidRequestException + { + return ssProxy.mutateSSTableRepairedState(repair, preview, keyspace, tables); + } + + public List getTablesForKeyspace(String keyspace) { + return ssProxy.getTablesForKeyspace(keyspace); + } } class ColumnFamilyStoreMBeanIterator implements Iterator> diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index dd3291b6d2ac..875d77ea3148 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -211,6 +211,7 @@ public int execute(String... args) SetTraceProbability.class, Sjk.class, Snapshot.class, + SSTableRepairedSet.class, Status.class, StatusAutoCompaction.class, StatusBackup.class, diff --git a/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java b/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java new file mode 100644 index 000000000000..32ee09f93590 --- /dev/null +++ b/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools.nodetool; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; + + +import io.airlift.airline.Arguments; +import io.airlift.airline.Command; +import io.airlift.airline.Option; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.NodeTool; + +@Command(name = "sstablerepairedset", description = "Set the repaired state of SSTables for given keyspace/tables") +public class SSTableRepairedSet extends NodeTool.NodeToolCmd +{ + @Arguments(usage = " []", description = "The keyspace optionally followed by one or more tables", required = true) + protected List args = new ArrayList<>(); + + @Option(title = "really-set", + name = { "--really-set" }, + description = "Really set the repaired state of SSTables. If not set, only print SSTables that would be affected.") + protected boolean reallySet = false; + + @Option(title = "is-repaired", + name = { "--is-repaired" }, + description = "Set SSTables to repaired state.") + protected boolean isRepaired = false; + + @Option(title = "is-unrepaired", + name = { "--is-unrepaired" }, + description = "Set SSTables to unrepaired state.") + protected boolean isUnrepaired = false; + + @Override + public void execute(NodeProbe probe) + { + PrintStream out = probe.output().out; + + String message; + if (reallySet) + { + message = "Mutating repaired state of SSTables for"; + } + else + { + message = "Previewing repaired state mutation of SSTables for"; + } + + if (args.isEmpty()) + { + out.println("At least a keyspace name must be provided."); + return; + } + String keyspace = args.get(0); + + List tables; + if (args.size() > 1) + { + tables = args.subList(1, args.size()); + message += " tables " + String.join(", ", tables) + " in"; + } + else + { + tables = probe.getTablesForKeyspace(keyspace); + message += " all tables in"; + } + message += " keyspace " + keyspace; + + if (isRepaired == isUnrepaired) + { + out.println("Exactly one of --is-repaired or --is-unrepaired must be provided."); + return; + } + message += " to " + (isRepaired ? "repaired" : "unrepaired"); + out.println(message); + + try + { + List mutatedSSTables = probe.mutateSSTableRepairedState(isRepaired, !reallySet, keyspace, new ArrayList<>(tables)); + if (!reallySet) + out.println("The following SSTables would be mutated:"); + else + out.println("The following SSTables were mutated:"); + for (String sstable : mutatedSSTables) + out.println(sstable); + } + catch (InvalidRequestException e) + { + out.println(e.getMessage()); + } + } +} diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java index 119b8104d4a9..d4f6d6a7e475 100644 --- a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java +++ b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java @@ -1,21 +1,21 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.cassandra.service; @@ -23,12 +23,17 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.*; +import java.util.stream.Collectors; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.SystemKeyspace; + import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -42,6 +47,8 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.gms.ApplicationState; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.gms.VersionedValue; @@ -56,10 +63,17 @@ import static org.apache.cassandra.ServerTestUtils.cleanup; import static org.apache.cassandra.ServerTestUtils.mkdirs; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public class StorageServiceServerTest { + public static final String keyspace = "test_keyspace"; + public static ColumnFamilyStore table1; + public static ColumnFamilyStore table2; + @BeforeClass public static void setUp() throws ConfigurationException { @@ -72,6 +86,21 @@ public static void setUp() throws ConfigurationException mkdirs(); cleanup(); StorageService.instance.initServer(0); + + SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), + SchemaLoader.standardCFMD(keyspace, "table1").build(), + SchemaLoader.standardCFMD(keyspace, "table2").build()); + table1 = Keyspace.open(keyspace).getColumnFamilyStore("table1"); + assert table1 != null; + table2 = Keyspace.open(keyspace).getColumnFamilyStore("table2"); + assert table2 != null; + } + + @Before + public void clearData() + { + table1.truncateBlocking(); + table2.truncateBlocking(); } @Test @@ -552,6 +581,7 @@ public void testCreateRepairRangeFrom() throws Exception /** * Test that StorageService.getNativeAddress returns the correct value based on available yaml and gossip state + * * @throws Exception */ @Test @@ -563,7 +593,7 @@ public void testGetNativeAddress() throws Exception //Default to using the provided address with the configured port assertEquals("127.0.0.2:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeaddress(internalAddress, true)); - VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(Murmur3Partitioner.instance); + VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(Murmur3Partitioner.instance); //If we don't have the port use the gossip address, but with the configured port Gossiper.instance.getEndpointStateForEndpoint(internalAddress).addApplicationState(ApplicationState.RPC_ADDRESS, valueFactory.rpcaddress(InetAddress.getByName("127.0.0.3"))); assertEquals("127.0.0.3:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeaddress(internalAddress, true)); @@ -584,7 +614,7 @@ public void testGetNativeAddressIPV6() throws Exception //Default to using the provided address with the configured port assertEquals("[0:0:0:0:0:0:0:3]:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeaddress(internalAddressIPV6, true)); - VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(Murmur3Partitioner.instance); + VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(Murmur3Partitioner.instance); //If RPC_ADDRESS is present with an IPv6 address, we should properly bracket encode the IP with the configured port. Gossiper.instance.getEndpointStateForEndpoint(internalAddressIPV6).addApplicationState(ApplicationState.RPC_ADDRESS, valueFactory.rpcaddress(InetAddress.getByName("0:0:0:0:0:0:5a:3"))); assertEquals("[0:0:0:0:0:0:5a:3]:" + DatabaseDescriptor.getNativeTransportPort(), StorageService.instance.getNativeaddress(internalAddressIPV6, true)); @@ -636,7 +666,7 @@ public void isReplacingSameHostAddressAndHostIdTest() throws UnknownHostExceptio try { UUID differentHostId = UUID.randomUUID(); - Assert.assertFalse(StorageService.instance.isReplacingSameHostAddressAndHostId(differentHostId)); + assertFalse(StorageService.instance.isReplacingSameHostAddressAndHostId(differentHostId)); final String hostAddress = FBUtilities.getBroadcastAddressAndPort().getHostAddress(false); UUID localHostId = SystemKeyspace.getOrInitializeLocalHostId(); @@ -648,15 +678,108 @@ public void isReplacingSameHostAddressAndHostIdTest() throws UnknownHostExceptio // Check detects replacing the same host address with a different host id System.setProperty("cassandra.replace_address", hostAddress); - Assert.assertFalse(StorageService.instance.isReplacingSameHostAddressAndHostId(differentHostId)); + assertFalse(StorageService.instance.isReplacingSameHostAddressAndHostId(differentHostId)); // Check tolerates the DNS entry going away for the replace_address System.setProperty("cassandra.replace_address", "unresolvable.host.local."); - Assert.assertFalse(StorageService.instance.isReplacingSameHostAddressAndHostId(differentHostId)); + assertFalse(StorageService.instance.isReplacingSameHostAddressAndHostId(differentHostId)); } finally { System.clearProperty("cassandra.replace_address"); } } + + @Test + public void testGetTablesForKeyspace() + { + List result = StorageService.instance.getTablesForKeyspace(keyspace); + + assertEquals(Arrays.asList(table1.name, table2.name), result.stream().sorted().collect(Collectors.toList())); + } + + @Test + public void testGetTablesForKeyspaceNotFound() + { + String missingKeyspace = "MISSING_KEYSPACE"; + try + { + StorageService.instance.getTablesForKeyspace(missingKeyspace); + fail("Expected an AssertionError to be thrown"); + } + catch (AssertionError e) + { + assertEquals("Unknown keyspace " + missingKeyspace, e.getMessage()); + } + } + + @Test + public void testMutateSSTableRepairedStateTableNotFound() + { + try + { + StorageService.instance.mutateSSTableRepairedState(true, false, keyspace, Arrays.asList("MISSING_TABLE")); + fail("Expected an InvalidRequestException to be thrown"); + } + catch (InvalidRequestException e) + { + // Test passed + } + } + + @Test + public void testMutateSSTableRepairedStateTablePreview() + { + SchemaLoader.insertData(keyspace, table1.name, 0, 1); + table1.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + assertEquals(1, table1.getLiveSSTables().size()); + + List result = StorageService.instance.mutateSSTableRepairedState(true, true, keyspace, Arrays.asList(table1.name)); + + assertEquals(1, result.size()); + table1.getLiveSSTables().forEach(sstable -> { + assertFalse(sstable.isRepaired()); + assertTrue(result.contains(sstable.descriptor.baseFilename())); + }); + } + + @Test + public void testMutateSSTableRepairedStateTableRepaired() + { + SchemaLoader.insertData(keyspace, table1.name, 0, 1); + table1.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + SchemaLoader.insertData(keyspace, table1.name, 0, 1); + table1.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + assertEquals(2, table1.getLiveSSTables().size()); + table1.getLiveSSTables().forEach(sstable -> { + assertFalse(sstable.isRepaired()); + }); + + List result = StorageService.instance.mutateSSTableRepairedState(true, false, keyspace, Arrays.asList(table1.name)); + + assertEquals(2, result.size()); + table1.getLiveSSTables().forEach(sstable -> { + assertTrue(sstable.isRepaired()); + assertTrue(result.contains(sstable.descriptor.baseFilename())); + }); + } + + @Test + public void testMutateSSTableRepairedStateTableUnrepaired() throws Exception + { + SchemaLoader.insertData(keyspace, table1.name, 0, 1); + table1.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + SchemaLoader.insertData(keyspace, table1.name, 0, 1); + table1.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + table1.getCompactionStrategyManager().mutateRepaired(table1.getLiveSSTables(), 1, null, false); + assertEquals(2, table1.getLiveSSTables().stream().filter(SSTableReader::isRepaired).count()); + + List result = StorageService.instance.mutateSSTableRepairedState(false, false, keyspace, Arrays.asList(table1.name)); + + assertEquals(2, result.size()); + table1.getLiveSSTables().forEach(sstable -> { + assertFalse(sstable.isRepaired()); + assertTrue(result.contains(sstable.descriptor.baseFilename())); + }); + } } \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java b/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java index 87b9ff93fded..62b6b9b5ec16 100644 --- a/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java +++ b/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java @@ -50,6 +50,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.utils.BreaksJMX; import org.assertj.core.api.Assertions; import org.reflections.Reflections; @@ -98,6 +99,7 @@ public class JMXStandardsTest .add(IllegalStateException.class) .add(ClassNotFoundException.class) .add(OpenDataException.class) + .add(InvalidRequestException.class) .build(); /** * This list is a set of types under java.* and javax.*, but are too vague that could cause issues; this does not diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java new file mode 100644 index 000000000000..0a2ead81b778 --- /dev/null +++ b/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools.nodetool; + +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Arrays; + +import org.junit.Before; +import org.junit.Test; + +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.Output; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SSTableRepairedSetTest +{ + @Mock + private NodeProbe probe; + + private SSTableRepairedSet cmd; + + @Before + public void setUp() + { + MockitoAnnotations.initMocks(this); + PrintStream noopStream = new PrintStream(new OutputStream() + { + @Override + public void write(int b) + { + } + }); + when(probe.output()).thenReturn(new Output(noopStream, noopStream)); + cmd = new SSTableRepairedSet(); + } + + @Test + public void testNoKeyspace() + { + cmd.execute(probe); + verify(probe, never()).mutateSSTableRepairedState(anyBoolean(), anyBoolean(), anyString(), anyList()); + } + + @Test + public void testBothRepairedAndUnrepaired() + { + cmd.args = Arrays.asList("keyspace"); + cmd.execute(probe); + verify(probe, never()).mutateSSTableRepairedState(anyBoolean(), anyBoolean(), anyString(), anyList()); + } + + @Test + public void testNeitherRepairedNorUnrepaired() + { + cmd.args = Arrays.asList("keyspace"); + cmd.execute(probe); + verify(probe, never()).mutateSSTableRepairedState(anyBoolean(), anyBoolean(), anyString(), anyList()); + } + + @Test + public void testRepairedPreview() + { + cmd.args = Arrays.asList("keyspace"); + cmd.isRepaired = true; + cmd.execute(probe); + verify(probe).mutateSSTableRepairedState(true, true, "keyspace", new ArrayList<>()); + } + + @Test + public void testUnrepairedReallySet() + { + cmd.args = Arrays.asList("keyspace"); + cmd.isUnrepaired = true; + cmd.reallySet = true; + cmd.execute(probe); + verify(probe).mutateSSTableRepairedState(false, false, "keyspace", new ArrayList<>()); + } + + @Test + public void testExecuteWithTableNames() + { + cmd.args = Arrays.asList("keyspace", "table1", "table2"); + cmd.isRepaired = true; + cmd.reallySet = true; + cmd.execute(probe); + verify(probe).mutateSSTableRepairedState(true, false, "keyspace", Arrays.asList("table1", "table2")); + } +} From 1d54edf83b1a6e98b41225ee4f1a7a03022e7c6c Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 25 Aug 2024 14:07:59 -0700 Subject: [PATCH 011/199] Change SkippedTablesCount metric to SkippedTokenRangesCount --- .../cassandra/metrics/AutoRepairMetrics.java | 7 +++--- .../repair/autorepair/AutoRepair.java | 22 +++++++++-------- .../repair/autorepair/AutoRepairConfig.java | 10 ++++---- .../repair/autorepair/AutoRepairState.java | 24 +++++++++---------- .../autorepair/AutoRepairConfigTest.java | 4 ++-- .../AutoRepairParameterizedTest.java | 19 +++++++-------- .../autorepair/AutoRepairStateTest.java | 12 +++++----- 7 files changed, 49 insertions(+), 49 deletions(-) diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java index 426034ffcd4b..d765899631b7 100644 --- a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java @@ -34,10 +34,11 @@ public class AutoRepairMetrics public Gauge repairsInProgress; public Gauge nodeRepairTimeInSec; public Gauge clusterRepairTimeInSec; - public Gauge skippedTablesCount; public Gauge longestUnrepairedSec; public Gauge succeededTokenRangesCount; public Gauge failedTokenRangesCount; + public Gauge skippedTokenRangesCount; + public Counter repairTurnMyTurn; public Counter repairTurnMyTurnDueToPriority; public Counter repairTurnMyTurnForceRepair; @@ -72,11 +73,11 @@ public Integer getValue() } }); - skippedTablesCount = Metrics.register(factory.createMetricName("SkippedTablesCount"), new Gauge() + skippedTokenRangesCount = Metrics.register(factory.createMetricName("SkippedTokenRangesCount"), new Gauge() { public Integer getValue() { - return AutoRepair.instance.getRepairState(repairType).getRepairSkippedTablesCount(); + return AutoRepair.instance.getRepairState(repairType).getSkippedTokenRangesCount(); } }); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 2968a1c40428..1808cdef3a95 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -176,13 +176,13 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) AutoRepairUtils.updateStartAutoRepairHistory(repairType, myId, timeFunc.get(), turn); repairState.setRepairKeyspaceCount(0); - repairState.setRepairSkippedTablesCount(0); repairState.setRepairInProgress(true); repairState.setTotalTablesConsideredForRepair(0); repairState.setTotalMVTablesConsideredForRepair(0); int failedTokenRanges = 0; int succeededTokenRanges = 0; + int skippedTokenRanges = 0; List keyspaces = new ArrayList<>(); Keyspace.all().forEach(keyspaces::add); @@ -205,6 +205,9 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) String keyspaceName = keyspace.getName(); try { + List> subRangesToBeRepaired = tokenRangeSplitters.get(repairType).getRange(repairType, primaryRangeOnly, keyspaceName, tableName); + int totalSubRanges = subRangesToBeRepaired.size(); + ColumnFamilyStore columnFamilyStore = keyspace.getColumnFamilyStore(tableName); if (!columnFamilyStore.metadata().params.automatedRepair.get(repairType).repairEnabled()) { @@ -219,7 +222,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) { logger.info("Too many SSTables for repair, not doing repair on table {}.{} " + "totalSSTables {}", keyspaceName, tableName, columnFamilyStore.getLiveSSTables().size()); - repairState.setRepairSkippedTablesCount(repairState.getRepairSkippedTablesCount() + 1); + skippedTokenRanges += totalSubRanges; continue; } @@ -233,8 +236,6 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) } long tableStartTime = timeFunc.get(); Set> ranges = new HashSet<>(); - List> subRangesToBeRepaired = tokenRangeSplitters.get(repairType).getRange(repairType, primaryRangeOnly, keyspaceName, tableName); - int totalSubRanges = subRangesToBeRepaired.size(); int totalProcessedSubRanges = 0; for (Pair token : subRangesToBeRepaired) { @@ -249,7 +250,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) { if (AutoRepairUtils.keyspaceMaxRepairTimeExceeded(repairType, tableStartTime, tablesToBeRepaired.size())) { - repairState.setRepairSkippedTablesCount(repairState.getRepairSkippedTablesCount() + tablesToBeRepaired.size()); + skippedTokenRanges += totalSubRanges - totalProcessedSubRanges; logger.info("Keyspace took too much time to repair hence skipping it {}", keyspaceName); break; @@ -259,7 +260,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) { if (AutoRepairUtils.tableMaxRepairTimeExceeded(repairType, tableStartTime)) { - repairState.setRepairSkippedTablesCount(repairState.getRepairSkippedTablesCount() + 1); + skippedTokenRanges += totalSubRanges - totalProcessedSubRanges; logger.info("Table took too much time to repair hence skipping it {}.{}", keyspaceName, tableName); break; @@ -325,7 +326,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) } } } - cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, millisToWait, failedTokenRanges, succeededTokenRanges); + cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, millisToWait, failedTokenRanges, succeededTokenRanges, skippedTokenRanges); } else { @@ -384,7 +385,7 @@ private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairCon } private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, UUID myId, - long startTime, long millisToWait, int failedTokenRanges, int succeededTokenRanges) throws InterruptedException + long startTime, long millisToWait, int failedTokenRanges, int succeededTokenRanges, int skippedTokenRanges) throws InterruptedException { //if it was due to priority then remove it now if (turn == MY_TURN_DUE_TO_PRIORITY) @@ -395,13 +396,14 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType repairState.setFailedTokenRangesCount(failedTokenRanges); repairState.setSucceededTokenRangesCount(succeededTokenRanges); + repairState.setSkippedTokenRangesCount(skippedTokenRanges); repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - startTime)); long timeInHours = TimeUnit.SECONDS.toHours(repairState.getNodeRepairTimeInSec()); logger.info("Local {} repair time {} hour(s), stats: repairKeyspaceCount {}, " + "repairTokenRangesSuccessCount {}, repairTokenRangesFailureCount {}, " + - "repairTableSkipCount {}", repairType, timeInHours, repairState.getRepairKeyspaceCount(), + "repairTokenRangesSkipCount {}", repairType, timeInHours, repairState.getRepairKeyspaceCount(), repairState.getSucceededTokenRangesCount(), repairState.getFailedTokenRangesCount(), - repairState.getRepairSkippedTablesCount()); + repairState.getSkippedTokenRangesCount()); if (repairState.getLastRepairTime() != 0) { repairState.setClusterRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 8e1c321dd1c0..5cbb4f9f5bb6 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -257,12 +257,12 @@ public String getTokenRangeSplitter(RepairType repairType) public void setInitialSchedulerDelay(RepairType repairType, String initialSchedulerDelay) { ensureOverrides(repairType); - repair_type_overrides.get(repairType).intial_scheduler_delay = new DurationSpec.IntSecondsBound(initialSchedulerDelay); + repair_type_overrides.get(repairType).initial_scheduler_delay = new DurationSpec.IntSecondsBound(initialSchedulerDelay); } public DurationSpec.IntSecondsBound getInitialSchedulerDelay(RepairType repairType) { - return applyOverrides(repairType, opt -> opt.intial_scheduler_delay); + return applyOverrides(repairType, opt -> opt.initial_scheduler_delay); } // Options configures auto-repair behavior for a given repair type. @@ -299,7 +299,7 @@ protected static Options getDefaultOptions() opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h"); opts.mv_repair_enabled = true; opts.token_range_splitter = DefaultAutoRepairTokenSplitter.class.getName(); - opts.intial_scheduler_delay = new DurationSpec.IntSecondsBound("15m"); // 15 minutes + opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("15m"); // 15 minutes return opts; } @@ -366,7 +366,7 @@ protected static Options getDefaultOptions() // The default implementation splits the tokens based on the token ranges owned by this node divided by the number of 'number_of_subranges' public volatile String token_range_splitter; // the minimum delay in seconds after a node starts before the scheduler starts running repair - public volatile DurationSpec.IntSecondsBound intial_scheduler_delay; + public volatile DurationSpec.IntSecondsBound initial_scheduler_delay; public String toString() { @@ -385,7 +385,7 @@ public String toString() ", table_max_repair_time=" + table_max_repair_time + ", mv_repair_enabled=" + mv_repair_enabled + ", token_range_splitter=" + token_range_splitter + - ", intial_scheduler_delay=" + intial_scheduler_delay + + ", intial_scheduler_delay=" + initial_scheduler_delay + '}'; } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index 87fdace2693c..d4ab1cf34266 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -76,8 +76,6 @@ public abstract class AutoRepairState implements ProgressListener @VisibleForTesting protected int repairKeyspaceCount = 0; @VisibleForTesting - protected int repairTableSkipCount = 0; - @VisibleForTesting protected int totalMVTablesConsideredForRepair = 0; @VisibleForTesting protected int totalDisabledTablesRepairCount = 0; @@ -87,6 +85,8 @@ public abstract class AutoRepairState implements ProgressListener @VisibleForTesting protected int succeededTokenRangesCount = 0; @VisibleForTesting + protected int skippedTokenRangesCount = 0; + @VisibleForTesting protected AutoRepairHistory longestUnrepairedNode; @VisibleForTesting protected Condition condition = newOneTimeCondition(); @@ -184,16 +184,6 @@ public boolean isRepairInProgress() return repairInProgress; } - public void setRepairSkippedTablesCount(int count) - { - repairTableSkipCount = count; - } - - public int getRepairSkippedTablesCount() - { - return repairTableSkipCount; - } - public int getLongestUnrepairedSec() { if (longestUnrepairedNode == null) @@ -258,6 +248,16 @@ public int getSucceededTokenRangesCount() return succeededTokenRangesCount; } + public void setSkippedTokenRangesCount(int count) + { + skippedTokenRangesCount = count; + } + + public int getSkippedTokenRangesCount() + { + return skippedTokenRangesCount; + } + public boolean isSuccess() { return success; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index aa3db7835d54..923e35b73796 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -455,13 +455,13 @@ public void testSetInitialSchedulerDelay() { config.setInitialSchedulerDelay(repairType, "5s"); - assert config.repair_type_overrides.get(repairType).intial_scheduler_delay.toSeconds() == 5; + assert config.repair_type_overrides.get(repairType).initial_scheduler_delay.toSeconds() == 5; } @Test public void testGetInitialSchedulerDelay() { - config.global_settings.intial_scheduler_delay = new DurationSpec.IntSecondsBound("5s"); + config.global_settings.initial_scheduler_delay = new DurationSpec.IntSecondsBound("5s"); int result = config.getInitialSchedulerDelay(repairType).toSeconds(); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index bd261760be16..1ddc41ff791e 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -65,7 +65,6 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.progress.ProgressEvent; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; @@ -97,8 +96,6 @@ public class AutoRepairParameterizedTest extends CQLTester @Mock ScheduledExecutorPlus mockExecutor; @Mock - ProgressEvent progressEvent; - @Mock AutoRepairState autoRepairState; @Mock RepairRunnable repairRunnable; @@ -417,15 +414,15 @@ public void testSkipRepairSSTableCountHigherThreshold() int beforeCount = config.getRepairSSTableCountHigherThreshold(repairType); config.setMVRepairEnabled(repairType, true); config.setRepairSSTableCountHigherThreshold(repairType, 9); - assertEquals(0, state.getRepairSkippedTablesCount()); - assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + assertEquals(0, state.getSkippedTokenRangesCount()); + assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); state.setLastRepairTime(0); AutoRepair.instance.repair(repairType, 0); assertEquals(1, state.getTotalMVTablesConsideredForRepair()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); // skipping one time for the base table and another time for MV table - assertEquals(2, state.getRepairSkippedTablesCount()); - assertEquals(2, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + assertEquals(2, state.getSkippedTokenRangesCount()); + assertEquals(2, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); // set it to higher value, and this time, the tables should not be skipped config.setRepairSSTableCountHigherThreshold(repairType, 11); @@ -433,9 +430,9 @@ public void testSkipRepairSSTableCountHigherThreshold() state.setLastRepairTime(0); AutoRepair.instance.repair(repairType, 0); assertEquals(1, state.getTotalMVTablesConsideredForRepair()); - assertEquals(0, state.getRepairSkippedTablesCount()); + assertEquals(0, state.getSkippedTokenRangesCount()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); - assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); } @Test @@ -468,7 +465,7 @@ public void testMetrics() assertTrue(AutoRepairMetricsManager.getMetrics(repairType).nodeRepairTimeInSec.getValue() > 0); assertTrue(AutoRepairMetricsManager.getMetrics(repairType).clusterRepairTimeInSec.getValue() > 0); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).repairTurnMyTurn.getCount()); - assertTrue(AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue() > 0); + assertTrue(AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue() > 0); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).longestUnrepairedSec.getValue().intValue()); config.setAutoRepairTableMaxRepairTime(repairType, String.valueOf(Integer.MAX_VALUE-1) + 's'); @@ -480,7 +477,7 @@ public void testMetrics() when(autoRepairState.getLongestUnrepairedSec()).thenReturn(10); AutoRepair.instance.repair(repairType, 0); - assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); assertTrue(AutoRepairMetricsManager.getMetrics(repairType).failedTokenRangesCount.getValue() > 0); assertTrue(AutoRepairMetricsManager.getMetrics(repairType).succeededTokenRangesCount.getValue() > 0); assertTrue(AutoRepairMetricsManager.getMetrics(repairType).longestUnrepairedSec.getValue() > 0); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java index 19aa0a8b439c..340a21f97bfa 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -203,20 +203,20 @@ public void testIsRepairInProgress() { } @Test - public void testSetRepairSkippedTablesCount() { + public void testSetSkippedTokenRangesCount() { AutoRepairState state = RepairType.getAutoRepairState(repairType); - state.setRepairSkippedTablesCount(1); + state.setSkippedTokenRangesCount(1); - assertEquals(1, state.repairTableSkipCount); + assertEquals(1, state.skippedTokenRangesCount); } @Test - public void testGetRepairSkippedTablesCount() { + public void testGetSkippedTokenRangesCount() { AutoRepairState state = RepairType.getAutoRepairState(repairType); - state.repairTableSkipCount = 1; + state.skippedTokenRangesCount = 1; - assertEquals(1, state.getRepairSkippedTablesCount()); + assertEquals(1, state.getSkippedTokenRangesCount()); } @Test From 6d7c351b7f358820fbed1c521b1b607ce7032b4a Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 26 Aug 2024 15:12:41 -0700 Subject: [PATCH 012/199] config description --- .../cassandra/pages/operating/metrics.adoc | 21 +++++----- .../repair/autorepair/AutoRepairConfig.java | 42 +++++++++++-------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/doc/modules/cassandra/pages/operating/metrics.adoc b/doc/modules/cassandra/pages/operating/metrics.adoc index 68163e627a10..69948a00c90a 100644 --- a/doc/modules/cassandra/pages/operating/metrics.adoc +++ b/doc/modules/cassandra/pages/operating/metrics.adoc @@ -952,9 +952,9 @@ Metrics specifc to automated repair. Reported name format: *Metric Name*:: -`org.apache.cassandra.metrics.AutoRepairMetrics.` +`org.apache.cassandra.metrics.AutoRepair.` *JMX MBean*:: -`org.apache.cassandra.metrics:type=AutoRepairMetrics name=` +`org.apache.cassandra.metrics:type=AutoRepair name= repairType=` [cols=",,",options="header",] |=== @@ -963,19 +963,20 @@ Reported name format: on the node |NodeRepairTimeInSec |Gauge |Time taken to repair -the node +the node in seconds |ClusterRepairTimeInSec |Gauge |Time taken to repair -the entire Cassandra cluster - -|SkippedTablesCount |Gauge |Number of tables skipped -on the node +the entire Cassandra cluster in seconds |LongestUnrepairedSec |Gauge |Time since the last repair -ran on the node +ran on the node in seconds + +|SucceededTokenRangesCount |Gauge |Number of token ranges successfully repaired on the node -|FailedTablesCount |Gauge |Number of tables encountered -failure during repair on the node +|FailedTokenRangesCount |Gauge |Number of token ranges failed to repair on the node + +|SkippedTokenRangesCount |Gauge |Number of token ranges skipped +on the node |TotalMVTablesConsideredForRepair |Gauge |Number of materialized views considered on the node diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 5cbb4f9f5bb6..9522d35bcdc8 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -37,10 +37,16 @@ public class AutoRepairConfig implements Serializable { // enable/disable auto repair globally, overrides all other settings. Cannot be modified dynamically. + // if it is set to false, then no repair will be scheduled, including full and incremental repairs by this framework. + // if it is set to true, then this repair scheduler will consult another config available for each RepairType, and based on that config, it will schedule repairs. public final Boolean enabled; - // the interval in seconds between checks for eligible repair operations. Cannot be modified dynamically. + // the interval between successive checks for repair scheduler to check if either the ongoing repair is completed or if + // none is going, then check if it's time to schedule or wait public final DurationSpec.IntSecondsBound repair_check_interval = new DurationSpec.IntSecondsBound("5m"); - // configures how long repair history is kept for a replaced node + // when any nodes leave the ring then the repair schedule needs to adjust the order, etc. + // the repair scheduler keeps the deleted hosts information in its persisted metadata for the defined interval in this config. + // This information is useful so the scheduler is absolutely sure that the node is indeed removed from the ring, and then it can adjust the repair schedule accordingly. + // So, the duration in this config determinses for how long deleted host's information is kept in the scheduler's metadata. public volatile DurationSpec.IntSecondsBound history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound("2h"); // global_settings overides Options.defaultOptions for all repair types public volatile Options global_settings; @@ -287,7 +293,7 @@ protected static Options getDefaultOptions() opts.enabled = false; opts.repair_by_keyspace = false; - opts.number_of_subranges = 1; + opts.number_of_subranges = 16; opts.number_of_repair_threads = 1; opts.parallel_repair_count_in_group = 1; opts.parallel_repair_percentage_in_group = 0; @@ -316,32 +322,32 @@ protected static Options getDefaultOptions() // If you do not use v-nodes or the number of v-nodes is pretty small, say 8, setting this value to a higher number, say 16, will be useful to repair on a smaller range, and the chance of succeeding is higher. public volatile Integer number_of_subranges; // the number of repair threads to run for a given invoked Repair Job. - // Once the scheduler schedules one Job, then howmany threads to use inside that job will be controlled through this parameter. + // Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. // This is similar to -j for repair options for the nodetool repair command. public volatile Integer number_of_repair_threads; // the number of repair sessions that can run in parallel in a single group // The number of nodes running repair parallelly. If parallelrepaircount is set, it will choose the larger value of the two. The default is 3. // This configuration controls how many nodes would run repair in parallel. - // The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. + // The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. // If one or more node(s) finish repair, then the framework automatically picks up the next candidate and ensures the maximum number of nodes running repair do not exceed “3”. public volatile Integer parallel_repair_count_in_group; // the number of repair sessions that can run in parallel in a single groupas a percentage // of the total number of nodes in the group [0,100] // The percentage of nodes in the cluster that run repair parallelly. If parallelrepaircount is set, it will choose the larger value of the two. - //The problem with a fixed number of nodes (the above property) is that in a large-scale environment, + // The problem with a fixed number of nodes (the above property) is that in a large-scale environment, // the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. - //The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. + // The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. // So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. // Extremely fewer manual interventions as it will rarely violate the repair SLA for customers public volatile Integer parallel_repair_percentage_in_group; // the upper threshold of SSTables allowed to participate in a single repair session // Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. - // This is to avoid penalizing good neighbors with an outlier. + // This is to avoid penalizing good tables (neighbors) with an outlier. public volatile Integer sstable_upper_threshold; - // the minimum time in hours between repairs of the same token range - // The minimum number of hours to run one repair cycle is 24 hours. The default is 24 hours. - // This means that if auto repair finishes one round on one cluster within 24 hours, it won’t start a new round. - // This is applicable for extremely tiny clusters, say 3 nodes. + // the minimum time in hours between repairing the same node again. This is useful for extremely tiny clusters, say 5 nodes, which finishes + // repair quicly. + // The default is 24 hours. This means that if the scheduler finishes one round on all the nodes in < 24 hours. On a given node it won’t start a new repair round + // until the last repair conducted on a given node is < 24 hours. public volatile DurationSpec.IntSecondsBound min_repair_interval; // specifies a denylist of datacenters to repair // This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. @@ -350,22 +356,22 @@ protected static Options getDefaultOptions() // It is the same as -pr in nodetool repair options. public volatile Boolean repair_primary_token_range_only; // configures whether to force immediate repair on new nodes + // default it is set to 'false'; this is useful if you want to repair new nodes immediately after they join the ring. public volatile Boolean force_repair_new_node; - // the maximum time in seconds that a repair session can run for a single table - // Max time for repairing one table, if exceeded, skip the table. The default is 6 * 60 * 60, which is 6 hours. + // the maximum time that a repair session can run for a single table + // Max time for repairing one table on a given node, if exceeded, skip the table. The default is 6 hours. // Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. // Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. - // Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator telling them they are violating SLA even if I am a neighbor, and it would require a lot of back-and-forth manual interventions, etc. + // Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator and would require a lot of back-and-forth manual interventions, etc. // So, the idea here is to penalize the outliers instead of good candidates. This can easily be configured with a higher value if we want to disable the functionality. - // Please note the repair will still run in parallel on other nodes, this is to address outliers on a given node. public volatile DurationSpec.IntSecondsBound table_max_repair_time; - // the default is 'true'. MVs are mutated at LOCAL_ONE consistency level in Cassandra. + // the default is 'true'. // This flag determines whether the auto-repair framework needs to run anti-entropy, a.k.a, repair on the MV table or not. public volatile Boolean mv_repair_enabled; // the default is DefaultAutoRepairTokenSplitter.class.getName(). The class should implement IAutoRepairTokenRangeSplitter. // The default implementation splits the tokens based on the token ranges owned by this node divided by the number of 'number_of_subranges' public volatile String token_range_splitter; - // the minimum delay in seconds after a node starts before the scheduler starts running repair + // the minimum delay after a node starts before the scheduler starts running repair public volatile DurationSpec.IntSecondsBound initial_scheduler_delay; public String toString() From becf9c145eda6a2bfbc7742580d5fb997a5cb410 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 28 Aug 2024 20:45:16 -0700 Subject: [PATCH 013/199] Implement repair retries for auto-repair scheduler --- .../repair/autorepair/AutoRepair.java | 62 +++- .../repair/autorepair/AutoRepairConfig.java | 25 ++ .../repair/autorepair/AutoRepairUtils.java | 2 +- .../cassandra/service/AutoRepairService.java | 12 + .../service/AutoRepairServiceMBean.java | 2 + .../org/apache/cassandra/tools/NodeProbe.java | 10 + .../tools/nodetool/GetAutoRepairConfig.java | 2 + .../tools/nodetool/SetAutoRepairConfig.java | 31 +- .../AutoRepairParameterizedTest.java | 57 ++++ .../service/AutoRepairServiceTest.java | 202 ++++++++++++ .../nodetool/SetAutoRepairConfigTest.java | 312 ++++++++++++++++++ 11 files changed, 689 insertions(+), 28 deletions(-) create mode 100644 test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java create mode 100644 test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 1808cdef3a95..9aa8534be407 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -27,11 +27,13 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Supplier; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Uninterruptibles; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.utils.Pair; @@ -79,6 +81,9 @@ public class AutoRepair @VisibleForTesting protected static Consumer> shuffleFunc = java.util.Collections::shuffle; + @VisibleForTesting + protected static BiConsumer sleepFunc = Uninterruptibles::sleepUninterruptibly; + protected final Map tokenRangeSplitters = new EnumMap<>(AutoRepairConfig.RepairType.class); @@ -276,20 +281,38 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) if ((totalProcessedSubRanges % config.getRepairThreads(repairType) == 0) || (totalProcessedSubRanges == totalSubRanges)) { - RepairRunnable task = repairState.getRepairRunnable(keyspaceName, - config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : ImmutableList.of(tableName), - ranges, primaryRangeOnly); - repairState.resetWaitCondition(); - Future f = repairRunnableExecutors.get(repairType).submit(task); - try + int retryCount = 0; + Future f = null; + while (retryCount <= config.getRepairMaxRetries()) { - repairState.waitForRepairToComplete(); + RepairRunnable task = repairState.getRepairRunnable(keyspaceName, + config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : ImmutableList.of(tableName), + ranges, primaryRangeOnly); + repairState.resetWaitCondition(); + f = repairRunnableExecutors.get(repairType).submit(task); + try + { + repairState.waitForRepairToComplete(); + } + catch (InterruptedException e) + { + logger.error("Exception in cond await:", e); + } + if (repairState.isSuccess()) + { + break; + } + else if (retryCount < config.getRepairMaxRetries()) + { + boolean cancellationStatus = f.cancel(true); + logger.warn("Repair failed for range {}-{} for {}.{} with cancellationStatus: {} retrying after {} seconds...", + childStartToken, childEndToken, + keyspaceName, config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, + cancellationStatus, config.getRepairRetryBackoff().toSeconds()); + sleepFunc.accept(config.getRepairRetryBackoff().toSeconds(), TimeUnit.SECONDS); + } + retryCount++; } - catch (InterruptedException e) - { - logger.error("Exception in cond await:", e); - } - //check repair status if (repairState.isSuccess()) { @@ -300,11 +323,15 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) } else { - boolean cancellationStatus = f.cancel(true); - //in future we can add retry, etc. - logger.info("Repair failed for range {}-{} for {}.{} total subranges: {}," + - "processed subranges: {}, cancellationStatus: {}", childStartToken, childEndToken, - keyspaceName, config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, totalSubRanges, totalProcessedSubRanges, cancellationStatus); + boolean cancellationStatus = true; + if (f != null) + { + cancellationStatus = f.cancel(true); + } + //in the future we can add retry, etc. + logger.error("Repair failed for range {}-{} for {}.{} after {} retries, total subranges: {}," + + "processed subranges: {}, cancellationStatus: {}", childStartToken.toString(), childEndToken.toString(), keyspaceName, + config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, retryCount, totalSubRanges, totalProcessedSubRanges, cancellationStatus); failedTokenRanges += ranges.size(); } ranges.clear(); @@ -394,6 +421,7 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType AutoRepairUtils.removePriorityStatus(repairType, myId); } + logger.info("TEST123"); repairState.setFailedTokenRangesCount(failedTokenRanges); repairState.setSucceededTokenRangesCount(succeededTokenRanges); repairState.setSkippedTokenRangesCount(skippedTokenRanges); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 9522d35bcdc8..e3e7d34658e9 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -48,6 +48,11 @@ public class AutoRepairConfig implements Serializable // This information is useful so the scheduler is absolutely sure that the node is indeed removed from the ring, and then it can adjust the repair schedule accordingly. // So, the duration in this config determinses for how long deleted host's information is kept in the scheduler's metadata. public volatile DurationSpec.IntSecondsBound history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound("2h"); + // the maximum number of retries for a repair session. + public volatile Integer repair_max_retries = 3; + // the backoff time in seconds for retrying a repair session. + public volatile DurationSpec.LongSecondsBound repair_retry_backoff = new DurationSpec.LongSecondsBound("60s"); + // global_settings overides Options.defaultOptions for all repair types public volatile Options global_settings; @@ -108,6 +113,26 @@ public void setAutoRepairHistoryClearDeleteHostsBufferInterval(String duration) history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound(duration); } + public int getRepairMaxRetries() + { + return repair_max_retries; + } + + public void setRepairMaxRetries(int maxRetries) + { + repair_max_retries = maxRetries; + } + + public DurationSpec.LongSecondsBound getRepairRetryBackoff() + { + return repair_retry_backoff; + } + + public void setRepairRetryBackoff(String interval) + { + repair_retry_backoff = new DurationSpec.LongSecondsBound(interval); + } + public boolean isAutoRepairEnabled(RepairType repairType) { return enabled && applyOverrides(repairType, opt -> opt.enabled); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index 003d5cf6412e..9ef64c9bac43 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -728,7 +728,7 @@ public static void addPriorityHosts(RepairType repairType, Set 0) + if (!hostIds.isEmpty()) { SetSerializer serializer = SetSerializer.getInstance(UUIDSerializer.instance, UTF8Type.instance.comparatorSet); addPriorityHost.execute(QueryState.forInternalCalls(), diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index b07d770fb2de..60073b20d4b9 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -104,6 +104,18 @@ public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration) config.setAutoRepairHistoryClearDeleteHostsBufferInterval(duration); } + @Override + public void setAutoRepairMaxRetriesCount(int retries) + { + config.setRepairMaxRetries(retries); + } + + @Override + public void setAutoRepairRetryBackoff(String interval) + { + config.setRepairRetryBackoff(interval); + } + @Override public void setRepairSSTableCountHigherThreshold(RepairType repairType, int sstableHigherThreshold) { diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index 5dda1b157517..d01c6d61db1d 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -44,6 +44,8 @@ public interface AutoRepairServiceMBean public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration); + public void setAutoRepairMaxRetriesCount(int retries); + public void setAutoRepairRetryBackoff(String interval); public void setRepairSSTableCountHigherThreshold(RepairType repairType, int ssTableHigherThreshold); public void setAutoRepairTableMaxRepairTime(RepairType repairType, String autoRepairTableMaxRepairTime); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 331916f26431..a32a6b323047 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2190,6 +2190,16 @@ public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration) autoRepairProxy.setAutoRepairHistoryClearDeleteHostsBufferDuration(duration); } + public void setAutoRepairMaxRetriesCount(int retries) + { + autoRepairProxy.setAutoRepairMaxRetriesCount(retries); + } + + public void setAutoRepairRetryBackoff(String interval) + { + autoRepairProxy.setAutoRepairRetryBackoff(interval); + } + public void setRepairSSTableCountHigherThreshold(AutoRepairConfig.RepairType repairType, int ssTableHigherThreshold) { autoRepairProxy.setRepairSSTableCountHigherThreshold(repairType, ssTableHigherThreshold); diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java index feaf4c98ae17..cfb005542efc 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java @@ -47,6 +47,8 @@ public void execute(NodeProbe probe) sb.append("repair scheduler configuration:"); sb.append("\n\trepair eligibility check interval: " + config.getRepairCheckInterval()); sb.append("\n\tTTL for repair history for dead nodes: " + config.getAutoRepairHistoryClearDeleteHostsBufferInterval()); + sb.append("\n\tmax retries for repair: " + config.getRepairMaxRetries()); + sb.append("\n\tretry backoff: " + config.getRepairRetryBackoff()); for (RepairType repairType : RepairType.values()) { sb.append(formatRepairTypeConfig(probe, repairType, config)); diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index bd00809dcf22..bf2d290ca54d 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -44,7 +44,8 @@ public class SetAutoRepairConfig extends NodeToolCmd description = "autorepair param and value.\nPossible autorepair parameters are as following: " + "[number_of_repair_threads|number_of_subranges|min_repair_interval|sstable_upper_threshold" + "|enabled|table_max_repair_time|priority_hosts|forcerepair_hosts|ignore_dcs" + - "|history_clear_delete_hosts_buffer_interval|repair_primary_token_range_only|parallel_repair_count_in_group|parallel_repair_percentage_in_group|mv_repair_enabled]", + "|history_clear_delete_hosts_buffer_interval|repair_primary_token_range_only" + + "|parallel_repair_count|parallel_repair_percentage|mv_repair_enabled|repair_max_retries|repair_retry_backoff]", required = true) protected List args = new ArrayList<>(); @@ -58,7 +59,6 @@ public class SetAutoRepairConfig extends NodeToolCmd @Override public void execute(NodeProbe probe) { - checkArgument(repairType != null, "--repair-type is required."); checkArgument(args.size() == 2, "setautorepairconfig requires param-type, and value args."); String paramType = args.get(0); String paramVal = args.get(1); @@ -69,10 +69,21 @@ public void execute(NodeProbe probe) return; } - if (paramType.equals("history_clear_delete_hosts_buffer_interval")) + // options that do not require --repair-type option + switch (paramType) { - probe.setAutoRepairHistoryClearDeleteHostsBufferDuration(paramVal); - return; + case "history_clear_delete_hosts_buffer_interval": + probe.setAutoRepairHistoryClearDeleteHostsBufferDuration(paramVal); + return; + case "repair_max_retries": + probe.setAutoRepairMaxRetriesCount(Integer.parseInt(paramVal)); + return; + case "repair_retry_backoff": + probe.setAutoRepairRetryBackoff(paramVal); + return; + default: + // proceed to options that require --repair-type option + break; } // options below require --repair-type option @@ -99,14 +110,14 @@ public void execute(NodeProbe probe) probe.setAutoRepairTableMaxRepairTime(repairType, paramVal); break; case "priority_hosts": - hosts = validateLocalGroupHosts(probe, repairType, paramVal); + hosts = validateLocalGroupHosts(paramVal); if (!hosts.isEmpty()) { probe.setRepairPriorityForHosts(repairType, hosts); } break; case "forcerepair_hosts": - hosts = validateLocalGroupHosts(probe, repairType, paramVal); + hosts = validateLocalGroupHosts(paramVal); if (!hosts.isEmpty()) { probe.setForceRepairForHosts(repairType, hosts); @@ -123,10 +134,10 @@ public void execute(NodeProbe probe) case "repair_primary_token_range_only": probe.setPrimaryTokenRangeOnly(repairType, Boolean.parseBoolean(paramVal)); break; - case "parallel_repair_count_in_group": + case "parallel_repair_count": probe.setParallelRepairCountInGroup(repairType, Integer.parseInt(paramVal)); break; - case "parallel_repair_percentage_in_group": + case "parallel_repair_percentage": probe.setParallelRepairPercentageInGroup(repairType, Integer.parseInt(paramVal)); break; case "mv_repair_enabled": @@ -137,7 +148,7 @@ public void execute(NodeProbe probe) } } - private Set validateLocalGroupHosts(NodeProbe probe, RepairType repairType, String paramVal) { + private Set validateLocalGroupHosts(String paramVal) { Set hosts = new HashSet<>(); for (String host : Splitter.on(',').split(paramVal)) { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 1ddc41ff791e..f770b2105188 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import com.google.common.collect.ImmutableMap; @@ -619,4 +620,60 @@ public void testRepairTakesLastRepairTimeFromDB() // repair scheduler should load the repair time from the DB assertEquals(lastRepairTime, AutoRepair.instance.repairStates.get(repairType).getLastRepairTime()); } + + @Test + public void testRepairMaxRetries() + { + when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())).thenReturn(repairRunnable); + when(autoRepairState.isSuccess()).thenReturn(false); + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + AtomicInteger sleepCalls = new AtomicInteger(); + AutoRepair.sleepFunc = (Long duration, TimeUnit unit) -> { + sleepCalls.getAndIncrement(); + assertEquals(TimeUnit.SECONDS, unit); + assertEquals(config.getRepairRetryBackoff().toSeconds(), (long) duration); + }; + config.setRepairMinInterval(repairType, "0s"); + AutoRepair.instance.repairStates.put(repairType, autoRepairState); + + AutoRepair.instance.repair(repairType, 0); + + //system_auth.role_permissions,system_auth.network_permissions,system_auth.role_members,system_auth.roles, + // system_auth.resource_role_permissons_index,system_traces.sessions,system_traces.events,ks.tbl, + // system_distributed.auto_repair_priority,system_distributed.repair_history,system_distributed.auto_repair_history, + // system_distributed.view_build_status,system_distributed.parent_repair_history,system_distributed.partition_denylist + int exptedTablesGoingThroughRepair = 14; + assertEquals(config.getRepairMaxRetries()*exptedTablesGoingThroughRepair, sleepCalls.get()); + verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(0); + verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); + verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(exptedTablesGoingThroughRepair); + } + + @Test + public void testRepairSuccessAfterRetry() + { + when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())).thenReturn(repairRunnable); + + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + AtomicInteger sleepCalls = new AtomicInteger(); + AutoRepair.sleepFunc = (Long duration, TimeUnit unit) -> { + sleepCalls.getAndIncrement(); + assertEquals(TimeUnit.SECONDS, unit); + assertEquals(config.getRepairRetryBackoff().toSeconds(), (long) duration); + }; + when(autoRepairState.isSuccess()).then((invocationOnMock) -> { + if (sleepCalls.get() == 0) { + return false; + } + return true; + }); + config.setRepairMinInterval(repairType, "0s"); + AutoRepair.instance.repairStates.put(repairType, autoRepairState); + AutoRepair.instance.repair(repairType, 0); + + assertEquals(1, sleepCalls.get()); + verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(14); + verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); + verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); + } } diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java new file mode 100644 index 000000000000..0e8997d9d6c6 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Set; +import java.util.UUID; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.collect.ImmutableSet; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Suite; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.repair.autorepair.AutoRepairKeyspace; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils; +import org.apache.cassandra.schema.SchemaConstants; + +import static org.apache.cassandra.Util.setAutoRepairEnabled; +import static org.junit.Assert.assertEquals; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ AutoRepairServiceTest.BasicTests.class, AutoRepairServiceTest.SetterTests.class }) +public class AutoRepairServiceTest +{ + public static class BasicTests + { + private static AutoRepairService autoRepairService; + private static AutoRepairConfig config; + + @Before + public void setUp() + { + config = new AutoRepairConfig(); + autoRepairService = new AutoRepairService(); + autoRepairService.config = config; + } + + + @Test + public void testSetup() + { + AutoRepairService.instance.config = null; + + AutoRepairService.setup(); + + assertEquals(DatabaseDescriptor.getAutoRepairConfig(), AutoRepairService.instance.config); + } + + @Test + public void testGetAutoRepairConfigReturnsConfig() + { + assertEquals(config, autoRepairService.getAutoRepairConfig()); + } + + @Test + public void testsetAutoRepairHistoryClearDeleteHostsBufferInSecV2() + { + autoRepairService.setAutoRepairHistoryClearDeleteHostsBufferDuration("100s"); + + assertEquals(100, config.getAutoRepairHistoryClearDeleteHostsBufferInterval().toSeconds()); + } + + + @Test + public void testsetAutoRepairMaxRetriesCount() + { + autoRepairService.setAutoRepairMaxRetriesCount(101); + + assertEquals(101, config.getRepairMaxRetries()); + } + + + @Test + public void testsetAutoRepairRetryBackoffInSec() + { + autoRepairService.setAutoRepairRetryBackoff("102s"); + + assertEquals(102, config.getRepairRetryBackoff().toSeconds()); + } + } + + @RunWith(Parameterized.class) + public static class SetterTests extends CQLTester + { + private static final AutoRepairConfig config = new AutoRepairConfig(true); + + @Parameterized.Parameter + public AutoRepairConfig.RepairType repairType; + + @Parameterized.Parameter(1) + public T arg; + + @Parameterized.Parameter(2) + public BiConsumer setter; + + @Parameterized.Parameter(3) + public Function getter; + + @Parameterized.Parameters(name = "{index}: repairType={0}, arg={1}") + public static Collection testCases() + { + return Stream.of( + forEachRepairType(true, AutoRepairService.instance::setAutoRepairEnabled, config::isAutoRepairEnabled), + forEachRepairType(100, AutoRepairService.instance::setRepairThreads, config::getRepairThreads), + forEachRepairType(200, AutoRepairService.instance::setRepairSubRangeNum, config::getRepairSubRangeNum), + forEachRepairType(400, AutoRepairService.instance::setRepairSSTableCountHigherThreshold, config::getRepairSSTableCountHigherThreshold), + forEachRepairType(ImmutableSet.of("dc1", "dc2"), AutoRepairService.instance::setIgnoreDCs, config::getIgnoreDCs), + forEachRepairType(true, AutoRepairService.instance::setPrimaryTokenRangeOnly, config::getRepairPrimaryTokenRangeOnly), + forEachRepairType(600, AutoRepairService.instance::setParallelRepairPercentageInGroup, config::getParallelRepairPercentageInGroup), + forEachRepairType(700, AutoRepairService.instance::setParallelRepairCountInGroup, config::getParallelRepairCountInGroup), + forEachRepairType(true, AutoRepairService.instance::setMVRepairEnabled, config::getMVRepairEnabled), + forEachRepairType(ImmutableSet.of(InetAddressAndPort.getLocalHost()), AutoRepairService.instance::setRepairPriorityForHosts, AutoRepairUtils::getPriorityHosts), + forEachRepairType(ImmutableSet.of(InetAddressAndPort.getLocalHost()), AutoRepairService.instance::setForceRepairForHosts, SetterTests::isLocalHostForceRepair) + ).flatMap(Function.identity()).collect(Collectors.toList()); + } + + private static Set isLocalHostForceRepair(AutoRepairConfig.RepairType type) + { + UUID hostId = Gossiper.instance.getHostId(InetAddressAndPort.getLocalHost()); + UntypedResultSet resultSet = QueryProcessor.executeInternal(String.format( + "SELECT force_repair FROM %s.%s WHERE host_id = %s and repair_type = '%s'", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, hostId, type)); + + if (!resultSet.isEmpty() && resultSet.one().getBoolean("force_repair")) + { + return ImmutableSet.of(InetAddressAndPort.getLocalHost()); + } + return ImmutableSet.of(); + } + + private static Stream forEachRepairType(T arg, BiConsumer setter, Function getter) + { + Object[][] testCases = new Object[AutoRepairConfig.RepairType.values().length][4]; + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + testCases[repairType.ordinal()] = new Object[]{ repairType, arg, setter, getter }; + } + + return Arrays.stream(testCases); + } + + @BeforeClass + public static void setup() throws Exception + { + setAutoRepairEnabled(true); + requireNetwork(); + DatabaseDescriptor.setMaterializedViewsEnabled(false); + DatabaseDescriptor.setCDCEnabled(false); + AutoRepairUtils.setup(); + AutoRepairService.instance.config = config; + } + + @Before + public void prepare() + { + QueryProcessor.executeInternal(String.format( + "TRUNCATE %s.%s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY)); + QueryProcessor.executeInternal(String.format( + "TRUNCATE %s.%s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY)); + } + + @Test + public void testSetters() + { + setter.accept(repairType, arg); + assertEquals(arg, getter.apply(repairType)); + } + } +} diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java new file mode 100644 index 000000000000..2a69cefcace9 --- /dev/null +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools.nodetool; + +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Suite; + +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.tools.NodeProbe; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.junit.Assert.fail; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ SetAutoRepairConfigTest.NoParamTests.class, SetAutoRepairConfigTest.RepairTypeParamTests.class, + SetAutoRepairConfigTest.RepairTypeAndArgsParamsTests.class }) +public class SetAutoRepairConfigTest +{ + protected static AutoRepairConfig config; + + protected static SetAutoRepairConfig cmd; + + public static void before(NodeProbe probeMock, PrintStream outMock) + { + config = new AutoRepairConfig(true); + when(probeMock.getAutoRepairConfig()).thenReturn(config); + cmd = new SetAutoRepairConfig(); + cmd.out = outMock; + } + + public static class NoParamTests + { + @Mock + private static NodeProbe probe; + + @Mock + private static PrintStream out; + + @Before + public void setUp() + { + MockitoAnnotations.initMocks(this); + before(probe, out); + } + + @Test + public void testHistoryDeleteHostsClearBufferInSec() + { + cmd.args = ImmutableList.of("history_clear_delete_hosts_buffer_interval", "1s"); + + cmd.execute(probe); + + verify(probe, times(1)).setAutoRepairHistoryClearDeleteHostsBufferDuration("1s"); + + // test scenario when auto repair is disabled + when(probe.getAutoRepairConfig()).thenReturn(new AutoRepairConfig(false)); + + cmd.execute(probe); + + // test new calls are not made when auto repair is disabled + verify(probe, times(1)).setAutoRepairHistoryClearDeleteHostsBufferDuration("1s"); + } + + @Test + public void testRepairMaxRetries() + { + cmd.args = ImmutableList.of("repair_max_retries", "2"); + + cmd.execute(probe); + + verify(probe, times(1)).setAutoRepairMaxRetriesCount(2); + } + + + @Test + public void testRetryBackoffInSec() + { + cmd.args = ImmutableList.of("repair_retry_backoff", "3s"); + + cmd.execute(probe); + + verify(probe, times(1)).setAutoRepairRetryBackoff("3s"); + } + } + + @RunWith(Parameterized.class) + public static class RepairTypeParamTests + { + @Mock + private static NodeProbe probe; + + @Mock + private static PrintStream out; + + @Parameterized.Parameter + public AutoRepairConfig.RepairType repairType; + + @Parameterized.Parameters(name = "repairType={0}") + public static Object[] data() + { + return AutoRepairConfig.RepairType.values(); + } + + private static InetAddressAndPort localEndpoint; + private static InetAddressAndPort otherEndpoint; + + @Before + public void setUp() throws Exception + { + MockitoAnnotations.initMocks(this); + before(probe, out); + localEndpoint = InetAddressAndPort.getByName("127.0.0.1:7000"); + otherEndpoint = localEndpoint.withPort(localEndpoint.getPort() + 1); + } + + @Test(expected = IllegalArgumentException.class) + public void testNoArgs() + { + cmd.repairType = repairType; + cmd.execute(probe); + } + + @Test + public void testRepairSchedulingDisabled() + { + when(probe.getAutoRepairConfig()).thenReturn(new AutoRepairConfig(false)); + cmd.repairType = repairType; + cmd.args = ImmutableList.of("threads", "1"); + + cmd.execute(probe); + + verify(out, times(1)).println("Auto-repair is not enabled"); + verify(probe, times(0)).setRepairThreads(repairType, 1); + } + + @Test + public void testRepairTypeDisabled() + { + config.setAutoRepairEnabled(repairType, false); + cmd.repairType = repairType; + cmd.args = ImmutableList.of("number_of_repair_threads", "1"); + + cmd.execute(probe); + + verify(probe, times(1)).setRepairThreads(repairType, 1); + } + + + @Test + public void testV2FlagMissing() + { + cmd.repairType = repairType; + cmd.args = ImmutableList.of("threads", "1"); + + try + { + cmd.execute(probe); + + fail("expected IllegalArgumentException"); + } + catch (IllegalArgumentException e) + { + // expected + } + + verify(probe, times(0)).setRepairThreads(repairType, 0); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidParamType() + { + cmd.repairType = repairType; + cmd.args = ImmutableList.of("unknown_type", "1"); + + cmd.execute(probe); + } + + + @Test + public void testPriorityHosts() + { + cmd.repairType = repairType; + cmd.args = ImmutableList.of("priority_hosts", String.join(",", localEndpoint.toString().substring(1), otherEndpoint.toString().substring(1))); + + cmd.execute(probe); + + verify(probe, times(1)).setRepairPriorityForHosts(repairType, ImmutableSet.of(localEndpoint, otherEndpoint)); + } + + @Test + public void testForceRepairHosts() + { + cmd.repairType = repairType; + cmd.args = ImmutableList.of("forcerepair_hosts", String.join(",", localEndpoint.toString().substring(1), otherEndpoint.toString().substring(1))); + + cmd.execute(probe); + + verify(probe, times(1)).setForceRepairForHosts(repairType, ImmutableSet.of(localEndpoint, otherEndpoint)); + } + } + + @RunWith(Parameterized.class) + public static class RepairTypeAndArgsParamsTests + { + @Parameterized.Parameter + public AutoRepairConfig.RepairType repairType; + + @Parameterized.Parameter(1) + public String paramType; + + @Parameterized.Parameter(2) + public String paramVal; + + @Parameterized.Parameter(3) + public Consumer verifyFunc; + + @Parameterized.Parameters(name = "repairType={0},paramType={1}") + public static Collection testCases() + { + return Stream.of( + forEachRepairType("enabled", "true", (type) -> verify(probe, times(1)).setAutoRepairEnabled(type, true)), + forEachRepairType("number_of_repair_threads", "1", (type) -> verify(probe, times(1)).setRepairThreads(type, 1)), + forEachRepairType("number_of_subranges", "2", (type) -> verify(probe, times(1)).setRepairSubRangeNum(type, 2)), + forEachRepairType("min_repair_interval", "3h", (type) -> verify(probe, times(1)).setRepairMinInterval(type, "3h")), + forEachRepairType("sstable_upper_threshold", "4", (type) -> verify(probe, times(1)).setRepairSSTableCountHigherThreshold(type, 4)), + forEachRepairType("table_max_repair_time", "5s", (type) -> verify(probe, times(1)).setAutoRepairTableMaxRepairTime(type, "5s")), + forEachRepairType("repair_primary_token_range_only", "true", (type) -> verify(probe, times(1)).setPrimaryTokenRangeOnly(type, true)), + forEachRepairType("parallel_repair_count", "6", (type) -> verify(probe, times(1)).setParallelRepairCountInGroup(type, 6)), + forEachRepairType("parallel_repair_percentage", "7", (type) -> verify(probe, times(1)).setParallelRepairPercentageInGroup(type, 7)), + forEachRepairType("mv_repair_enabled", "true", (type) -> verify(probe, times(1)).setMVRepairEnabled(type, true)), + forEachRepairType("ignore_dcs", "dc1,dc2", (type) -> verify(probe, times(1)).setAutoRepairIgnoreDCs(type, ImmutableSet.of("dc1", "dc2"))) + ).flatMap(Function.identity()).collect(Collectors.toList()); + } + + private static Stream forEachRepairType(String paramType, String paramVal, Consumer verifyFunc) + { + Object[][] testCases = new Object[AutoRepairConfig.RepairType.values().length][4]; + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + testCases[repairType.ordinal()] = new Object[]{ repairType, paramType, paramVal, verifyFunc }; + } + + return Arrays.stream(testCases); + } + + @Mock + private static NodeProbe probe; + + @Mock + private static PrintStream out; + + @Before + public void setUp() + { + MockitoAnnotations.initMocks(this); + before(probe, out); + } + + @Test + public void test() + { + cmd.repairType = repairType; + cmd.args = ImmutableList.of(paramType, paramVal); + + cmd.execute(probe); + + verifyFunc.accept(repairType); + + // test scenario when auto repair is disabled + when(probe.getAutoRepairConfig()).thenReturn(new AutoRepairConfig(false)); + + cmd.execute(probe); + + // test new calls are not made when auto repair is disabled + verifyFunc.accept(repairType); + } + } +} From 429bda85d7a5c8dbe44bfddf7074fc2620436448 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 30 Aug 2024 09:35:48 -0700 Subject: [PATCH 014/199] Extend nodetool sstablerepairedset to affect all keyspaces when no keyspace is provided in args --- .../repair/autorepair/AutoRepair.java | 1 - .../tools/nodetool/SSTableRepairedSet.java | 72 +++++++++---------- .../nodetool/SSTableRepairedSetTest.java | 55 +++++++------- 3 files changed, 61 insertions(+), 67 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 9aa8534be407..c4994be50e70 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -421,7 +421,6 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType AutoRepairUtils.removePriorityStatus(repairType, myId); } - logger.info("TEST123"); repairState.setFailedTokenRangesCount(failedTokenRanges); repairState.setSucceededTokenRangesCount(succeededTokenRanges); repairState.setSkippedTokenRangesCount(skippedTokenRanges); diff --git a/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java b/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java index 32ee09f93590..f22f3ca0df40 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java @@ -33,7 +33,7 @@ @Command(name = "sstablerepairedset", description = "Set the repaired state of SSTables for given keyspace/tables") public class SSTableRepairedSet extends NodeTool.NodeToolCmd { - @Arguments(usage = " []", description = "The keyspace optionally followed by one or more tables", required = true) + @Arguments(usage = "[ ]", description = "Optional keyspace followed by zero or more tables") protected List args = new ArrayList<>(); @Option(title = "really-set", @@ -56,57 +56,49 @@ public void execute(NodeProbe probe) { PrintStream out = probe.output().out; + if (isRepaired == isUnrepaired) + { + out.println("Exactly one of --is-repaired or --is-unrepaired must be provided."); + return; + } + String message; if (reallySet) - { message = "Mutating repaired state of SSTables for"; - } else - { message = "Previewing repaired state mutation of SSTables for"; - } - if (args.isEmpty()) - { - out.println("At least a keyspace name must be provided."); - return; - } - String keyspace = args.get(0); + List keyspaces = parseOptionalKeyspace(args, probe, KeyspaceSet.NON_LOCAL_STRATEGY); + List tables = List.of(parseOptionalTables(args)); - List tables; - if (args.size() > 1) - { - tables = args.subList(1, args.size()); - message += " tables " + String.join(", ", tables) + " in"; - } + if (args.isEmpty()) + message += " all keyspaces"; else - { - tables = probe.getTablesForKeyspace(keyspace); - message += " all tables in"; - } - message += " keyspace " + keyspace; - - if (isRepaired == isUnrepaired) - { - out.println("Exactly one of --is-repaired or --is-unrepaired must be provided."); - return; - } + message += tables.isEmpty() ? " all tables" : " tables " + String.join(", ", tables) + + " in keyspace " + keyspaces.get(0); message += " to " + (isRepaired ? "repaired" : "unrepaired"); out.println(message); - try + List sstableList = new ArrayList<>(); + for (String keyspace : keyspaces) { - List mutatedSSTables = probe.mutateSSTableRepairedState(isRepaired, !reallySet, keyspace, new ArrayList<>(tables)); - if (!reallySet) - out.println("The following SSTables would be mutated:"); - else - out.println("The following SSTables were mutated:"); - for (String sstable : mutatedSSTables) - out.println(sstable); - } - catch (InvalidRequestException e) - { - out.println(e.getMessage()); + try + { + sstableList.addAll(probe.mutateSSTableRepairedState(isRepaired, !reallySet, keyspace, + tables.isEmpty() + ? probe.getTablesForKeyspace(keyspace) // mutate all tables + : tables)); // mutate specific tables + } + catch (InvalidRequestException e) + { + out.println(e.getMessage()); + } } + if (!reallySet) + out.println("The following SSTables would be mutated:"); + else + out.println("The following SSTables were mutated:"); + for (String sstable : sstableList) + out.println(sstable); } } diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java index 0a2ead81b778..139cde3d839d 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java @@ -22,6 +22,7 @@ import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import org.junit.Before; import org.junit.Test; @@ -35,6 +36,7 @@ import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -46,56 +48,57 @@ public class SSTableRepairedSetTest private SSTableRepairedSet cmd; @Before - public void setUp() - { + public void setUp() { MockitoAnnotations.initMocks(this); - PrintStream noopStream = new PrintStream(new OutputStream() - { - @Override - public void write(int b) - { - } - }); + PrintStream noopStream = new PrintStream(new OutputStream() {@Override public void write(int b) {}}); when(probe.output()).thenReturn(new Output(noopStream, noopStream)); cmd = new SSTableRepairedSet(); } @Test - public void testNoKeyspace() - { + public void testNoKeyspace() { + when(probe.getNonLocalStrategyKeyspaces()).thenReturn(new ArrayList<>(List.of("ks1", "ks2"))); + when(probe.getKeyspaces()).thenReturn(new ArrayList<>(List.of("ks1", "ks2"))); + when(probe.getTablesForKeyspace("ks1")).thenReturn(new ArrayList<>(List.of("table1", "table2"))); + when(probe.getTablesForKeyspace("ks2")).thenReturn(new ArrayList<>(List.of("table3", "table4"))); + cmd.isRepaired = true; + cmd.reallySet = true; + cmd.execute(probe); - verify(probe, never()).mutateSSTableRepairedState(anyBoolean(), anyBoolean(), anyString(), anyList()); + + verify(probe, times(1)).mutateSSTableRepairedState(true, false, "ks1", List.of("table1", "table2")); + verify(probe, times(1)).mutateSSTableRepairedState(true, false, "ks2", List.of("table3", "table4")); } @Test - public void testBothRepairedAndUnrepaired() - { - cmd.args = Arrays.asList("keyspace"); + public void testBothRepairedAndUnrepaired() { + cmd.args = List.of("keyspace"); + cmd.isRepaired = true; + cmd.isUnrepaired = true; cmd.execute(probe); verify(probe, never()).mutateSSTableRepairedState(anyBoolean(), anyBoolean(), anyString(), anyList()); } @Test - public void testNeitherRepairedNorUnrepaired() - { - cmd.args = Arrays.asList("keyspace"); + public void testNeitherRepairedNorUnrepaired() { + cmd.args = List.of("keyspace"); cmd.execute(probe); verify(probe, never()).mutateSSTableRepairedState(anyBoolean(), anyBoolean(), anyString(), anyList()); } @Test - public void testRepairedPreview() - { - cmd.args = Arrays.asList("keyspace"); + public void testRepairedPreview() { + cmd.args = List.of("keyspace"); + when(probe.getKeyspaces()).thenReturn(new ArrayList<>(List.of("keyspace"))); cmd.isRepaired = true; cmd.execute(probe); verify(probe).mutateSSTableRepairedState(true, true, "keyspace", new ArrayList<>()); } @Test - public void testUnrepairedReallySet() - { - cmd.args = Arrays.asList("keyspace"); + public void testUnrepairedReallySet() { + cmd.args = List.of("keyspace"); + when(probe.getKeyspaces()).thenReturn(new ArrayList<>(List.of("keyspace"))); cmd.isUnrepaired = true; cmd.reallySet = true; cmd.execute(probe); @@ -103,9 +106,9 @@ public void testUnrepairedReallySet() } @Test - public void testExecuteWithTableNames() - { + public void testExecuteWithTableNames() { cmd.args = Arrays.asList("keyspace", "table1", "table2"); + when(probe.getKeyspaces()).thenReturn(new ArrayList<>(List.of("keyspace"))); cmd.isRepaired = true; cmd.reallySet = true; cmd.execute(probe); From f9102289c372ad813cfb20b37aa9ffcc5285babc Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 30 Aug 2024 15:16:12 -0700 Subject: [PATCH 015/199] Make repair session timeout configurable --- .../repair/autorepair/AutoRepair.java | 2 +- .../repair/autorepair/AutoRepairConfig.java | 21 +++++++++++++++---- .../repair/autorepair/AutoRepairState.java | 5 +++-- .../cassandra/service/AutoRepairService.java | 5 +++++ .../service/AutoRepairServiceMBean.java | 2 ++ .../org/apache/cassandra/tools/NodeProbe.java | 5 +++++ .../tools/nodetool/GetAutoRepairConfig.java | 1 + .../tools/nodetool/SetAutoRepairConfig.java | 9 ++++++-- .../autorepair/AutoRepairConfigTest.java | 8 +++++++ .../AutoRepairParameterizedTest.java | 4 +++- .../autorepair/AutoRepairStateTest.java | 5 +++-- 11 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index c4994be50e70..6f9b6c4a751f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -292,7 +292,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) f = repairRunnableExecutors.get(repairType).submit(task); try { - repairState.waitForRepairToComplete(); + repairState.waitForRepairToComplete(config.getRepairSessionTimeout(repairType)); } catch (InterruptedException e) { diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index e3e7d34658e9..6e187255df77 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -296,6 +296,17 @@ public DurationSpec.IntSecondsBound getInitialSchedulerDelay(RepairType repairTy return applyOverrides(repairType, opt -> opt.initial_scheduler_delay); } + public DurationSpec.IntSecondsBound getRepairSessionTimeout(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.repair_session_timeout); + } + + public void setRepairSessionTimeout(RepairType repairType, String repairSessionTimeout) + { + ensureOverrides(repairType); + repair_type_overrides.get(repairType).repair_session_timeout = new DurationSpec.IntSecondsBound(repairSessionTimeout); + } + // Options configures auto-repair behavior for a given repair type. // All fields can be modified dynamically. public static class Options implements Serializable @@ -331,6 +342,7 @@ protected static Options getDefaultOptions() opts.mv_repair_enabled = true; opts.token_range_splitter = DefaultAutoRepairTokenSplitter.class.getName(); opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("15m"); // 15 minutes + opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h"); // 3 hours return opts; } @@ -398,6 +410,9 @@ protected static Options getDefaultOptions() public volatile String token_range_splitter; // the minimum delay after a node starts before the scheduler starts running repair public volatile DurationSpec.IntSecondsBound initial_scheduler_delay; + // repair session timeout - this is applicable for each repair session + // the major issue with Repair is a session sometimes hangs; so this timeout is useful to unblock such problems + public volatile DurationSpec.IntSecondsBound repair_session_timeout; public String toString() { @@ -417,6 +432,7 @@ public String toString() ", mv_repair_enabled=" + mv_repair_enabled + ", token_range_splitter=" + token_range_splitter + ", intial_scheduler_delay=" + initial_scheduler_delay + + ", repair_session_timeout=" + repair_session_timeout + '}'; } } @@ -446,9 +462,6 @@ protected void ensureOverrides(RepairType repairType) repair_type_overrides = new EnumMap<>(RepairType.class); } - if (repair_type_overrides.get(repairType) == null) - { - repair_type_overrides.put(repairType, new Options()); - } + repair_type_overrides.computeIfAbsent(repairType, k -> new Options()); } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index d4ab1cf34266..a4869c77425b 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; +import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.view.TableViews; @@ -135,10 +136,10 @@ public void progress(String tag, ProgressEvent event) } } - public void waitForRepairToComplete() throws InterruptedException + public void waitForRepairToComplete(DurationSpec.IntSecondsBound repairSessionTimeout) throws InterruptedException { //if for some reason we don't hear back on repair progress for sometime - if (!condition.await(12, TimeUnit.HOURS)) + if (!condition.await(repairSessionTimeout.toSeconds(), TimeUnit.SECONDS)) { success = false; } diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 60073b20d4b9..cbef9889fb27 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -156,4 +156,9 @@ public void setMVRepairEnabled(RepairType repairType, boolean enabled) { config.setMVRepairEnabled(repairType, enabled); } + + public void setRepairSessionTimeout(RepairType repairType, String timeout) + { + config.setRepairSessionTimeout(repairType, timeout); + } } diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index d01c6d61db1d..4ebf118e023e 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -59,4 +59,6 @@ public interface AutoRepairServiceMBean public void setMVRepairEnabled(RepairType repairType, boolean enabled); public AutoRepairConfig getAutoRepairConfig(); + + public void setRepairSessionTimeout(RepairType repairType, String timeout); } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index a32a6b323047..ebe00e90ab28 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2241,6 +2241,11 @@ public List mutateSSTableRepairedState(boolean repair, boolean preview, public List getTablesForKeyspace(String keyspace) { return ssProxy.getTablesForKeyspace(keyspace); } + + public void setRepairSessionTimeout(AutoRepairConfig.RepairType repairType, String timeout) + { + autoRepairProxy.setRepairSessionTimeout(repairType, timeout); + } } class ColumnFamilyStoreMBeanIterator implements Iterator> diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java index cfb005542efc..30dd1ff889c4 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java @@ -74,6 +74,7 @@ private String formatRepairTypeConfig(NodeProbe probe, RepairType repairType, Au sb.append("\n\tpercentage of parallel repairs within group: " + config.getParallelRepairPercentageInGroup(repairType)); sb.append("\n\tmv repair enabled: " + config.getMVRepairEnabled(repairType)); sb.append("\n\tinitial scheduler delay: " + config.getInitialSchedulerDelay(repairType)); + sb.append("\n\trepair setssion timeout: " + config.getRepairSessionTimeout(repairType)); return sb.toString(); } diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index bf2d290ca54d..b467d9b82998 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -19,6 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; + import io.airlift.airline.Arguments; import io.airlift.airline.Command; import io.airlift.airline.Option; @@ -45,7 +46,7 @@ public class SetAutoRepairConfig extends NodeToolCmd "[number_of_repair_threads|number_of_subranges|min_repair_interval|sstable_upper_threshold" + "|enabled|table_max_repair_time|priority_hosts|forcerepair_hosts|ignore_dcs" + "|history_clear_delete_hosts_buffer_interval|repair_primary_token_range_only" + - "|parallel_repair_count|parallel_repair_percentage|mv_repair_enabled|repair_max_retries|repair_retry_backoff]", + "|parallel_repair_count|parallel_repair_percentage|mv_repair_enabled|repair_max_retries|repair_retry_backoff|repair_session_timeout]", required = true) protected List args = new ArrayList<>(); @@ -143,12 +144,16 @@ public void execute(NodeProbe probe) case "mv_repair_enabled": probe.setMVRepairEnabled(repairType, Boolean.parseBoolean(paramVal)); break; + case "repair_session_timeout": + probe.setRepairSessionTimeout(repairType, paramVal); + break; default: throw new IllegalArgumentException("Unknown parameter: " + paramType); } } - private Set validateLocalGroupHosts(String paramVal) { + private Set validateLocalGroupHosts(String paramVal) + { Set hosts = new HashSet<>(); for (String host : Splitter.on(',').split(paramVal)) { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 923e35b73796..6d25dbeb20b4 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -468,4 +468,12 @@ public void testGetInitialSchedulerDelay() assertEquals(5, result); } + @Test + public void testSetRepairSessionTimeout() + { + config.setRepairSessionTimeout(repairType, "1h"); + + assert config.repair_type_overrides.get(repairType).repair_session_timeout.toSeconds() == 3600; + } + } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index f770b2105188..1604055e4b88 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -31,6 +31,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; +import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.cql3.statements.schema.TableAttributes; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -489,6 +490,7 @@ public void testRepairWaitsForRepairToFinishBeforeSchedullingNewSession() throws { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setMVRepairEnabled(repairType, false); + config.setRepairRetryBackoff("0s"); when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())) .thenReturn(repairRunnable); AutoRepair.instance.repairStates.put(repairType, autoRepairState); @@ -506,7 +508,7 @@ public void testRepairWaitsForRepairToFinishBeforeSchedullingNewSession() throws assertEquals("resetWaitCondition was not called before waitForRepairToComplete", resetWaitConditionCalls.get(), waitForRepairCompletedCalls.get()); return null; - }).when(autoRepairState).waitForRepairToComplete(); + }).when(autoRepairState).waitForRepairToComplete(config.getRepairSessionTimeout(repairType)); AutoRepair.instance.repair(repairType, 0); AutoRepair.instance.repair(repairType, 0); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java index 340a21f97bfa..657b11310f64 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -31,6 +31,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; import org.apache.cassandra.repair.autorepair.AutoRepairUtils.AutoRepairHistory; @@ -125,7 +126,7 @@ public void testWaitForRepairToComplete() throws Exception state.condition.signalAll(); Condition finishedCondition = Condition.newOneTimeCondition(); Callable waitForRepairToComplete = () -> { - state.waitForRepairToComplete(); + state.waitForRepairToComplete(new DurationSpec.IntSecondsBound("12h")); finishedCondition.signalAll(); return null; }; @@ -363,7 +364,7 @@ public void testWaitForRepairToCompleteDoesNotSetSuccessWhenProgressReceivesErro state.progress("test", progressEvent); assertFalse(state.success); - state.waitForRepairToComplete(); + state.waitForRepairToComplete(new DurationSpec.IntSecondsBound("12h")); assertFalse(state.success); } From 1034390aab85e7e2aa8ccf78e10249387809e1c9 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 5 Sep 2024 18:31:09 -0700 Subject: [PATCH 016/199] Comments from Joshua M --- .../repair/autorepair/AutoRepairConfig.java | 29 +++++++++---------- .../autorepair/AutoRepairConfigTest.java | 10 +++---- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 6e187255df77..97868a594f25 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -238,24 +238,24 @@ public void setRepairPrimaryTokenRangeOnly(RepairType repairType, boolean primar public int getParallelRepairPercentageInGroup(RepairType repairType) { - return applyOverrides(repairType, opt -> opt.parallel_repair_percentage_in_group); + return applyOverrides(repairType, opt -> opt.parallel_repair_percentage); } public void setParallelRepairPercentageInGroup(RepairType repairType, int percentageInGroup) { ensureOverrides(repairType); - repair_type_overrides.get(repairType).parallel_repair_percentage_in_group = percentageInGroup; + repair_type_overrides.get(repairType).parallel_repair_percentage = percentageInGroup; } public int getParallelRepairCountInGroup(RepairType repairType) { - return applyOverrides(repairType, opt -> opt.parallel_repair_count_in_group); + return applyOverrides(repairType, opt -> opt.parallel_repair_count); } public void setParallelRepairCountInGroup(RepairType repairType, int countInGroup) { ensureOverrides(repairType); - repair_type_overrides.get(repairType).parallel_repair_count_in_group = countInGroup; + repair_type_overrides.get(repairType).parallel_repair_count = countInGroup; } public boolean getMVRepairEnabled(RepairType repairType) @@ -331,15 +331,15 @@ protected static Options getDefaultOptions() opts.repair_by_keyspace = false; opts.number_of_subranges = 16; opts.number_of_repair_threads = 1; - opts.parallel_repair_count_in_group = 1; - opts.parallel_repair_percentage_in_group = 0; + opts.parallel_repair_count = 3; + opts.parallel_repair_percentage = 3; opts.sstable_upper_threshold = 10000; opts.min_repair_interval = new DurationSpec.IntSecondsBound("24h"); opts.ignore_dcs = new HashSet<>(); opts.repair_primary_token_range_only = true; opts.force_repair_new_node = false; opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h"); - opts.mv_repair_enabled = true; + opts.mv_repair_enabled = false; opts.token_range_splitter = DefaultAutoRepairTokenSplitter.class.getName(); opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("15m"); // 15 minutes opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h"); // 3 hours @@ -362,13 +362,12 @@ protected static Options getDefaultOptions() // Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. // This is similar to -j for repair options for the nodetool repair command. public volatile Integer number_of_repair_threads; - // the number of repair sessions that can run in parallel in a single group - // The number of nodes running repair parallelly. If parallelrepaircount is set, it will choose the larger value of the two. The default is 3. + // The number of nodes running repair parallelly. If parallel_repair_count is set, it will choose the larger value of the two. The default is 3. // This configuration controls how many nodes would run repair in parallel. // The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. // If one or more node(s) finish repair, then the framework automatically picks up the next candidate and ensures the maximum number of nodes running repair do not exceed “3”. - public volatile Integer parallel_repair_count_in_group; - // the number of repair sessions that can run in parallel in a single groupas a percentage + public volatile Integer parallel_repair_count; + // the number of repair nodes that can run in parallel // of the total number of nodes in the group [0,100] // The percentage of nodes in the cluster that run repair parallelly. If parallelrepaircount is set, it will choose the larger value of the two. // The problem with a fixed number of nodes (the above property) is that in a large-scale environment, @@ -376,7 +375,7 @@ protected static Options getDefaultOptions() // The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. // So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. // Extremely fewer manual interventions as it will rarely violate the repair SLA for customers - public volatile Integer parallel_repair_percentage_in_group; + public volatile Integer parallel_repair_percentage; // the upper threshold of SSTables allowed to participate in a single repair session // Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. // This is to avoid penalizing good tables (neighbors) with an outlier. @@ -421,8 +420,8 @@ public String toString() ", repair_by_keyspace=" + repair_by_keyspace + ", number_of_subranges=" + number_of_subranges + ", number_of_repair_threads=" + number_of_repair_threads + - ", parallel_repair_count_in_group=" + parallel_repair_count_in_group + - ", parallel_repair_percentage_in_group=" + parallel_repair_percentage_in_group + + ", parallel_repair_count_in_group=" + parallel_repair_count + + ", parallel_repair_percentage_in_group=" + parallel_repair_percentage + ", sstable_upper_threshold=" + sstable_upper_threshold + ", min_repair_interval=" + min_repair_interval + ", ignore_dcs=" + ignore_dcs + @@ -432,7 +431,7 @@ public String toString() ", mv_repair_enabled=" + mv_repair_enabled + ", token_range_splitter=" + token_range_splitter + ", intial_scheduler_delay=" + initial_scheduler_delay + - ", repair_session_timeout=" + repair_session_timeout + + ", repair_session_timeout=" + repair_session_timeout + '}'; } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 6d25dbeb20b4..07eb9070fe85 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -340,7 +340,7 @@ public void testSetRepairPrimaryTokenRangeOnly() @Test public void testGetParallelRepairPercentageInGroup() { - config.global_settings.parallel_repair_percentage_in_group = 5; + config.global_settings.parallel_repair_percentage = 5; int result = config.getParallelRepairPercentageInGroup(repairType); @@ -352,13 +352,13 @@ public void testSetParallelRepairPercentageInGroup() { config.setParallelRepairPercentageInGroup(repairType, 5); - assert config.repair_type_overrides.get(repairType).parallel_repair_percentage_in_group == 5; + assert config.repair_type_overrides.get(repairType).parallel_repair_percentage == 5; } @Test public void testGetParallelRepairCountInGroup() { - config.global_settings.parallel_repair_count_in_group = 5; + config.global_settings.parallel_repair_count = 5; int result = config.getParallelRepairCountInGroup(repairType); @@ -370,7 +370,7 @@ public void testSetParallelRepairCountInGroup() { config.setParallelRepairCountInGroup(repairType, 5); - assert config.repair_type_overrides.get(repairType).parallel_repair_count_in_group == 5; + assert config.repair_type_overrides.get(repairType).parallel_repair_count == 5; } @Test @@ -432,7 +432,7 @@ public void testGetDefaultOptionsMVRepairIsEnabledByDefault() { Options defaultOptions = Options.getDefaultOptions(); - assertTrue(defaultOptions.mv_repair_enabled); + assertFalse(defaultOptions.mv_repair_enabled); } @Test From d9dfdbc0ec33f4fbb2073f4aba7a80887dcc2507 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sat, 7 Sep 2024 16:32:20 -0700 Subject: [PATCH 017/199] AutoRepair scheduler InJvm dtest --- .../repair/autorepair/AutoRepair.java | 4 +- .../tools/nodetool/SSTableRepairedSet.java | 3 +- .../test/repair/AutoRepairSchedulerTest.java | 128 ++++++++++++++++++ .../config/DatabaseDescriptorRefTest.java | 9 ++ .../config/YamlConfigurationLoaderTest.java | 10 +- .../AutoRepairParameterizedTest.java | 1 - .../nodetool/SSTableRepairedSetTest.java | 27 ++-- 7 files changed, 159 insertions(+), 23 deletions(-) create mode 100644 test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 6f9b6c4a751f..69db9e609afb 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -114,7 +114,7 @@ public void setup() for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) { repairExecutors.get(repairType).scheduleWithFixedDelay( - () -> repair(repairType, 60000), + () -> repair(repairType, 5000), config.getInitialSchedulerDelay(repairType).toSeconds(), config.getRepairCheckInterval().toSeconds(), TimeUnit.SECONDS); @@ -442,7 +442,7 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType if (timeInHours == 0 && millisToWait > 0) { //If repair finished quickly, happens for an empty instance, in such case - //wait for a minute so that the JMX metrics can detect the repairInProgress + //wait for some duration so that the JMX metrics can detect the repairInProgress logger.info("Wait for {} milliseconds for repair type {}.", millisToWait, repairType); Thread.sleep(millisToWait); } diff --git a/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java b/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java index f22f3ca0df40..e1773fe47b7f 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java @@ -20,6 +20,7 @@ import java.io.PrintStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; @@ -69,7 +70,7 @@ public void execute(NodeProbe probe) message = "Previewing repaired state mutation of SSTables for"; List keyspaces = parseOptionalKeyspace(args, probe, KeyspaceSet.NON_LOCAL_STRATEGY); - List tables = List.of(parseOptionalTables(args)); + List tables = new ArrayList<>(Arrays.asList(parseOptionalTables(args))); if (args.isEmpty()) message += " all keyspaces"; diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java new file mode 100644 index 000000000000..49b4300dc50b --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.repair; + +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.concurrent.TimeUnit; + +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.Uninterruptibles; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.repair.autorepair.AutoRepair; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.repair.autorepair.AutoRepairKeyspace; + +import static org.apache.cassandra.schema.SchemaConstants.DISTRIBUTED_KEYSPACE_NAME; +import static org.junit.Assert.assertEquals; + +public class AutoRepairSchedulerTest extends TestBaseImpl +{ + + private static Cluster cluster; + static SimpleDateFormat sdf; + + @BeforeClass + public static void init() throws IOException + { + // Define the expected date format pattern + String pattern = "EEE MMM dd HH:mm:ss z yyyy"; + // Create SimpleDateFormat object with the given pattern + sdf = new SimpleDateFormat(pattern); + sdf.setLenient(false); + cluster = Cluster.build(3).withConfig(config -> config + .set("auto_repair", + ImmutableMap.of( + "repair_type_overrides", + ImmutableMap.of(AutoRepairConfig.RepairType.full.toString(), + ImmutableMap.of( + "initial_scheduler_delay", "5s", + "enabled", "true", + "parallel_repair_count", "1", + "parallel_repair_percentage", "0", + "min_repair_interval", "1s"), + AutoRepairConfig.RepairType.incremental.toString(), + ImmutableMap.of( + "initial_scheduler_delay", "5s", + "enabled", "true", + "parallel_repair_count", "1", + "parallel_repair_percentage", "0", + "min_repair_interval", "1s")))) + .set("auto_repair.enabled", "true") + .set("auto_repair.repair_check_interval", "10s")).start(); + + cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck text, v1 int, v2 int, PRIMARY KEY (pk, ck)) WITH read_repair='NONE'")); + } + + @Test + public void testScheduler() throws ParseException + { + // ensure there was no history of previous repair runs through the scheduler + Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s", DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY), ConsistencyLevel.QUORUM); + assertEquals(0, rows.length); + + cluster.forEach(i -> i.runOnInstance(() -> { + try + { + AutoRepair.instance.setup(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + })); + // wait for a couple of minutes for repair to go through on all three nodes + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MINUTES); + + validate(AutoRepairConfig.RepairType.full.toString()); + validate(AutoRepairConfig.RepairType.incremental.toString()); + } + + private void validate(String repairType) throws ParseException + { + Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s where repair_type='%s'", DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, repairType), ConsistencyLevel.QUORUM); + assertEquals(3, rows.length); + for (int node = 0; node < rows.length; node++) + { + Object[] row = rows[node]; + // repair_type + Assert.assertEquals(repairType, row[0].toString()); + // host_id + Assert.assertEquals(String.format("00000000-0000-4000-8000-%012d", node + 1), row[1].toString()); + // ensure there is a legit repair_start_ts and repair_finish_ts + sdf.parse(row[2].toString()); + sdf.parse(row[3].toString()); + // the reason why the repair was scheduled + Assert.assertEquals("MY_TURN", row[4].toString()); + for (Object col : row) + { + System.out.println("Data:" + col); + } + System.out.println("====================================="); + } + } +} diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java index 073afd619a26..48cadcbc64ca 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java @@ -89,6 +89,14 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.config.Config$RepairCommandPoolFullStrategy", "org.apache.cassandra.config.Config$UserFunctionTimeoutPolicy", "org.apache.cassandra.config.Config$CorruptedTombstoneStrategy", + "org.apache.cassandra.repair.autorepair.AutoRepairConfig", + "org.apache.cassandra.repair.autorepair.AutoRepairConfig$Options", + "org.apache.cassandra.repair.autorepair.DefaultAutoRepairTokenSplitter", + "org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter", + "org.apache.cassandra.repair.autorepair.FullRepairState", + "org.apache.cassandra.repair.autorepair.IncrementalRepairState", + "org.apache.cassandra.repair.autorepair.AutoRepairConfig$RepairType", + "org.apache.cassandra.repair.autorepair.AutoRepairState", "org.apache.cassandra.config.DatabaseDescriptor$ByteUnit", "org.apache.cassandra.config.DataRateSpec", "org.apache.cassandra.config.DataRateSpec$DataRateUnit", @@ -206,6 +214,7 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.utils.FBUtilities$1", "org.apache.cassandra.utils.CloseableIterator", "org.apache.cassandra.utils.Pair", + "org.apache.cassandra.utils.progress.ProgressListener", "org.apache.cassandra.utils.concurrent.UncheckedInterruptedException", "org.apache.cassandra.ConsoleAppender", "org.apache.cassandra.ConsoleAppender$1", diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java index 13b0a1d1574a..3de7dd458009 100644 --- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java +++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java @@ -229,11 +229,11 @@ public void fromMapTest() "optional", false, "enabled", true); Map autoRepairConfig = ImmutableMap.of("enabled", true, - "global_settings", ImmutableMap.of("repair_dc_groups", - ImmutableSet.of("all the groups")), + "global_settings", ImmutableMap.of("number_of_repair_threads", + 1), "repair_type_overrides", ImmutableMap.of( - "full", ImmutableMap.of("repair_dc_groups", - ImmutableSet.of("none of the groups")))); + "full", ImmutableMap.of("number_of_repair_threads", + 2))); Map map = new ImmutableMap.Builder() .put("storage_port", storagePort) .put("commitlog_sync", commitLogSync) @@ -254,7 +254,7 @@ public void fromMapTest() assertEquals(new DataStorageSpec.IntBytesBound("5B"), config.internode_socket_send_buffer_size); // Check names backward compatibility (CASSANDRA-17141 and CASSANDRA-15234) assertEquals(new DataStorageSpec.IntBytesBound("5B"), config.internode_socket_receive_buffer_size); // Check names backward compatibility (CASSANDRA-17141 and CASSANDRA-15234) assertEquals(true, config.auto_repair.enabled); - assertEquals(6 * 60 * 60L, config.auto_repair.getAutoRepairTableMaxRepairTime(AutoRepairConfig.RepairType.incremental)); + assertEquals(new DurationSpec.IntSecondsBound("6h"), config.auto_repair.getAutoRepairTableMaxRepairTime(AutoRepairConfig.RepairType.incremental)); config.auto_repair.setMVRepairEnabled(AutoRepairConfig.RepairType.incremental, false); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 1604055e4b88..015e9d61ea6b 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -31,7 +31,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; -import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.cql3.statements.schema.TableAttributes; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java index 139cde3d839d..bc168c0752d7 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java @@ -22,7 +22,6 @@ import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import org.junit.Before; import org.junit.Test; @@ -57,22 +56,22 @@ public void setUp() { @Test public void testNoKeyspace() { - when(probe.getNonLocalStrategyKeyspaces()).thenReturn(new ArrayList<>(List.of("ks1", "ks2"))); - when(probe.getKeyspaces()).thenReturn(new ArrayList<>(List.of("ks1", "ks2"))); - when(probe.getTablesForKeyspace("ks1")).thenReturn(new ArrayList<>(List.of("table1", "table2"))); - when(probe.getTablesForKeyspace("ks2")).thenReturn(new ArrayList<>(List.of("table3", "table4"))); + when(probe.getNonLocalStrategyKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("ks1", "ks2"))); + when(probe.getKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("ks1", "ks2"))); + when(probe.getTablesForKeyspace("ks1")).thenReturn(new ArrayList<>(Arrays.asList("table1", "table2"))); + when(probe.getTablesForKeyspace("ks2")).thenReturn(new ArrayList<>(Arrays.asList("table3", "table4"))); cmd.isRepaired = true; cmd.reallySet = true; cmd.execute(probe); - verify(probe, times(1)).mutateSSTableRepairedState(true, false, "ks1", List.of("table1", "table2")); - verify(probe, times(1)).mutateSSTableRepairedState(true, false, "ks2", List.of("table3", "table4")); + verify(probe, times(1)).mutateSSTableRepairedState(true, false, "ks1", Arrays.asList("table1", "table2")); + verify(probe, times(1)).mutateSSTableRepairedState(true, false, "ks2", Arrays.asList("table3", "table4")); } @Test public void testBothRepairedAndUnrepaired() { - cmd.args = List.of("keyspace"); + cmd.args = Arrays.asList("keyspace"); cmd.isRepaired = true; cmd.isUnrepaired = true; cmd.execute(probe); @@ -81,15 +80,15 @@ public void testBothRepairedAndUnrepaired() { @Test public void testNeitherRepairedNorUnrepaired() { - cmd.args = List.of("keyspace"); + cmd.args = Arrays.asList("keyspace"); cmd.execute(probe); verify(probe, never()).mutateSSTableRepairedState(anyBoolean(), anyBoolean(), anyString(), anyList()); } @Test public void testRepairedPreview() { - cmd.args = List.of("keyspace"); - when(probe.getKeyspaces()).thenReturn(new ArrayList<>(List.of("keyspace"))); + cmd.args = Arrays.asList("keyspace"); + when(probe.getKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("keyspace"))); cmd.isRepaired = true; cmd.execute(probe); verify(probe).mutateSSTableRepairedState(true, true, "keyspace", new ArrayList<>()); @@ -97,8 +96,8 @@ public void testRepairedPreview() { @Test public void testUnrepairedReallySet() { - cmd.args = List.of("keyspace"); - when(probe.getKeyspaces()).thenReturn(new ArrayList<>(List.of("keyspace"))); + cmd.args = Arrays.asList("keyspace"); + when(probe.getKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("keyspace"))); cmd.isUnrepaired = true; cmd.reallySet = true; cmd.execute(probe); @@ -108,7 +107,7 @@ public void testUnrepairedReallySet() { @Test public void testExecuteWithTableNames() { cmd.args = Arrays.asList("keyspace", "table1", "table2"); - when(probe.getKeyspaces()).thenReturn(new ArrayList<>(List.of("keyspace"))); + when(probe.getKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("keyspace"))); cmd.isRepaired = true; cmd.reallySet = true; cmd.execute(probe); From a5ae28993eb05c2e61d952d0d441869a321b18e6 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 23 Sep 2024 18:19:36 -0700 Subject: [PATCH 018/199] Increase CircleCI parallelism 1-->4 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e48d91b5031..d90de438d53d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,7 +24,7 @@ jobs: resource_class: medium working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 1 + parallelism: 4 steps: - attach_workspace: at: /home/cassandra From eeea77e02afbef3371d4b76d47ad9cb2725277dc Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 24 Sep 2024 14:07:04 -0700 Subject: [PATCH 019/199] Fix a test case failure org.apache.cassandra.schema.SchemaKeyspaceTest::testConversionsInverses --- src/java/org/apache/cassandra/schema/TableParams.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index b8b10c768494..fa9145df8ead 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -241,7 +241,7 @@ public boolean equals(Object o) && extensions.equals(p.extensions) && cdc == p.cdc && readRepair == p.readRepair - && automatedRepair == p.automatedRepair; + && automatedRepair.equals(p.automatedRepair); } @Override From 61ee3d8cee69dee79c192d26b434887ffe740754 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 24 Sep 2024 14:29:08 -0700 Subject: [PATCH 020/199] Trigger automatic CircleCI jobs --- .circleci/config.yml | 85 -------------------------------------------- 1 file changed, 85 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d90de438d53d..ed226a2ab22c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9463,228 +9463,190 @@ workflows: java8_separate_tests: jobs: - start_j8_build: - type: approval - j8_build: requires: - start_j8_build - start_j8_unit_tests: - type: approval - j8_unit_tests: requires: - start_j8_unit_tests - j8_build - start_j8_jvm_dtests: - type: approval - j8_jvm_dtests: requires: - start_j8_jvm_dtests - j8_build - start_j8_jvm_dtests_vnode: - type: approval - j8_jvm_dtests_vnode: requires: - start_j8_jvm_dtests_vnode - j8_build - start_j11_jvm_dtests: - type: approval - j11_jvm_dtests: requires: - start_j11_jvm_dtests - j8_build - start_j11_jvm_dtests_vnode: - type: approval - j11_jvm_dtests_vnode: requires: - start_j11_jvm_dtests_vnode - j8_build - start_j8_simulator_dtests: - type: approval - j8_simulator_dtests: requires: - start_j8_simulator_dtests - j8_build - start_j8_cqlshlib_tests: - type: approval - j8_cqlshlib_tests: requires: - start_j8_cqlshlib_tests - j8_build - start_j8_cqlshlib_cython_tests: - type: approval - j8_cqlshlib_cython_tests: requires: - start_j8_cqlshlib_cython_tests - j8_build - start_j11_cqlshlib_tests: - type: approval - j11_cqlshlib_tests: requires: - start_j11_cqlshlib_tests - j8_build - start_j11_cqlshlib_cython_tests: - type: approval - j11_cqlshlib_cython_tests: requires: - start_j11_cqlshlib_cython_tests - j8_build - start_j11_unit_tests: - type: approval - j11_unit_tests: requires: - start_j11_unit_tests - j8_build - start_j8_utests_long: - type: approval - j8_utests_long: requires: - start_j8_utests_long - j8_build - start_j11_utests_long: - type: approval - j11_utests_long: requires: - start_j11_utests_long - j8_build - start_j8_utests_cdc: - type: approval - j8_utests_cdc: requires: - start_j8_utests_cdc - j8_build - start_j11_utests_cdc: - type: approval - j11_utests_cdc: requires: - start_j11_utests_cdc - j8_build - start_j8_utests_compression: - type: approval - j8_utests_compression: requires: - start_j8_utests_compression - j8_build - start_j11_utests_compression: - type: approval - j11_utests_compression: requires: - start_j11_utests_compression - j8_build - start_j8_utests_stress: - type: approval - j8_utests_stress: requires: - start_j8_utests_stress - j8_build - start_j11_utests_stress: - type: approval - j11_utests_stress: requires: - start_j11_utests_stress - j8_build - start_j8_utests_fqltool: - type: approval - j8_utests_fqltool: requires: - start_j8_utests_fqltool - j8_build - start_j11_utests_fqltool: - type: approval - j11_utests_fqltool: requires: - start_j11_utests_fqltool - j8_build - start_j8_utests_system_keyspace_directory: - type: approval - j8_utests_system_keyspace_directory: requires: - start_j8_utests_system_keyspace_directory - j8_build - start_j11_utests_system_keyspace_directory: - type: approval - j11_utests_system_keyspace_directory: requires: - start_j11_utests_system_keyspace_directory - j8_build - start_j8_dtest_jars_build: - type: approval - j8_dtest_jars_build: requires: - j8_build - start_j8_dtest_jars_build - start_jvm_upgrade_dtests: - type: approval - j8_jvm_upgrade_dtests: requires: - start_jvm_upgrade_dtests - j8_dtest_jars_build - start_j8_dtests: - type: approval - j8_dtests: requires: - start_j8_dtests - j8_build - start_j8_dtests_vnode: - type: approval - j8_dtests_vnode: requires: - start_j8_dtests_vnode - j8_build - start_j8_dtests_offheap: - type: approval - j8_dtests_offheap: requires: - start_j8_dtests_offheap - j8_build - start_j11_dtests: - type: approval - j11_dtests: requires: - start_j11_dtests - j8_build - start_j11_dtests_vnode: - type: approval - j11_dtests_vnode: requires: - start_j11_dtests_vnode - j8_build - start_j11_dtests_offheap: - type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j8_build - start_j8_dtests_large: - type: approval - j8_dtests_large: requires: - start_j8_dtests_large - j8_build - start_j8_dtests_large_vnode: - type: approval - j8_dtests_large_vnode: requires: - start_j8_dtests_large_vnode - j8_build - start_j11_dtests_large: - type: approval - j11_dtests_large: requires: - start_j11_dtests_large - j8_build - start_j11_dtests_large_vnode: - type: approval - j11_dtests_large_vnode: requires: - start_j11_dtests_large_vnode - j8_build - start_upgrade_dtests: - type: approval - j8_upgrade_dtests: requires: - start_upgrade_dtests - j8_build - start_j8_cqlsh_tests: - type: approval - j8_cqlsh_dtests_py3: requires: - start_j8_cqlsh_tests @@ -9710,7 +9672,6 @@ workflows: - start_j8_cqlsh_tests - j8_build - start_j8_cqlsh_tests_offheap: - type: approval - j8_cqlsh_dtests_py3_offheap: requires: - start_j8_cqlsh_tests_offheap @@ -9724,7 +9685,6 @@ workflows: - start_j8_cqlsh_tests_offheap - j8_build - start_j11_cqlsh_tests: - type: approval - j11_cqlsh_dtests_py3: requires: - start_j11_cqlsh_tests @@ -9750,7 +9710,6 @@ workflows: - start_j11_cqlsh_tests - j8_build - start_j11_cqlsh_tests_offheap: - type: approval - j11_cqlsh_dtests_py3_offheap: requires: - start_j11_cqlsh_tests_offheap @@ -9766,7 +9725,6 @@ workflows: java8_pre-commit_tests: jobs: - start_pre-commit_tests: - type: approval - j8_build: requires: - start_pre-commit_tests @@ -9804,7 +9762,6 @@ workflows: requires: - j8_build - start_utests_long: - type: approval - j8_utests_long: requires: - start_utests_long @@ -9814,7 +9771,6 @@ workflows: - start_utests_long - j8_build - start_utests_cdc: - type: approval - j8_utests_cdc: requires: - start_utests_cdc @@ -9824,7 +9780,6 @@ workflows: - start_utests_cdc - j8_build - start_utests_compression: - type: approval - j8_utests_compression: requires: - start_utests_compression @@ -9834,7 +9789,6 @@ workflows: - start_utests_compression - j8_build - start_utests_stress: - type: approval - j8_utests_stress: requires: - start_utests_stress @@ -9844,7 +9798,6 @@ workflows: - start_utests_stress - j8_build - start_utests_fqltool: - type: approval - j8_utests_fqltool: requires: - start_utests_fqltool @@ -9854,7 +9807,6 @@ workflows: - start_utests_fqltool - j8_build - start_utests_system_keyspace_directory: - type: approval - j8_utests_system_keyspace_directory: requires: - j8_build @@ -9863,7 +9815,6 @@ workflows: - start_utests_system_keyspace_directory - j8_build - start_jvm_upgrade_dtests: - type: approval - j8_dtest_jars_build: requires: - j8_build @@ -9878,7 +9829,6 @@ workflows: requires: - j8_build - start_j8_dtests_offheap: - type: approval - j8_dtests_offheap: requires: - start_j8_dtests_offheap @@ -9890,13 +9840,11 @@ workflows: requires: - j8_build - start_j11_dtests_offheap: - type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j8_build - start_j8_dtests_large: - type: approval - j8_dtests_large: requires: - start_j8_dtests_large @@ -9906,7 +9854,6 @@ workflows: - start_j8_dtests_large - j8_build - start_j11_dtests_large: - type: approval - j11_dtests_large: requires: - start_j11_dtests_large @@ -9916,7 +9863,6 @@ workflows: - start_j11_dtests_large - j8_build - start_upgrade_dtests: - type: approval - j8_upgrade_dtests: requires: - j8_build @@ -9940,7 +9886,6 @@ workflows: requires: - j8_build - start_j8_cqlsh_dtests_offheap: - type: approval - j8_cqlsh_dtests_py3_offheap: requires: - start_j8_cqlsh_dtests_offheap @@ -9972,7 +9917,6 @@ workflows: requires: - j8_build - start_j11_cqlsh-dtests-offheap: - type: approval - j11_cqlsh_dtests_py3_offheap: requires: - start_j11_cqlsh-dtests-offheap @@ -9988,72 +9932,60 @@ workflows: java11_separate_tests: jobs: - start_j11_build: - type: approval - j11_build: requires: - start_j11_build - start_j11_unit_tests: - type: approval - j11_unit_tests: requires: - start_j11_unit_tests - j11_build - start_j11_jvm_dtests: - type: approval - j11_jvm_dtests: requires: - start_j11_jvm_dtests - j11_build - start_j11_jvm_dtests_vnode: - type: approval - j11_jvm_dtests_vnode: requires: - start_j11_jvm_dtests_vnode - j11_build - start_j11_cqlshlib_tests: - type: approval - j11_cqlshlib_tests: requires: - start_j11_cqlshlib_tests - j11_build - start_j11_cqlshlib_cython_tests: - type: approval - j11_cqlshlib_cython_tests: requires: - start_j11_cqlshlib_cython_tests - j11_build - start_j11_dtests: - type: approval - j11_dtests: requires: - start_j11_dtests - j11_build - start_j11_dtests_vnode: - type: approval - j11_dtests_vnode: requires: - start_j11_dtests_vnode - j11_build - start_j11_dtests_offheap: - type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j11_build - start_j11_dtests_large: - type: approval - j11_dtests_large: requires: - start_j11_dtests_large - j11_build - start_j11_dtests_large_vnode: - type: approval - j11_dtests_large_vnode: requires: - start_j11_dtests_large_vnode - j11_build - start_j11_cqlsh_tests: - type: approval - j11_cqlsh_dtests_py3: requires: - start_j11_cqlsh_tests @@ -10079,7 +10011,6 @@ workflows: - start_j11_cqlsh_tests - j11_build - start_j11_cqlsh-dtests-offheap: - type: approval - j11_cqlsh_dtests_py3_offheap: requires: - start_j11_cqlsh-dtests-offheap @@ -10093,37 +10024,31 @@ workflows: - start_j11_cqlsh-dtests-offheap - j11_build - start_j11_utests_long: - type: approval - j11_utests_long: requires: - start_j11_utests_long - j11_build - start_j11_utests_cdc: - type: approval - j11_utests_cdc: requires: - start_j11_utests_cdc - j11_build - start_j11_utests_compression: - type: approval - j11_utests_compression: requires: - start_j11_utests_compression - j11_build - start_j11_utests_stress: - type: approval - j11_utests_stress: requires: - start_j11_utests_stress - j11_build - start_j11_utests_fqltool: - type: approval - j11_utests_fqltool: requires: - start_j11_utests_fqltool - j11_build - start_j11_utests_system_keyspace_directory: - type: approval - j11_utests_system_keyspace_directory: requires: - start_j11_utests_system_keyspace_directory @@ -10131,7 +10056,6 @@ workflows: java11_pre-commit_tests: jobs: - start_pre-commit_tests: - type: approval - j11_build: requires: - start_pre-commit_tests @@ -10157,13 +10081,11 @@ workflows: requires: - j11_build - start_j11_dtests_offheap: - type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j11_build - start_j11_dtests_large: - type: approval - j11_dtests_large: requires: - start_j11_dtests_large @@ -10191,7 +10113,6 @@ workflows: requires: - j11_build - start_j11_cqlsh-dtests-offheap: - type: approval - j11_cqlsh_dtests_py3_offheap: requires: - start_j11_cqlsh-dtests-offheap @@ -10205,37 +10126,31 @@ workflows: - start_j11_cqlsh-dtests-offheap - j11_build - start_utests_long: - type: approval - j11_utests_long: requires: - start_utests_long - j11_build - start_utests_cdc: - type: approval - j11_utests_cdc: requires: - start_utests_cdc - j11_build - start_utests_compression: - type: approval - j11_utests_compression: requires: - start_utests_compression - j11_build - start_utests_stress: - type: approval - j11_utests_stress: requires: - start_utests_stress - j11_build - start_utests_fqltool: - type: approval - j11_utests_fqltool: requires: - start_utests_fqltool - j11_build - start_utests_system_keyspace_directory: - type: approval - j11_utests_system_keyspace_directory: requires: - start_utests_system_keyspace_directory From 4a3c2dee8d88876e8dfd77eee0969ba29fff6bad Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 24 Sep 2024 14:31:09 -0700 Subject: [PATCH 021/199] CircleCI ask for a manual approval --- .circleci/config.yml | 85 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ed226a2ab22c..d90de438d53d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9463,190 +9463,228 @@ workflows: java8_separate_tests: jobs: - start_j8_build: + type: approval - j8_build: requires: - start_j8_build - start_j8_unit_tests: + type: approval - j8_unit_tests: requires: - start_j8_unit_tests - j8_build - start_j8_jvm_dtests: + type: approval - j8_jvm_dtests: requires: - start_j8_jvm_dtests - j8_build - start_j8_jvm_dtests_vnode: + type: approval - j8_jvm_dtests_vnode: requires: - start_j8_jvm_dtests_vnode - j8_build - start_j11_jvm_dtests: + type: approval - j11_jvm_dtests: requires: - start_j11_jvm_dtests - j8_build - start_j11_jvm_dtests_vnode: + type: approval - j11_jvm_dtests_vnode: requires: - start_j11_jvm_dtests_vnode - j8_build - start_j8_simulator_dtests: + type: approval - j8_simulator_dtests: requires: - start_j8_simulator_dtests - j8_build - start_j8_cqlshlib_tests: + type: approval - j8_cqlshlib_tests: requires: - start_j8_cqlshlib_tests - j8_build - start_j8_cqlshlib_cython_tests: + type: approval - j8_cqlshlib_cython_tests: requires: - start_j8_cqlshlib_cython_tests - j8_build - start_j11_cqlshlib_tests: + type: approval - j11_cqlshlib_tests: requires: - start_j11_cqlshlib_tests - j8_build - start_j11_cqlshlib_cython_tests: + type: approval - j11_cqlshlib_cython_tests: requires: - start_j11_cqlshlib_cython_tests - j8_build - start_j11_unit_tests: + type: approval - j11_unit_tests: requires: - start_j11_unit_tests - j8_build - start_j8_utests_long: + type: approval - j8_utests_long: requires: - start_j8_utests_long - j8_build - start_j11_utests_long: + type: approval - j11_utests_long: requires: - start_j11_utests_long - j8_build - start_j8_utests_cdc: + type: approval - j8_utests_cdc: requires: - start_j8_utests_cdc - j8_build - start_j11_utests_cdc: + type: approval - j11_utests_cdc: requires: - start_j11_utests_cdc - j8_build - start_j8_utests_compression: + type: approval - j8_utests_compression: requires: - start_j8_utests_compression - j8_build - start_j11_utests_compression: + type: approval - j11_utests_compression: requires: - start_j11_utests_compression - j8_build - start_j8_utests_stress: + type: approval - j8_utests_stress: requires: - start_j8_utests_stress - j8_build - start_j11_utests_stress: + type: approval - j11_utests_stress: requires: - start_j11_utests_stress - j8_build - start_j8_utests_fqltool: + type: approval - j8_utests_fqltool: requires: - start_j8_utests_fqltool - j8_build - start_j11_utests_fqltool: + type: approval - j11_utests_fqltool: requires: - start_j11_utests_fqltool - j8_build - start_j8_utests_system_keyspace_directory: + type: approval - j8_utests_system_keyspace_directory: requires: - start_j8_utests_system_keyspace_directory - j8_build - start_j11_utests_system_keyspace_directory: + type: approval - j11_utests_system_keyspace_directory: requires: - start_j11_utests_system_keyspace_directory - j8_build - start_j8_dtest_jars_build: + type: approval - j8_dtest_jars_build: requires: - j8_build - start_j8_dtest_jars_build - start_jvm_upgrade_dtests: + type: approval - j8_jvm_upgrade_dtests: requires: - start_jvm_upgrade_dtests - j8_dtest_jars_build - start_j8_dtests: + type: approval - j8_dtests: requires: - start_j8_dtests - j8_build - start_j8_dtests_vnode: + type: approval - j8_dtests_vnode: requires: - start_j8_dtests_vnode - j8_build - start_j8_dtests_offheap: + type: approval - j8_dtests_offheap: requires: - start_j8_dtests_offheap - j8_build - start_j11_dtests: + type: approval - j11_dtests: requires: - start_j11_dtests - j8_build - start_j11_dtests_vnode: + type: approval - j11_dtests_vnode: requires: - start_j11_dtests_vnode - j8_build - start_j11_dtests_offheap: + type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j8_build - start_j8_dtests_large: + type: approval - j8_dtests_large: requires: - start_j8_dtests_large - j8_build - start_j8_dtests_large_vnode: + type: approval - j8_dtests_large_vnode: requires: - start_j8_dtests_large_vnode - j8_build - start_j11_dtests_large: + type: approval - j11_dtests_large: requires: - start_j11_dtests_large - j8_build - start_j11_dtests_large_vnode: + type: approval - j11_dtests_large_vnode: requires: - start_j11_dtests_large_vnode - j8_build - start_upgrade_dtests: + type: approval - j8_upgrade_dtests: requires: - start_upgrade_dtests - j8_build - start_j8_cqlsh_tests: + type: approval - j8_cqlsh_dtests_py3: requires: - start_j8_cqlsh_tests @@ -9672,6 +9710,7 @@ workflows: - start_j8_cqlsh_tests - j8_build - start_j8_cqlsh_tests_offheap: + type: approval - j8_cqlsh_dtests_py3_offheap: requires: - start_j8_cqlsh_tests_offheap @@ -9685,6 +9724,7 @@ workflows: - start_j8_cqlsh_tests_offheap - j8_build - start_j11_cqlsh_tests: + type: approval - j11_cqlsh_dtests_py3: requires: - start_j11_cqlsh_tests @@ -9710,6 +9750,7 @@ workflows: - start_j11_cqlsh_tests - j8_build - start_j11_cqlsh_tests_offheap: + type: approval - j11_cqlsh_dtests_py3_offheap: requires: - start_j11_cqlsh_tests_offheap @@ -9725,6 +9766,7 @@ workflows: java8_pre-commit_tests: jobs: - start_pre-commit_tests: + type: approval - j8_build: requires: - start_pre-commit_tests @@ -9762,6 +9804,7 @@ workflows: requires: - j8_build - start_utests_long: + type: approval - j8_utests_long: requires: - start_utests_long @@ -9771,6 +9814,7 @@ workflows: - start_utests_long - j8_build - start_utests_cdc: + type: approval - j8_utests_cdc: requires: - start_utests_cdc @@ -9780,6 +9824,7 @@ workflows: - start_utests_cdc - j8_build - start_utests_compression: + type: approval - j8_utests_compression: requires: - start_utests_compression @@ -9789,6 +9834,7 @@ workflows: - start_utests_compression - j8_build - start_utests_stress: + type: approval - j8_utests_stress: requires: - start_utests_stress @@ -9798,6 +9844,7 @@ workflows: - start_utests_stress - j8_build - start_utests_fqltool: + type: approval - j8_utests_fqltool: requires: - start_utests_fqltool @@ -9807,6 +9854,7 @@ workflows: - start_utests_fqltool - j8_build - start_utests_system_keyspace_directory: + type: approval - j8_utests_system_keyspace_directory: requires: - j8_build @@ -9815,6 +9863,7 @@ workflows: - start_utests_system_keyspace_directory - j8_build - start_jvm_upgrade_dtests: + type: approval - j8_dtest_jars_build: requires: - j8_build @@ -9829,6 +9878,7 @@ workflows: requires: - j8_build - start_j8_dtests_offheap: + type: approval - j8_dtests_offheap: requires: - start_j8_dtests_offheap @@ -9840,11 +9890,13 @@ workflows: requires: - j8_build - start_j11_dtests_offheap: + type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j8_build - start_j8_dtests_large: + type: approval - j8_dtests_large: requires: - start_j8_dtests_large @@ -9854,6 +9906,7 @@ workflows: - start_j8_dtests_large - j8_build - start_j11_dtests_large: + type: approval - j11_dtests_large: requires: - start_j11_dtests_large @@ -9863,6 +9916,7 @@ workflows: - start_j11_dtests_large - j8_build - start_upgrade_dtests: + type: approval - j8_upgrade_dtests: requires: - j8_build @@ -9886,6 +9940,7 @@ workflows: requires: - j8_build - start_j8_cqlsh_dtests_offheap: + type: approval - j8_cqlsh_dtests_py3_offheap: requires: - start_j8_cqlsh_dtests_offheap @@ -9917,6 +9972,7 @@ workflows: requires: - j8_build - start_j11_cqlsh-dtests-offheap: + type: approval - j11_cqlsh_dtests_py3_offheap: requires: - start_j11_cqlsh-dtests-offheap @@ -9932,60 +9988,72 @@ workflows: java11_separate_tests: jobs: - start_j11_build: + type: approval - j11_build: requires: - start_j11_build - start_j11_unit_tests: + type: approval - j11_unit_tests: requires: - start_j11_unit_tests - j11_build - start_j11_jvm_dtests: + type: approval - j11_jvm_dtests: requires: - start_j11_jvm_dtests - j11_build - start_j11_jvm_dtests_vnode: + type: approval - j11_jvm_dtests_vnode: requires: - start_j11_jvm_dtests_vnode - j11_build - start_j11_cqlshlib_tests: + type: approval - j11_cqlshlib_tests: requires: - start_j11_cqlshlib_tests - j11_build - start_j11_cqlshlib_cython_tests: + type: approval - j11_cqlshlib_cython_tests: requires: - start_j11_cqlshlib_cython_tests - j11_build - start_j11_dtests: + type: approval - j11_dtests: requires: - start_j11_dtests - j11_build - start_j11_dtests_vnode: + type: approval - j11_dtests_vnode: requires: - start_j11_dtests_vnode - j11_build - start_j11_dtests_offheap: + type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j11_build - start_j11_dtests_large: + type: approval - j11_dtests_large: requires: - start_j11_dtests_large - j11_build - start_j11_dtests_large_vnode: + type: approval - j11_dtests_large_vnode: requires: - start_j11_dtests_large_vnode - j11_build - start_j11_cqlsh_tests: + type: approval - j11_cqlsh_dtests_py3: requires: - start_j11_cqlsh_tests @@ -10011,6 +10079,7 @@ workflows: - start_j11_cqlsh_tests - j11_build - start_j11_cqlsh-dtests-offheap: + type: approval - j11_cqlsh_dtests_py3_offheap: requires: - start_j11_cqlsh-dtests-offheap @@ -10024,31 +10093,37 @@ workflows: - start_j11_cqlsh-dtests-offheap - j11_build - start_j11_utests_long: + type: approval - j11_utests_long: requires: - start_j11_utests_long - j11_build - start_j11_utests_cdc: + type: approval - j11_utests_cdc: requires: - start_j11_utests_cdc - j11_build - start_j11_utests_compression: + type: approval - j11_utests_compression: requires: - start_j11_utests_compression - j11_build - start_j11_utests_stress: + type: approval - j11_utests_stress: requires: - start_j11_utests_stress - j11_build - start_j11_utests_fqltool: + type: approval - j11_utests_fqltool: requires: - start_j11_utests_fqltool - j11_build - start_j11_utests_system_keyspace_directory: + type: approval - j11_utests_system_keyspace_directory: requires: - start_j11_utests_system_keyspace_directory @@ -10056,6 +10131,7 @@ workflows: java11_pre-commit_tests: jobs: - start_pre-commit_tests: + type: approval - j11_build: requires: - start_pre-commit_tests @@ -10081,11 +10157,13 @@ workflows: requires: - j11_build - start_j11_dtests_offheap: + type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j11_build - start_j11_dtests_large: + type: approval - j11_dtests_large: requires: - start_j11_dtests_large @@ -10113,6 +10191,7 @@ workflows: requires: - j11_build - start_j11_cqlsh-dtests-offheap: + type: approval - j11_cqlsh_dtests_py3_offheap: requires: - start_j11_cqlsh-dtests-offheap @@ -10126,31 +10205,37 @@ workflows: - start_j11_cqlsh-dtests-offheap - j11_build - start_utests_long: + type: approval - j11_utests_long: requires: - start_utests_long - j11_build - start_utests_cdc: + type: approval - j11_utests_cdc: requires: - start_utests_cdc - j11_build - start_utests_compression: + type: approval - j11_utests_compression: requires: - start_utests_compression - j11_build - start_utests_stress: + type: approval - j11_utests_stress: requires: - start_utests_stress - j11_build - start_utests_fqltool: + type: approval - j11_utests_fqltool: requires: - start_utests_fqltool - j11_build - start_utests_system_keyspace_directory: + type: approval - j11_utests_system_keyspace_directory: requires: - start_utests_system_keyspace_directory From 232df1c4a4f7802d94c49a052680116267911793 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 24 Sep 2024 20:27:33 -0700 Subject: [PATCH 022/199] Fix the JMXStandardsTest::interfaces --- .../apache/cassandra/repair/autorepair/AutoRepairConfig.java | 2 +- .../apache/cassandra/config/YamlConfigurationLoaderTest.java | 1 - test/unit/org/apache/cassandra/tools/JMXStandardsTest.java | 5 +++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 97868a594f25..5c839063c036 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -56,7 +56,7 @@ public class AutoRepairConfig implements Serializable // global_settings overides Options.defaultOptions for all repair types public volatile Options global_settings; - public enum RepairType + public enum RepairType implements Serializable { full, incremental; diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java index 3de7dd458009..ff7e0d0210dc 100644 --- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java +++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java @@ -29,7 +29,6 @@ import java.util.function.Predicate; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import org.junit.Test; import org.apache.cassandra.distributed.shared.WithProperties; diff --git a/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java b/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java index 62b6b9b5ec16..df0a2c1be2ea 100644 --- a/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java +++ b/test/unit/org/apache/cassandra/tools/JMXStandardsTest.java @@ -51,6 +51,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.utils.BreaksJMX; import org.assertj.core.api.Assertions; import org.reflections.Reflections; @@ -100,6 +102,9 @@ public class JMXStandardsTest .add(ClassNotFoundException.class) .add(OpenDataException.class) .add(InvalidRequestException.class) + .add(AutoRepairConfig.RepairType.class) + .add(InetAddressAndPort.class) + .add(AutoRepairConfig.class) .build(); /** * This list is a set of types under java.* and javax.*, but are too vague that could cause issues; this does not From 48d26ed588f989935e5b41258f10966617429bee Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 24 Sep 2024 21:00:25 -0700 Subject: [PATCH 023/199] Circle AutoStart J8 tests --- .circleci/config.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d90de438d53d..dd9fe01d0ad0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9463,36 +9463,30 @@ workflows: java8_separate_tests: jobs: - start_j8_build: - type: approval - j8_build: requires: - start_j8_build - start_j8_unit_tests: - type: approval - j8_unit_tests: requires: - start_j8_unit_tests - j8_build - start_j8_jvm_dtests: - type: approval - j8_jvm_dtests: requires: - start_j8_jvm_dtests - j8_build - start_j8_jvm_dtests_vnode: - type: approval - j8_jvm_dtests_vnode: requires: - start_j8_jvm_dtests_vnode - j8_build - start_j11_jvm_dtests: - type: approval - j11_jvm_dtests: requires: - start_j11_jvm_dtests - j8_build - start_j11_jvm_dtests_vnode: - type: approval - j11_jvm_dtests_vnode: requires: - start_j11_jvm_dtests_vnode From 8b05174eb582297260eb94623005722dd2302740 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 25 Sep 2024 21:15:17 -0700 Subject: [PATCH 024/199] Allow IR to run with MVs and CDC if write path replay is disabled --- .circleci/config.yml | 6 + .../config/CassandraRelevantProperties.java | 11 ++ .../db/streaming/CassandraStreamReceiver.java | 6 +- .../repair/autorepair/AutoRepair.java | 16 +-- .../repair/autorepair/AutoRepairConfig.java | 6 - .../cassandra/service/AutoRepairService.java | 18 +++ .../cassandra/streaming/StreamOperation.java | 4 +- .../streaming/CassandraSreamReceiverTest.java | 134 ++++++++++++++++++ .../autorepair/AutoRepairConfigTest.java | 57 -------- .../AutoRepairParameterizedTest.java | 58 ++++++++ .../repair/autorepair/AutoRepairTest.java | 16 ++- .../service/AutoRepairServiceTest.java | 63 +++++++- 12 files changed, 312 insertions(+), 83 deletions(-) create mode 100644 test/unit/org/apache/cassandra/db/streaming/CassandraSreamReceiverTest.java diff --git a/.circleci/config.yml b/.circleci/config.yml index dd9fe01d0ad0..d90de438d53d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9463,30 +9463,36 @@ workflows: java8_separate_tests: jobs: - start_j8_build: + type: approval - j8_build: requires: - start_j8_build - start_j8_unit_tests: + type: approval - j8_unit_tests: requires: - start_j8_unit_tests - j8_build - start_j8_jvm_dtests: + type: approval - j8_jvm_dtests: requires: - start_j8_jvm_dtests - j8_build - start_j8_jvm_dtests_vnode: + type: approval - j8_jvm_dtests_vnode: requires: - start_j8_jvm_dtests_vnode - j8_build - start_j11_jvm_dtests: + type: approval - j11_jvm_dtests: requires: - start_j11_jvm_dtests - j8_build - start_j11_jvm_dtests_vnode: + type: approval - j11_jvm_dtests_vnode: requires: - start_j11_jvm_dtests_vnode diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java index 4ee11ec591d4..380dfb0a3df2 100644 --- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java +++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java @@ -346,6 +346,17 @@ public enum CassandraRelevantProperties */ GOSSIP_SETTLE_POLL_SUCCESSES_REQUIRED("cassandra.gossip_settle_poll_success_required", "3"), + /** + * If set to true, mutations streamed during anti-entropy repair will be replayed via the regular write path for associated views. + */ + STREAMING_REQUIRES_VIEW_BUILD_DURING_REPAIR("cassandra.streaming.requires_view_build_during_repair", "true"), + + /** + * If set to true, streamed mutations via the regular write path for CDC. + * Deprecate this property in trunk (or 5.0) as a new config has been added to control this https://issues.apache.org/jira/browse/CASSANDRA-17666 + */ + STREAMING_REQUIRES_CDC_REPLAY("cassandra.streaming.requires_cdc_replay", "true"), + ; CassandraRelevantProperties(String key, String defaultVal) diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index 48de8b54fce8..95ede080738c 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -26,6 +26,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.io.sstable.SSTable; @@ -181,9 +182,10 @@ private boolean hasCDC(ColumnFamilyStore cfs) * For CDC-enabled tables, we want to ensure that the mutations are run through the CommitLog so they * can be archived by the CDC process on discard. */ - private boolean requiresWritePath(ColumnFamilyStore cfs) + public boolean requiresWritePath(ColumnFamilyStore cfs) { - return hasCDC(cfs) || cfs.streamToMemtable() || (session.streamOperation().requiresViewBuild() && hasViews(cfs)); + // TODO: Deprecate STREAMING_REQUIRES_CDC_REPLAY property in trunk (or 5.0) as a new config has been added to control this https://issues.apache.org/jira/browse/CASSANDRA-17666 + return (hasCDC(cfs) && CassandraRelevantProperties.STREAMING_REQUIRES_CDC_REPLAY.getBoolean()) || cfs.streamToMemtable() || (session.streamOperation().requiresViewBuild() && hasViews(cfs)); } private void sendThroughWritePath(ColumnFamilyStore cfs, Collection readers) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 69db9e609afb..90b7a07fbcca 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -47,7 +47,6 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.Tables; @@ -105,14 +104,15 @@ protected AutoRepair() public void setup() { - verifyIsSafeToEnable(); - AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); AutoRepairService.setup(); AutoRepairUtils.setup(); for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) { + if (config.isAutoRepairEnabled(repairType)) + AutoRepairService.instance.checkCanRun(repairType); + repairExecutors.get(repairType).scheduleWithFixedDelay( () -> repair(repairType, 5000), config.getInitialSchedulerDelay(repairType).toSeconds(), @@ -121,15 +121,6 @@ public void setup() } } - @VisibleForTesting - protected void verifyIsSafeToEnable() - { - AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); - if (config.isAutoRepairEnabled(AutoRepairConfig.RepairType.incremental) && - (DatabaseDescriptor.getMaterializedViewsEnabled() || DatabaseDescriptor.isCDCEnabled())) - throw new ConfigurationException("Cannot enable incremental repair with materialized views or CDC enabled"); - } - // repairAsync runs a repair session of the given type asynchronously. public void repairAsync(AutoRepairConfig.RepairType repairType, long millisToWait) { @@ -145,6 +136,7 @@ public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) logger.debug("Auto-repair is disabled for repair type {}", repairType); return; } + AutoRepairService.instance.checkCanRun(repairType); AutoRepairState repairState = repairStates.get(repairType); try { diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 5c839063c036..b43262e9a4e4 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -30,9 +30,7 @@ import com.google.common.annotations.VisibleForTesting; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; -import org.apache.cassandra.exceptions.ConfigurationException; public class AutoRepairConfig implements Serializable { @@ -140,10 +138,6 @@ public boolean isAutoRepairEnabled(RepairType repairType) public void setAutoRepairEnabled(RepairType repairType, boolean enabled) { - if (enabled && repairType == RepairType.incremental && - (DatabaseDescriptor.getMaterializedViewsEnabled() || DatabaseDescriptor.isCDCEnabled())) - throw new ConfigurationException("Cannot enable incremental repair with materialized views or CDC enabled"); - ensureOverrides(repairType); repair_type_overrides.get(repairType).enabled = enabled; } diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index cbef9889fb27..3d59207a8c28 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -17,7 +17,9 @@ */ package org.apache.cassandra.service; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; @@ -52,6 +54,21 @@ public static void setup() MBeanWrapper.instance.registerMBean(instance, MBEAN_NAME); } + public void checkCanRun(RepairType repairType) + { + if (!config.isAutoRepairSchedulingEnabled()) + throw new ConfigurationException("Auto-repair scheduller is disabled."); + + if (repairType != RepairType.incremental) + return; + + if (CassandraRelevantProperties.STREAMING_REQUIRES_VIEW_BUILD_DURING_REPAIR.getBoolean()) + throw new ConfigurationException("Cannot run incremental repair while materialized view replay is enabled."); + + if (CassandraRelevantProperties.STREAMING_REQUIRES_CDC_REPLAY.getBoolean()) + throw new ConfigurationException("Cannot run incremental repair while CDC replay is enabled."); + } + @Override public AutoRepairConfig getAutoRepairConfig() { @@ -61,6 +78,7 @@ public AutoRepairConfig getAutoRepairConfig() @Override public void setAutoRepairEnabled(RepairType repairType, boolean enabled) { + checkCanRun(repairType); config.setAutoRepairEnabled(repairType, enabled); } diff --git a/src/java/org/apache/cassandra/streaming/StreamOperation.java b/src/java/org/apache/cassandra/streaming/StreamOperation.java index 8151b47b22b1..1e1be1d1be89 100644 --- a/src/java/org/apache/cassandra/streaming/StreamOperation.java +++ b/src/java/org/apache/cassandra/streaming/StreamOperation.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.streaming; +import org.apache.cassandra.config.CassandraRelevantProperties; + public enum StreamOperation { OTHER("Other"), // Fallback to avoid null types when deserializing from string @@ -26,7 +28,7 @@ public enum StreamOperation BOOTSTRAP("Bootstrap", false), REBUILD("Rebuild", false), BULK_LOAD("Bulk Load"), - REPAIR("Repair"); + REPAIR("Repair", CassandraRelevantProperties.STREAMING_REQUIRES_VIEW_BUILD_DURING_REPAIR.getBoolean()); private final String description; private final boolean requiresViewBuild; diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraSreamReceiverTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraSreamReceiverTest.java new file mode 100644 index 000000000000..43020feac991 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraSreamReceiverTest.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.streaming; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.streaming.StreamOperation; +import org.apache.cassandra.streaming.StreamSession; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +public class CassandraSreamReceiverTest extends CQLTester +{ + @Mock + private StreamSession session; + + private static final String CDC_TABLE = "cdc_table"; + private static final String MV_TABLE = "mv_table"; + private static final String CDC_MV_TABLE = "cdc_mv_table"; + private static final String NO_CDC_MV_TABLE = "no_cdc_mv_table"; + + @BeforeClass + public static void beforeClass() + { + System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); + } + + @Before + public void setup() + { + MockitoAnnotations.initMocks(this); + QueryProcessor.executeInternal(String.format("CREATE TABLE IF NOT EXISTS %s.%s (pk int PRIMARY KEY, v int) WITH cdc=true", KEYSPACE, CDC_TABLE)); + QueryProcessor.executeInternal(String.format("CREATE TABLE IF NOT EXISTS %s.%s (pk int PRIMARY KEY, v int) WITH cdc=false", KEYSPACE, MV_TABLE)); + QueryProcessor.executeInternal(String.format("CREATE MATERIALIZED VIEW IF NOT EXISTS %s.mv AS SELECT * FROM %s.%s WHERE pk IS NOT NULL PRIMARY KEY (pk)", KEYSPACE, KEYSPACE, MV_TABLE)); + QueryProcessor.executeInternal(String.format("CREATE TABLE IF NOT EXISTS %s.%s (pk int PRIMARY KEY, v int) WITH cdc=true", KEYSPACE, CDC_MV_TABLE)); + QueryProcessor.executeInternal(String.format("CREATE MATERIALIZED VIEW IF NOT EXISTS %s.mv2 AS SELECT * FROM %s.%s WHERE pk IS NOT NULL PRIMARY KEY (pk)", KEYSPACE, KEYSPACE, CDC_MV_TABLE)); + QueryProcessor.executeInternal(String.format("CREATE TABLE IF NOT EXISTS %s.%s (pk int PRIMARY KEY, v int) WITH cdc=false", KEYSPACE, NO_CDC_MV_TABLE)); + } + + @Test + public void testRequiresWritePathRepair() + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CDC_MV_TABLE); + when(session.streamOperation()).thenReturn(StreamOperation.REPAIR); + CassandraStreamReceiver receiver = new CassandraStreamReceiver(cfs, session, 1); + + assertTrue(receiver.requiresWritePath(cfs)); + } + + @Test + public void testRequiresWritePathBulkLoad() + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CDC_MV_TABLE); + when(session.streamOperation()).thenReturn(StreamOperation.BULK_LOAD); + CassandraStreamReceiver receiver = new CassandraStreamReceiver(cfs, session, 1); + + assertTrue(receiver.requiresWritePath(cfs)); + } + + @Test + public void testRequiresWritePathNoCDCOrMV() + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(NO_CDC_MV_TABLE); + when(session.streamOperation()).thenReturn(StreamOperation.BULK_LOAD); + CassandraStreamReceiver receiver = new CassandraStreamReceiver(cfs, session, 1); + + assertFalse(receiver.requiresWritePath(cfs)); + } + + @Test + public void testRequiresWritePathRepairMVOnly() + { + // Access the private field using reflection + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(MV_TABLE); + when(session.streamOperation()).thenReturn(StreamOperation.REPAIR); + CassandraStreamReceiver receiver = new CassandraStreamReceiver(cfs, session, 1); + + assertTrue(receiver.requiresWritePath(cfs)); + } + + @Test + public void testRequiresWritePathRepairCDCWithSystemProp() + { + System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); + + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CDC_TABLE); + when(session.streamOperation()).thenReturn(StreamOperation.REPAIR); + CassandraStreamReceiver receiver = new CassandraStreamReceiver(cfs, session, 1); + + assertTrue(receiver.requiresWritePath(cfs)); + + System.clearProperty("cassandra.streaming.requires_cdc_replay"); + } + + @Test + public void testDoesNotRequiresWritePathRepairCDCOnly() + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CDC_TABLE); + when(session.streamOperation()).thenReturn(StreamOperation.BULK_LOAD); + CassandraStreamReceiver receiver1 = new CassandraStreamReceiver(cfs, session, 1); + assertFalse(receiver1.requiresWritePath(cfs)); + + System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); + CassandraStreamReceiver receiver2 = new CassandraStreamReceiver(cfs, session, 1); + assertTrue(receiver2.requiresWritePath(cfs)); + + } +} diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 07eb9070fe85..da7154077bc2 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -38,7 +38,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -119,62 +118,6 @@ public void testSetAutoRepairEnabledNoMVOrCDC() assertTrue(config.repair_type_overrides.get(repairType).enabled); } - @Test - public void testSetAutoRepairEnabledWithMV() - { - DatabaseDescriptor.setCDCEnabled(false); - DatabaseDescriptor.setMaterializedViewsEnabled(true); - - try - { - config.setAutoRepairEnabled(repairType, true); - - if (repairType == AutoRepairConfig.RepairType.incremental) - { - assertFalse(config.repair_type_overrides.get(repairType).enabled); // IR should not be allowed with MV - assertNotEquals(AutoRepairConfig.RepairType.incremental, repairType); // should receive exception - } - else - { - assertTrue(config.repair_type_overrides.get(repairType).enabled); - } - } - catch (ConfigurationException e) - { - // should throw only if repairType is incremental - assertEquals(AutoRepairConfig.RepairType.incremental, repairType); - } - } - - @Test - public void testSetAutoRepairEnabledWithCDC() - { - DatabaseDescriptor.setCDCEnabled(true); - DatabaseDescriptor.setMaterializedViewsEnabled(false); - - try - { - config.setAutoRepairEnabled(repairType, true); - - - if (repairType == AutoRepairConfig.RepairType.incremental) - { - assertFalse(config.repair_type_overrides.get(repairType).enabled); // IR should not be allowed with CDC - assertNotEquals(AutoRepairConfig.RepairType.incremental, repairType); // should receive exception - } - else - { - assertTrue(config.repair_type_overrides.get(repairType).enabled); - } - } - catch (ConfigurationException e) - { - // should throw only if repairType is incremental - assertEquals(AutoRepairConfig.RepairType.incremental, repairType); - } - } - - @Test public void testSetRepairByKeyspace() { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 015e9d61ea6b..559e8ea91438 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -34,12 +34,14 @@ import org.apache.cassandra.cql3.statements.schema.TableAttributes; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.schema.AutoRepairParams; import org.apache.cassandra.schema.TableParams; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.Pair; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -77,6 +79,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.doAnswer; @@ -155,6 +158,8 @@ public static void setupClass() throws Exception @Before public void setup() { + System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); + System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); MockitoAnnotations.initMocks(this); Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).truncateBlocking(); @@ -178,6 +183,12 @@ public void setup() AutoRepair.shuffleFunc = java.util.Collections::shuffle; } + @After + public void tearDown() + { + System.clearProperty("cassandra.streaming.requires_view_build_during_repair"); + System.clearProperty("cassandra.streaming.requires_cdc_replay"); + } private void resetCounters() { @@ -677,4 +688,51 @@ public void testRepairSuccessAfterRetry() verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); } + + @Test + public void testRepairThrowsForIRWithMVReplay() + { + AutoRepair.instance.setup(); + System.setProperty("cassandra.streaming.requires_view_build_during_repair", "true"); + + if (repairType == AutoRepairConfig.RepairType.incremental) + { + try + { + AutoRepair.instance.repair(repairType, 0); + fail("Expected ConfigurationException"); + } + catch (ConfigurationException ignored) + { + } + } + else + { + AutoRepair.instance.repair(repairType, 0); + } + } + + + @Test + public void testRepairThrowsForIRWithCDCReplay() + { + AutoRepair.instance.setup(); + System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); + + if (repairType == AutoRepairConfig.RepairType.incremental) + { + try + { + AutoRepair.instance.repair(repairType, 0); + fail("Expected ConfigurationException"); + } + catch (ConfigurationException ignored) + { + } + } + else + { + AutoRepair.instance.repair(repairType, 0); + } + } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index c1837aa6a2cc..3592186365d1 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -47,6 +48,13 @@ public static void setupClass() throws Exception requireNetwork(); } + @Before + public void setup() + { + System.clearProperty("cassandra.streaming.requires_view_build_during_repair"); + System.clearProperty("cassandra.streaming.requires_cdc_replay"); + } + @Test public void testSetup() { @@ -64,9 +72,10 @@ public void testSetup() } @Test(expected = ConfigurationException.class) - public void testSetupFailsWhenIREnabledWithCDC() + public void testSetupFailsWhenIREnabledWithCDCReplay() { DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.incremental, true); + System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); DatabaseDescriptor.setCDCEnabled(true); AutoRepair instance = new AutoRepair(); @@ -74,11 +83,10 @@ public void testSetupFailsWhenIREnabledWithCDC() } @Test(expected = ConfigurationException.class) - public void testSetupFailsWhenIREnabledWithMV() + public void testSetupFailsWhenIREnabledWithMVReplay() { DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.incremental, true); - DatabaseDescriptor.setMaterializedViewsEnabled(true); - + System.setProperty("cassandra.streaming.requires_view_build_during_repair", "true"); AutoRepair instance = new AutoRepair(); instance.setup(); } diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index 0e8997d9d6c6..8de570a29e3b 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -28,6 +28,7 @@ import java.util.stream.Stream; import com.google.common.collect.ImmutableSet; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -39,6 +40,7 @@ import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; @@ -53,7 +55,7 @@ @Suite.SuiteClasses({ AutoRepairServiceTest.BasicTests.class, AutoRepairServiceTest.SetterTests.class }) public class AutoRepairServiceTest { - public static class BasicTests + public static class BasicTests extends CQLTester { private static AutoRepairService autoRepairService; private static AutoRepairConfig config; @@ -61,11 +63,21 @@ public static class BasicTests @Before public void setUp() { + System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); + System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); + DatabaseDescriptor.setMaterializedViewsEnabled(false); + DatabaseDescriptor.setCDCEnabled(false); config = new AutoRepairConfig(); autoRepairService = new AutoRepairService(); autoRepairService.config = config; } + @After + public void tearDown() + { + System.clearProperty("cassandra.streaming.requires_view_build_during_repair"); + System.clearProperty("cassandra.streaming.requires_cdc_replay"); + } @Test public void testSetup() @@ -108,6 +120,53 @@ public void testsetAutoRepairRetryBackoffInSec() assertEquals(102, config.getRepairRetryBackoff().toSeconds()); } + + @Test(expected = ConfigurationException.class) + public void testSetAutoRepairEnabledThrowsWithSchedulerDisabled() + { + autoRepairService.config = new AutoRepairConfig(false); + + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + } + + @Test(expected = ConfigurationException.class) + public void testSetAutoRepairEnabledThrowsForIRWithMVReplay() + { + autoRepairService.config = new AutoRepairConfig(true); + System.setProperty("cassandra.streaming.requires_view_build_during_repair", "true"); + + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + } + + @Test + public void testSetAutoRepairEnabledDoesNotThrowForIRWithMVReplayDisabled() + { + autoRepairService.config = new AutoRepairConfig(true); + DatabaseDescriptor.setMaterializedViewsEnabled(true); + System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); + System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); + + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + } + + @Test(expected = ConfigurationException.class) + public void testSetAutoRepairEnabledThrowsForIRWithCDCReplay() + { + autoRepairService.config = new AutoRepairConfig(true); + System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); + + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + } + + @Test + public void testSetAutoRepairEnabledDoesNotThrowForIRWithCDCReplayDisabled() + { + autoRepairService.config = new AutoRepairConfig(true); + DatabaseDescriptor.setCDCEnabled(true); + System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); + + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + } } @RunWith(Parameterized.class) @@ -195,6 +254,8 @@ public void prepare() @Test public void testSetters() { + System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); + System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); setter.accept(repairType, arg); assertEquals(arg, getter.apply(repairType)); } From abde151b8cbbafce7ee8559d29fbc880518cb5d7 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 27 Sep 2024 16:10:43 -0700 Subject: [PATCH 025/199] Add a nodetool command to get the nodes going through repair --- .../repair/autorepair/AutoRepair.java | 5 + .../repair/autorepair/AutoRepairConfig.java | 19 ++-- .../repair/autorepair/AutoRepairUtils.java | 27 ++--- .../cassandra/service/AutoRepairService.java | 28 ++++- .../service/AutoRepairServiceMBean.java | 6 ++ .../org/apache/cassandra/tools/NodeProbe.java | 5 + .../org/apache/cassandra/tools/NodeTool.java | 1 + .../tools/nodetool/AutoRepairStatus.java | 70 ++++++++++++ .../tools/nodetool/GetAutoRepairConfig.java | 4 +- .../autorepair/AutoRepairConfigTest.java | 8 +- .../AutoRepairParameterizedTest.java | 7 ++ .../autorepair/AutoRepairUtilsTest.java | 17 ++- .../service/AutoRepairServiceTest.java | 51 ++++++++- .../tools/nodetool/AutoRepairStatusTest.java | 102 ++++++++++++++++++ 14 files changed, 304 insertions(+), 46 deletions(-) create mode 100644 src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java create mode 100644 test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 90b7a07fbcca..665ec6b78423 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.utils.Pair; @@ -124,6 +125,10 @@ public void setup() // repairAsync runs a repair session of the given type asynchronously. public void repairAsync(AutoRepairConfig.RepairType repairType, long millisToWait) { + if (!AutoRepairService.instance.getAutoRepairConfig().isAutoRepairEnabled(repairType)) + { + throw new ConfigurationException("Auto-repair is disabled for repair type " + repairType); + } repairExecutors.get(repairType).submit(() -> repair(repairType, millisToWait)); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index b43262e9a4e4..09e49d3d6cfc 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -230,26 +230,26 @@ public void setRepairPrimaryTokenRangeOnly(RepairType repairType, boolean primar repair_type_overrides.get(repairType).repair_primary_token_range_only = primaryTokenRangeOnly; } - public int getParallelRepairPercentageInGroup(RepairType repairType) + public int getParallelRepairPercentage(RepairType repairType) { return applyOverrides(repairType, opt -> opt.parallel_repair_percentage); } - public void setParallelRepairPercentageInGroup(RepairType repairType, int percentageInGroup) + public void setParallelRepairPercentage(RepairType repairType, int percentage) { ensureOverrides(repairType); - repair_type_overrides.get(repairType).parallel_repair_percentage = percentageInGroup; + repair_type_overrides.get(repairType).parallel_repair_percentage = percentage; } - public int getParallelRepairCountInGroup(RepairType repairType) + public int getParallelRepairCount(RepairType repairType) { return applyOverrides(repairType, opt -> opt.parallel_repair_count); } - public void setParallelRepairCountInGroup(RepairType repairType, int countInGroup) + public void setParallelRepairCount(RepairType repairType, int count) { ensureOverrides(repairType); - repair_type_overrides.get(repairType).parallel_repair_count = countInGroup; + repair_type_overrides.get(repairType).parallel_repair_count = count; } public boolean getMVRepairEnabled(RepairType repairType) @@ -305,9 +305,6 @@ public void setRepairSessionTimeout(RepairType repairType, String repairSessionT // All fields can be modified dynamically. public static class Options implements Serializable { - // The separator separating different DCs in repair_dc_groups - public static final String DC_GROUP_SEPARATOR = "\\|"; - // defaultOptions defines the default auto-repair behavior when no overrides are defined @VisibleForTesting protected static final Options defaultOptions = getDefaultOptions(); @@ -414,8 +411,8 @@ public String toString() ", repair_by_keyspace=" + repair_by_keyspace + ", number_of_subranges=" + number_of_subranges + ", number_of_repair_threads=" + number_of_repair_threads + - ", parallel_repair_count_in_group=" + parallel_repair_count + - ", parallel_repair_percentage_in_group=" + parallel_repair_percentage + + ", parallel_repair_count=" + parallel_repair_count + + ", parallel_repair_percentage=" + parallel_repair_percentage + ", sstable_upper_threshold=" + sstable_upper_threshold + ", min_repair_interval=" + min_repair_interval + ", ignore_dcs=" + ignore_dcs + diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index 9ef64c9bac43..fa2c0c91ac12 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -294,7 +294,7 @@ public String toString() } @VisibleForTesting - public static List getAutoRepairHistoryByGroupID(RepairType repairType) + public static List getAutoRepairHistory(RepairType repairType) { UntypedResultSet repairHistoryResult; @@ -325,11 +325,6 @@ public static List getAutoRepairHistoryByGroupID(RepairType r return null; } - public static List getAutoRepairHistoryForLocalGroup(RepairType repairType) - { - return getAutoRepairHistoryByGroupID(repairType); - } - // A host may add itself in delete hosts for some other hosts due to restart or some temp gossip issue. If a node's record // delete_hosts is not growing for more than 2 hours, we consider it as a normal node so we clear the delete_hosts for that node public static void clearDeleteHosts(RepairType repairType, UUID hostId) @@ -372,7 +367,7 @@ public static void setForceRepair(RepairType repairType, UUID hostId) public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType) { - List autoRepairHistories = getAutoRepairHistoryForLocalGroup(repairType); + List autoRepairHistories = getAutoRepairHistory(repairType); return getCurrentRepairStatus(repairType, autoRepairHistories); } @@ -443,7 +438,7 @@ public static TreeSet getHostIdsInCurrentRing(RepairType repairType) // This function will return the host ID for the node which has not been repaired for longest time public static AutoRepairHistory getHostWithLongestUnrepairTime(RepairType repairType) { - List autoRepairHistories = getAutoRepairHistoryForLocalGroup(repairType); + List autoRepairHistories = getAutoRepairHistory(repairType); return getHostWithLongestUnrepairTime(autoRepairHistories); } @@ -466,16 +461,16 @@ private static AutoRepairHistory getHostWithLongestUnrepairTime(List hostIdsInCurrentRing = getHostIdsInCurrentRing(repairType, allNodesInRing); logger.info("Total nodes qualified for repair {}", hostIdsInCurrentRing.size()); - List autoRepairHistories = getAutoRepairHistoryForLocalGroup(repairType); + List autoRepairHistories = getAutoRepairHistory(repairType); Set autoRepairHistoryIds = new HashSet<>(); // 1. Remove any node that is not part of group based on goissip info @@ -553,7 +548,7 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) } } - int parallelRepairNumber = getMaxNumberOfNodeRunAutoRepairInGroup(repairType, + int parallelRepairNumber = getMaxNumberOfNodeRunAutoRepair(repairType, autoRepairHistories == null ? 0 : autoRepairHistories.size()); logger.info("Will run repairs concurrently on {} node(s)", parallelRepairNumber); @@ -568,7 +563,7 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) else { // try to fetch again - autoRepairHistories = getAutoRepairHistoryForLocalGroup(repairType); + autoRepairHistories = getAutoRepairHistory(repairType); currentRepairStatus = getCurrentRepairStatus(repairType, autoRepairHistories); if (autoRepairHistories == null || currentRepairStatus == null) { @@ -632,7 +627,7 @@ else if (currentRepairStatus.hostIdsWithOnGoingForceRepair.contains(myId)) static void deleteAutoRepairHistory(RepairType repairType, UUID hostId) { - //delete the given hostId from current local group + //delete the given hostId delStatementRepairHistory.execute(QueryState.forInternalCalls(), QueryOptions.forInternalCalls(internalQueryCL, Lists.newArrayList(ByteBufferUtil.bytes(repairType.toString()), diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 3d59207a8c28..c0b305f2974d 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -26,7 +26,10 @@ import org.apache.cassandra.repair.autorepair.AutoRepairUtils; import org.apache.cassandra.utils.MBeanWrapper; +import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.UUID; import com.google.common.annotations.VisibleForTesting; @@ -161,13 +164,13 @@ public void setPrimaryTokenRangeOnly(RepairType repairType, boolean primaryToken @Override public void setParallelRepairPercentageInGroup(RepairType repairType, int percentageInGroup) { - config.setParallelRepairPercentageInGroup(repairType, percentageInGroup); + config.setParallelRepairPercentage(repairType, percentageInGroup); } @Override public void setParallelRepairCountInGroup(RepairType repairType, int countInGroup) { - config.setParallelRepairCountInGroup(repairType, countInGroup); + config.setParallelRepairCount(repairType, countInGroup); } public void setMVRepairEnabled(RepairType repairType, boolean enabled) @@ -179,4 +182,25 @@ public void setRepairSessionTimeout(RepairType repairType, String timeout) { config.setRepairSessionTimeout(repairType, timeout); } + + @Override + public Set getOnGoingRepairHostIds(RepairType rType) + { + Set hostIds = new HashSet<>(); + List histories = AutoRepairUtils.getAutoRepairHistory(rType); + if (histories == null) + { + return null; + } + AutoRepairUtils.CurrentRepairStatus currentRepairStatus = new AutoRepairUtils.CurrentRepairStatus(histories, AutoRepairUtils.getPriorityHostIds(rType)); + for (UUID id : currentRepairStatus.hostIdsWithOnGoingRepair) + { + hostIds.add(id.toString()); + } + for (UUID id : currentRepairStatus.hostIdsWithOnGoingForceRepair) + { + hostIds.add(id.toString()); + } + return hostIds; + } } diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index 4ebf118e023e..5e62e5630043 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -45,15 +45,19 @@ public interface AutoRepairServiceMBean public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration); public void setAutoRepairMaxRetriesCount(int retries); + public void setAutoRepairRetryBackoff(String interval); + public void setRepairSSTableCountHigherThreshold(RepairType repairType, int ssTableHigherThreshold); public void setAutoRepairTableMaxRepairTime(RepairType repairType, String autoRepairTableMaxRepairTime); + public void setIgnoreDCs(RepairType repairType, Set ignorDCs); public void setPrimaryTokenRangeOnly(RepairType repairType, boolean primaryTokenRangeOnly); public void setParallelRepairPercentageInGroup(RepairType repairType, int percentageInGroup); + public void setParallelRepairCountInGroup(RepairType repairType, int countInGroup); public void setMVRepairEnabled(RepairType repairType, boolean enabled); @@ -61,4 +65,6 @@ public interface AutoRepairServiceMBean public AutoRepairConfig getAutoRepairConfig(); public void setRepairSessionTimeout(RepairType repairType, String timeout); + + public Set getOnGoingRepairHostIds(RepairType rType); } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index ebe00e90ab28..e89294c5c341 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2246,6 +2246,11 @@ public void setRepairSessionTimeout(AutoRepairConfig.RepairType repairType, Stri { autoRepairProxy.setRepairSessionTimeout(repairType, timeout); } + + public Set getOnGoingRepairHostIds(AutoRepairConfig.RepairType type) + { + return autoRepairProxy.getOnGoingRepairHostIds(type); + } } class ColumnFamilyStoreMBeanIterator implements Iterator> diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index 875d77ea3148..973a755352af 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -92,6 +92,7 @@ public NodeTool(INodeProbeFactory nodeProbeFactory, Output output) public int execute(String... args) { List> commands = newArrayList( + AutoRepairStatus.class, Assassinate.class, CassHelp.class, CfHistograms.class, diff --git a/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java new file mode 100644 index 000000000000..b29095572a2f --- /dev/null +++ b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools.nodetool; + +import java.io.PrintStream; +import java.util.Set; + +import com.google.common.annotations.VisibleForTesting; + +import io.airlift.airline.Command; +import io.airlift.airline.Option; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.NodeTool; +import org.apache.cassandra.tools.nodetool.formatter.TableBuilder; + +import static com.google.common.base.Preconditions.checkArgument; + +@Command(name = "autorepairstatus", description = "Print autorepair status") +public class AutoRepairStatus extends NodeTool.NodeToolCmd +{ + @VisibleForTesting + @Option(title = "repair type", name = { "-t", "--repair-type" }, description = "Repair type") + protected AutoRepairConfig.RepairType repairType; + + @Override + public void execute(NodeProbe probe) + { + checkArgument(repairType != null, "--repair-type is required."); + PrintStream out = probe.output().out; + TableBuilder table = new TableBuilder(); + table.add("Active Repairs"); + Set ongoingRepairHostIds = probe.getOnGoingRepairHostIds(repairType); + table.add(getSetString(ongoingRepairHostIds)); + table.printTo(out); + } + + private String getSetString(Set hostIds) + { + if (hostIds.isEmpty()) + { + return "EMPTY"; + } + StringBuilder sb = new StringBuilder(); + for (String id : hostIds) + { + sb.append(id); + sb.append(","); + } + // remove last "," + sb.setLength(Math.max(sb.length() - 1, 0)); + return sb.toString(); + } +} diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java index 30dd1ff889c4..c62063b13b7f 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java @@ -70,8 +70,8 @@ private String formatRepairTypeConfig(NodeProbe probe, RepairType repairType, Au sb.append("\n\ttable max repair time in sec: " + config.getAutoRepairTableMaxRepairTime(repairType)); sb.append("\n\tignore datacenters: " + Joiner.on(',').skipNulls().join(config.getIgnoreDCs(repairType))); sb.append("\n\trepair primary token-range: " + config.getRepairPrimaryTokenRangeOnly(repairType)); - sb.append("\n\tnumber of parallel repairs within group: " + config.getParallelRepairCountInGroup(repairType)); - sb.append("\n\tpercentage of parallel repairs within group: " + config.getParallelRepairPercentageInGroup(repairType)); + sb.append("\n\tnumber of parallel repairs within group: " + config.getParallelRepairCount(repairType)); + sb.append("\n\tpercentage of parallel repairs within group: " + config.getParallelRepairPercentage(repairType)); sb.append("\n\tmv repair enabled: " + config.getMVRepairEnabled(repairType)); sb.append("\n\tinitial scheduler delay: " + config.getInitialSchedulerDelay(repairType)); sb.append("\n\trepair setssion timeout: " + config.getRepairSessionTimeout(repairType)); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index da7154077bc2..163870df6120 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -285,7 +285,7 @@ public void testGetParallelRepairPercentageInGroup() { config.global_settings.parallel_repair_percentage = 5; - int result = config.getParallelRepairPercentageInGroup(repairType); + int result = config.getParallelRepairPercentage(repairType); assertEquals(5, result); } @@ -293,7 +293,7 @@ public void testGetParallelRepairPercentageInGroup() @Test public void testSetParallelRepairPercentageInGroup() { - config.setParallelRepairPercentageInGroup(repairType, 5); + config.setParallelRepairPercentage(repairType, 5); assert config.repair_type_overrides.get(repairType).parallel_repair_percentage == 5; } @@ -303,7 +303,7 @@ public void testGetParallelRepairCountInGroup() { config.global_settings.parallel_repair_count = 5; - int result = config.getParallelRepairCountInGroup(repairType); + int result = config.getParallelRepairCount(repairType); assertEquals(5, result); } @@ -311,7 +311,7 @@ public void testGetParallelRepairCountInGroup() @Test public void testSetParallelRepairCountInGroup() { - config.setParallelRepairCountInGroup(repairType, 5); + config.setParallelRepairCount(repairType, 5); assert config.repair_type_overrides.get(repairType).parallel_repair_count == 5; } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 559e8ea91438..b0fd974a9928 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -226,6 +226,13 @@ private void executeCQL() .forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); } + @Test(expected = ConfigurationException.class) + public void testRepairAsyncWithRepairTypeDisabled() + { + AutoRepairService.instance.getAutoRepairConfig().setAutoRepairEnabled(repairType, false); + + AutoRepair.instance.repairAsync(repairType, 60); + } @Test public void testRepairAsync() diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index 6597c7159931..bb0307ab2bca 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -171,8 +171,7 @@ public void testGetAutoRepairHistoryForLocalGroup() SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); - List history = AutoRepairUtils.getAutoRepairHistoryForLocalGroup(repairType); - + List history = AutoRepairUtils.getAutoRepairHistory(repairType); assertNotNull(history); assertEquals(1, history.size()); assertEquals(hostId, history.get(0).hostId); @@ -181,7 +180,7 @@ public void testGetAutoRepairHistoryForLocalGroup() @Test public void testGetAutoRepairHistoryForLocalGroup_empty_history() { - List history = AutoRepairUtils.getAutoRepairHistoryForLocalGroup(repairType); + List history = AutoRepairUtils.getAutoRepairHistory(repairType); assertNull(history); } @@ -270,9 +269,9 @@ public void testGetHostWithLongestUnrepairTime() @Test public void testGetMaxNumberOfNodeRunAutoRepairInGroup_0_group_size() { - DatabaseDescriptor.getAutoRepairConfig().setParallelRepairCountInGroup(repairType, 2); + DatabaseDescriptor.getAutoRepairConfig().setParallelRepairCount(repairType, 2); - int count = AutoRepairUtils.getMaxNumberOfNodeRunAutoRepairInGroup(repairType, 0); + int count = AutoRepairUtils.getMaxNumberOfNodeRunAutoRepair(repairType, 0); assertEquals(2, count); } @@ -281,11 +280,11 @@ public void testGetMaxNumberOfNodeRunAutoRepairInGroup_0_group_size() @Test public void testGetMaxNumberOfNodeRunAutoRepairInGroup_percentage() { - DatabaseDescriptor.getAutoRepairConfig().setParallelRepairCountInGroup(repairType, 2); - DatabaseDescriptor.getAutoRepairConfig().setParallelRepairPercentageInGroup(repairType, 50); + DatabaseDescriptor.getAutoRepairConfig().setParallelRepairCount(repairType, 2); + DatabaseDescriptor.getAutoRepairConfig().setParallelRepairPercentage(repairType, 50); - int count = AutoRepairUtils.getMaxNumberOfNodeRunAutoRepairInGroup(repairType, 10); + int count = AutoRepairUtils.getMaxNumberOfNodeRunAutoRepair(repairType, 10); assertEquals(5, count); } @@ -464,7 +463,7 @@ public void testMyTurnToRunRepairShouldReturnMyTurnWhenRepairOngoing() { UUID myID = UUID.randomUUID(); UUID otherID = UUID.randomUUID(); - DatabaseDescriptor.getAutoRepairConfig().setParallelRepairCountInGroup(repairType, 5); + DatabaseDescriptor.getAutoRepairConfig().setParallelRepairCount(repairType, 5); long currentMillis = System.currentTimeMillis(); // finish time less than start time means that repair is ongoing AutoRepairUtils.insertNewRepairHistory(repairType, myID, currentMillis, currentMillis - 100); diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index 8de570a29e3b..d3befd024f21 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -49,6 +49,7 @@ import org.apache.cassandra.schema.SchemaConstants; import static org.apache.cassandra.Util.setAutoRepairEnabled; +import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; import static org.junit.Assert.assertEquals; @RunWith(Suite.class) @@ -169,6 +170,52 @@ public void testSetAutoRepairEnabledDoesNotThrowForIRWithCDCReplayDisabled() } } + @RunWith(Parameterized.class) + public static class RepairTypeTests extends CQLTester + { + @Parameterized.Parameter() + public AutoRepairConfig.RepairType repairType; + + private final UUID host1 = UUID.fromString("00000000-0000-0000-0000-000000000001"); + private final UUID host2 = UUID.fromString("00000000-0000-0000-0000-000000000002"); + + private AutoRepairService instance; + + @Parameterized.Parameters(name = "repairType={0}") + public static Collection repairTypes() + { + return Arrays.asList(AutoRepairConfig.RepairType.values()); + } + + + @BeforeClass + public static void setupClass() throws Exception + { + SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); + setAutoRepairEnabled(true); + requireNetwork(); + } + + @Before + public void setUpTest() + { + AutoRepairUtils.setup(); + instance = new AutoRepairService(); + } + + @Test + public void testGetOnGoingRepairHostIds() + { + long now = System.currentTimeMillis(); + AutoRepairUtils.insertNewRepairHistory(repairType, host1, now, now - 1000000); + AutoRepairUtils.insertNewRepairHistory(repairType, host2, now, now - 1000000); + + Set hosts = instance.getOnGoingRepairHostIds(repairType); + + assertEquals(ImmutableSet.of(host1.toString(), host2.toString()), hosts); + } + } + @RunWith(Parameterized.class) public static class SetterTests extends CQLTester { @@ -196,8 +243,8 @@ public static Collection testCases() forEachRepairType(400, AutoRepairService.instance::setRepairSSTableCountHigherThreshold, config::getRepairSSTableCountHigherThreshold), forEachRepairType(ImmutableSet.of("dc1", "dc2"), AutoRepairService.instance::setIgnoreDCs, config::getIgnoreDCs), forEachRepairType(true, AutoRepairService.instance::setPrimaryTokenRangeOnly, config::getRepairPrimaryTokenRangeOnly), - forEachRepairType(600, AutoRepairService.instance::setParallelRepairPercentageInGroup, config::getParallelRepairPercentageInGroup), - forEachRepairType(700, AutoRepairService.instance::setParallelRepairCountInGroup, config::getParallelRepairCountInGroup), + forEachRepairType(600, AutoRepairService.instance::setParallelRepairPercentageInGroup, config::getParallelRepairPercentage), + forEachRepairType(700, AutoRepairService.instance::setParallelRepairCountInGroup, config::getParallelRepairCount), forEachRepairType(true, AutoRepairService.instance::setMVRepairEnabled, config::getMVRepairEnabled), forEachRepairType(ImmutableSet.of(InetAddressAndPort.getLocalHost()), AutoRepairService.instance::setRepairPriorityForHosts, AutoRepairUtils::getPriorityHosts), forEachRepairType(ImmutableSet.of(InetAddressAndPort.getLocalHost()), AutoRepairService.instance::setForceRepairForHosts, SetterTests::isLocalHostForceRepair) diff --git a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java new file mode 100644 index 000000000000..87ab1a2f02c7 --- /dev/null +++ b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools.nodetool; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Collection; + +import com.google.common.collect.ImmutableSet; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.Output; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + +@RunWith(Parameterized.class) +public class AutoRepairStatusTest +{ + @Mock + private static NodeProbe probe; + + private ByteArrayOutputStream cmdOutput; + + @Mock + private static AutoRepairConfig config; + + private static AutoRepairStatus cmd; + + @Parameterized.Parameter() + public AutoRepairConfig.RepairType repairType; + + @Parameterized.Parameters(name = "repairType={0}") + public static Collection repairTypes() + { + return Arrays.asList(AutoRepairConfig.RepairType.values()); + } + + @Before + public void setUp() + { + MockitoAnnotations.initMocks(this); + cmdOutput = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(cmdOutput); + when(probe.output()).thenReturn(new Output(out, out)); + when(probe.getAutoRepairConfig()).thenReturn(config); + cmd = new AutoRepairStatus(); + } + + @Test(expected = IllegalArgumentException.class) + public void testExecuteWithoutRepairType() + { + cmd.repairType = null; + cmd.execute(probe); + } + + @Test + public void testExecuteWithNoNodes() + { + cmd.repairType = repairType; + + cmd.execute(probe); + assertEquals("Active Repairs\n" + + "EMPTY \n", cmdOutput.toString()); + } + + @Test + public void testExecute() + { + when(probe.getOnGoingRepairHostIds(repairType)).thenReturn(ImmutableSet.of("host1", "host2", "host3", "host4")); + cmd.repairType = repairType; + + cmd.execute(probe); + + assertEquals("Active Repairs \n" + + "host1,host2,host3,host4\n", cmdOutput.toString()); + } +} From 83086985b035742795087c438eeae15ca9f649f3 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 27 Sep 2024 16:15:30 -0700 Subject: [PATCH 026/199] Adjust a couple of default configuration values --- .../apache/cassandra/repair/autorepair/AutoRepairConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 09e49d3d6cfc..ac9394d9cfe1 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -49,7 +49,7 @@ public class AutoRepairConfig implements Serializable // the maximum number of retries for a repair session. public volatile Integer repair_max_retries = 3; // the backoff time in seconds for retrying a repair session. - public volatile DurationSpec.LongSecondsBound repair_retry_backoff = new DurationSpec.LongSecondsBound("60s"); + public volatile DurationSpec.LongSecondsBound repair_retry_backoff = new DurationSpec.LongSecondsBound("30s"); // global_settings overides Options.defaultOptions for all repair types public volatile Options global_settings; @@ -332,7 +332,7 @@ protected static Options getDefaultOptions() opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h"); opts.mv_repair_enabled = false; opts.token_range_splitter = DefaultAutoRepairTokenSplitter.class.getName(); - opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("15m"); // 15 minutes + opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("5m"); // 5 minutes opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h"); // 3 hours return opts; From 00b1bb1ae4b362a48494b6df391bdab50ef5cabb Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 27 Sep 2024 16:31:43 -0700 Subject: [PATCH 027/199] Rename EMPTY --> NONE for Nodetool autorepairstatus command --- .../org/apache/cassandra/tools/nodetool/AutoRepairStatus.java | 2 +- .../apache/cassandra/tools/nodetool/AutoRepairStatusTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java index b29095572a2f..882f2a4d3cda 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java +++ b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java @@ -55,7 +55,7 @@ private String getSetString(Set hostIds) { if (hostIds.isEmpty()) { - return "EMPTY"; + return "NONE"; } StringBuilder sb = new StringBuilder(); for (String id : hostIds) diff --git a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java index 87ab1a2f02c7..a92f9ec960fe 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java @@ -85,7 +85,7 @@ public void testExecuteWithNoNodes() cmd.execute(probe); assertEquals("Active Repairs\n" + - "EMPTY \n", cmdOutput.toString()); + "NONE \n", cmdOutput.toString()); } @Test From 35100fa905601f59b365c0b55535f42e2497a696 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 27 Sep 2024 17:08:45 -0700 Subject: [PATCH 028/199] Minor code refactoring to simplify the repair API by removing the SLEEP_IF_REPAIR_FINISHES_QUICKLY --- .../repair/autorepair/AutoRepair.java | 22 ++++--- .../autorepair/AutoRepairConfigTest.java | 1 + .../autorepair/AutoRepairKeyspaceTest.java | 2 + .../AutoRepairParameterizedTest.java | 62 ++++++++++--------- .../autorepair/AutoRepairStateTest.java | 1 + .../repair/autorepair/AutoRepairTest.java | 2 + .../autorepair/AutoRepairUtilsTest.java | 2 + 7 files changed, 53 insertions(+), 39 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 665ec6b78423..b33809ecc16a 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.utils.Pair; @@ -70,6 +71,9 @@ public class AutoRepair public static AutoRepair instance = new AutoRepair(); + // Sleep for 5 seconds if repair finishes quickly to flush JMX metrics; it happens only for Cassandra nodes with tiny amount of data. + public static DurationSpec.IntSecondsBound SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("5s"); + @VisibleForTesting protected final Map repairExecutors; @@ -115,7 +119,7 @@ public void setup() AutoRepairService.instance.checkCanRun(repairType); repairExecutors.get(repairType).scheduleWithFixedDelay( - () -> repair(repairType, 5000), + () -> repair(repairType), config.getInitialSchedulerDelay(repairType).toSeconds(), config.getRepairCheckInterval().toSeconds(), TimeUnit.SECONDS); @@ -123,17 +127,17 @@ public void setup() } // repairAsync runs a repair session of the given type asynchronously. - public void repairAsync(AutoRepairConfig.RepairType repairType, long millisToWait) + public void repairAsync(AutoRepairConfig.RepairType repairType) { if (!AutoRepairService.instance.getAutoRepairConfig().isAutoRepairEnabled(repairType)) { throw new ConfigurationException("Auto-repair is disabled for repair type " + repairType); } - repairExecutors.get(repairType).submit(() -> repair(repairType, millisToWait)); + repairExecutors.get(repairType).submit(() -> repair(repairType)); } // repair runs a repair session of the given type synchronously. - public void repair(AutoRepairConfig.RepairType repairType, long millisToWait) + public void repair(AutoRepairConfig.RepairType repairType) { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); if (!config.isAutoRepairEnabled(repairType)) @@ -350,7 +354,7 @@ else if (retryCount < config.getRepairMaxRetries()) } } } - cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, millisToWait, failedTokenRanges, succeededTokenRanges, skippedTokenRanges); + cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, failedTokenRanges, succeededTokenRanges, skippedTokenRanges); } else { @@ -409,7 +413,7 @@ private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairCon } private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, UUID myId, - long startTime, long millisToWait, int failedTokenRanges, int succeededTokenRanges, int skippedTokenRanges) throws InterruptedException + long startTime, int failedTokenRanges, int succeededTokenRanges, int skippedTokenRanges) throws InterruptedException { //if it was due to priority then remove it now if (turn == MY_TURN_DUE_TO_PRIORITY) @@ -436,12 +440,12 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType TimeUnit.SECONDS.toDays(repairState.getClusterRepairTimeInSec())); } repairState.setLastRepairTime(timeFunc.get()); - if (timeInHours == 0 && millisToWait > 0) + if (timeInHours == 0 && SLEEP_IF_REPAIR_FINISHES_QUICKLY.toSeconds() > 0) { //If repair finished quickly, happens for an empty instance, in such case //wait for some duration so that the JMX metrics can detect the repairInProgress - logger.info("Wait for {} milliseconds for repair type {}.", millisToWait, repairType); - Thread.sleep(millisToWait); + logger.info("Wait for {} for repair type {}.", SLEEP_IF_REPAIR_FINISHES_QUICKLY, repairType); + Thread.sleep(SLEEP_IF_REPAIR_FINISHES_QUICKLY.toMilliseconds()); } repairState.setRepairInProgress(false); AutoRepairUtils.updateFinishAutoRepairHistory(repairType, myId, timeFunc.get()); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 163870df6120..c42959eec32b 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -62,6 +62,7 @@ public void setUp() { config = new AutoRepairConfig(true); config.repair_type_overrides = null; + AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); } @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java index 579b996a12f5..43b8e4a5f4e8 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.TableMetadata; @@ -43,6 +44,7 @@ public class AutoRepairKeyspaceTest public static void setupDatabaseDescriptor() { DatabaseDescriptor.daemonInitialization(); + AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index b0fd974a9928..797fd1a9b05d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -31,6 +31,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; +import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.cql3.statements.schema.TableAttributes; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -118,6 +119,7 @@ public static Collection repairTypes() @BeforeClass public static void setupClass() throws Exception { + AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); setAutoRepairEnabled(true); requireNetwork(); @@ -231,7 +233,7 @@ public void testRepairAsyncWithRepairTypeDisabled() { AutoRepairService.instance.getAutoRepairConfig().setAutoRepairEnabled(repairType, false); - AutoRepair.instance.repairAsync(repairType, 60); + AutoRepair.instance.repairAsync(repairType); } @Test @@ -239,7 +241,7 @@ public void testRepairAsync() { AutoRepair.instance.repairExecutors.put(repairType, mockExecutor); - AutoRepair.instance.repairAsync(repairType, 60); + AutoRepair.instance.repairAsync(repairType); verify(mockExecutor, Mockito.times(1)).submit(any(Runnable.class)); } @@ -255,7 +257,7 @@ public void testRepairTurn() public void testRepair() { AutoRepairService.instance.getAutoRepairConfig().setRepairMinInterval(repairType, "0s"); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); long lastRepairTime = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); @@ -270,7 +272,7 @@ public void testTooFrequentRepairs() AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); //in the first round let repair run config.setRepairMinInterval(repairType, "0s"); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); int consideredTables = AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair(); Assert.assertNotSame(String.format("Expected total repaired tables > 0, actual value %s ", consideredTables), @@ -278,7 +280,7 @@ public void testTooFrequentRepairs() //if repair was done in last 24 hours then it should not trigger another repair config.setRepairMinInterval(repairType, "24h"); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); Assert.assertEquals(String.format("Expected repair time to be same, actual value lastRepairTime1 %d, " + "lastRepairTime2 %d", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); @@ -293,14 +295,14 @@ public void testNonFrequentRepairs() AutoRepairState state = AutoRepair.instance.repairStates.get(repairType); long prevCount = state.getTotalMVTablesConsideredForRepair(); AutoRepairService.instance.getAutoRepairConfig().setRepairMinInterval(repairType, "0s"); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); Assert.assertTrue(String.format("Expected lastRepairTime1 > 0, actual value lastRepairTime1 %d", lastRepairTime1), lastRepairTime1 > 0); UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); Assert.assertTrue("Expected my turn for the repair", AutoRepairUtils.myTurnToRunRepair(repairType, myId) != NOT_MY_TURN); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); Assert.assertNotSame(String.format("Expected repair time to be same, actual value lastRepairTime1 %d, " + "lastRepairTime2 ", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); @@ -320,9 +322,9 @@ public void testGetPriorityHosts() UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); Assert.assertTrue("Expected my turn for the repair", AutoRepairUtils.myTurnToRunRepair(repairType, myId) != NOT_MY_TURN); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); AutoRepairUtils.addPriorityHosts(repairType, Sets.newHashSet(FBUtilities.getBroadcastAddressAndPort())); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); Assert.assertSame(String.format("Priority host count is not same actual value %d, expected value %d", AutoRepairUtils.getPriorityHosts(repairType).size(), 0), AutoRepairUtils.getPriorityHosts(repairType).size(), 0); assertEquals(prevCount, state.getTotalMVTablesConsideredForRepair()); @@ -339,14 +341,14 @@ public void testCheckAutoRepairStartStop() throws Throwable config.setRepairMinInterval(repairType, "0s"); config.setAutoRepairEnabled(repairType, false); long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); //Since repair has not happened, both the last repair times should be same Assert.assertEquals(String.format("Expected lastRepairTime1 %d, and lastRepairTime2 %d to be same", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); config.setAutoRepairEnabled(repairType, true); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); //since repair is done now, so lastRepairTime1/lastRepairTime2 and lastRepairTime3 should not be same long lastRepairTime3 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); Assert.assertNotSame(String.format("Expected lastRepairTime1 %d, and lastRepairTime3 %d to be not same", @@ -385,19 +387,19 @@ public void testMVRepair() config.setMVRepairEnabled(repairType, true); config.setRepairMinInterval(repairType, "0s"); AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); config.setMVRepairEnabled(repairType, false); AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); config.setMVRepairEnabled(repairType, true); AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); } @@ -436,7 +438,7 @@ public void testSkipRepairSSTableCountHigherThreshold() assertEquals(0, state.getSkippedTokenRangesCount()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); state.setLastRepairTime(0); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); assertEquals(1, state.getTotalMVTablesConsideredForRepair()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); // skipping one time for the base table and another time for MV table @@ -447,7 +449,7 @@ public void testSkipRepairSSTableCountHigherThreshold() config.setRepairSSTableCountHigherThreshold(repairType, 11); config.setRepairSSTableCountHigherThreshold(repairType, beforeCount); state.setLastRepairTime(0); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); assertEquals(1, state.getTotalMVTablesConsideredForRepair()); assertEquals(0, state.getSkippedTokenRangesCount()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); @@ -478,7 +480,7 @@ public void testMetrics() }; AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(1000L); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); assertTrue(AutoRepairMetricsManager.getMetrics(repairType).nodeRepairTimeInSec.getValue() > 0); @@ -495,7 +497,7 @@ public void testMetrics() when(autoRepairState.getSucceededTokenRangesCount()).thenReturn(11); when(autoRepairState.getLongestUnrepairedSec()).thenReturn(10); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); assertTrue(AutoRepairMetricsManager.getMetrics(repairType).failedTokenRangesCount.getValue() > 0); assertTrue(AutoRepairMetricsManager.getMetrics(repairType).succeededTokenRangesCount.getValue() > 0); @@ -527,9 +529,9 @@ public void testRepairWaitsForRepairToFinishBeforeSchedullingNewSession() throws return null; }).when(autoRepairState).waitForRepairToComplete(config.getRepairSessionTimeout(repairType)); - AutoRepair.instance.repair(repairType, 0); - AutoRepair.instance.repair(repairType, 0); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); + AutoRepair.instance.repair(repairType); + AutoRepair.instance.repair(repairType); } @Test @@ -543,7 +545,7 @@ public void testDisabledAutoRepairForATableThroughTableLevelConfiguration() AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setRepairMinInterval(repairType, "0s"); int disabledTablesRepairCountBefore = AutoRepair.instance.repairStates.get(repairType).getTotalDisabledTablesRepairCount(); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); int consideredTables = AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair(); Assert.assertNotSame(String.format("Expected total repaired tables > 0, actual value %s ", consideredTables), consideredTables, 0); @@ -616,7 +618,7 @@ else if (list.get(0) instanceof String) AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setRepairMinInterval(repairType, "0s"); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); assertEquals(1, shuffleKeyspacesCall.get()); assertEquals(4, shuffleTablesCall.get()); @@ -632,7 +634,7 @@ public void testRepairTakesLastRepairTimeFromDB() AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(0); config.setRepairMinInterval(repairType, "1h"); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); // repair scheduler should not attempt to run repair as last repair time in DB is current time - 1s assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair()); @@ -655,7 +657,7 @@ public void testRepairMaxRetries() config.setRepairMinInterval(repairType, "0s"); AutoRepair.instance.repairStates.put(repairType, autoRepairState); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); //system_auth.role_permissions,system_auth.network_permissions,system_auth.role_members,system_auth.roles, // system_auth.resource_role_permissons_index,system_traces.sessions,system_traces.events,ks.tbl, @@ -688,7 +690,7 @@ public void testRepairSuccessAfterRetry() }); config.setRepairMinInterval(repairType, "0s"); AutoRepair.instance.repairStates.put(repairType, autoRepairState); - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); assertEquals(1, sleepCalls.get()); verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(14); @@ -706,7 +708,7 @@ public void testRepairThrowsForIRWithMVReplay() { try { - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); fail("Expected ConfigurationException"); } catch (ConfigurationException ignored) @@ -715,7 +717,7 @@ public void testRepairThrowsForIRWithMVReplay() } else { - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); } } @@ -730,7 +732,7 @@ public void testRepairThrowsForIRWithCDCReplay() { try { - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); fail("Expected ConfigurationException"); } catch (ConfigurationException ignored) @@ -739,7 +741,7 @@ public void testRepairThrowsForIRWithCDCReplay() } else { - AutoRepair.instance.repair(repairType, 0); + AutoRepair.instance.repair(repairType); } } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java index 657b11310f64..f0974dd83c1a 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -68,6 +68,7 @@ public static Collection repairTypes() @Before public void setUp() { + AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); initMocks(this); createTable(String.format("CREATE TABLE IF NOT EXISTS %s.%s (pk int PRIMARY KEY, v int)", KEYSPACE, testTable)); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index 3592186365d1..6137983b3ea7 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -27,6 +27,7 @@ import org.junit.Assert; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.exceptions.ConfigurationException; @@ -51,6 +52,7 @@ public static void setupClass() throws Exception @Before public void setup() { + AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); System.clearProperty("cassandra.streaming.requires_view_build_during_repair"); System.clearProperty("cassandra.streaming.requires_cdc_replay"); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index bb0307ab2bca..09685c5f5110 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -31,6 +31,7 @@ import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.marshal.UTF8Type; @@ -99,6 +100,7 @@ public static void setupClass() throws Exception @Before public void setup() { + AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); MockitoAnnotations.initMocks(this); DatabaseDescriptor.setEndpointSnitch(defaultSnitch); QueryProcessor.executeInternal(String.format( From c6811577a7d7b5dbe4519cb3c1691ab33f6c208a Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 30 Sep 2024 11:59:25 -0700 Subject: [PATCH 029/199] Fix unit test cases DescribeStatementTest & SchemaCQLHelperTest --- .../cassandra/cql3/statements/DescribeStatementTest.java | 8 ++++++-- .../unit/org/apache/cassandra/db/SchemaCQLHelperTest.java | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index 16ca40691d69..4d2c1b105058 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -1004,7 +1004,9 @@ private static String tableParametersCql() " AND memtable_flush_period_in_ms = 0\n" + " AND min_index_interval = 128\n" + " AND read_repair = 'BLOCKING'\n" + - " AND speculative_retry = '99p';"; + " AND speculative_retry = '99p'\n" + + " AND automated_repair_full = {'enabled': 'true'}\n" + + " AND automated_repair_incremental = {'enabled': 'true'};"; } private static String mvParametersCql() @@ -1024,7 +1026,9 @@ private static String mvParametersCql() " AND memtable_flush_period_in_ms = 0\n" + " AND min_index_interval = 128\n" + " AND read_repair = 'BLOCKING'\n" + - " AND speculative_retry = '99p';"; + " AND speculative_retry = '99p'\n" + + " AND automated_repair_full = {'enabled': 'true'}\n" + + " AND automated_repair_incremental = {'enabled': 'true'};"; } private static String keyspaceOutput() diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java index 63ef86110177..39b2449a0d66 100644 --- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java @@ -312,7 +312,9 @@ public void testCfmOptionsCQL() " AND memtable_flush_period_in_ms = 8\n" + " AND min_index_interval = 6\n" + " AND read_repair = 'BLOCKING'\n" + - " AND speculative_retry = 'ALWAYS';" + " AND speculative_retry = 'ALWAYS'\n" + + " AND automated_repair_full = {'enabled': 'true'}\n" + + " AND automated_repair_incremental = {'enabled': 'true'};" )); } From 364a181015ae42fcc9af61274053f5bfd3d17f6c Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 30 Sep 2024 13:42:40 -0700 Subject: [PATCH 030/199] Safeguard the setup call to ensure it it done only once --- .../repair/autorepair/AutoRepair.java | 37 ++++++++++++------- .../repair/autorepair/AutoRepairTest.java | 27 ++++++++++++-- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index b33809ecc16a..87d318c7e76f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -90,6 +90,7 @@ public class AutoRepair protected final Map tokenRangeSplitters = new EnumMap<>(AutoRepairConfig.RepairType.class); + private boolean isSetupDone = false; @VisibleForTesting protected AutoRepair() @@ -109,20 +110,30 @@ protected AutoRepair() public void setup() { - AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); - AutoRepairService.setup(); - AutoRepairUtils.setup(); - - for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + // Ensure setup is done only once; this is only for unit tests + // For production, this method should be called only once. + synchronized (this) { - if (config.isAutoRepairEnabled(repairType)) - AutoRepairService.instance.checkCanRun(repairType); - - repairExecutors.get(repairType).scheduleWithFixedDelay( - () -> repair(repairType), - config.getInitialSchedulerDelay(repairType).toSeconds(), - config.getRepairCheckInterval().toSeconds(), - TimeUnit.SECONDS); + if (isSetupDone) + { + return; + } + AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); + AutoRepairService.setup(); + AutoRepairUtils.setup(); + + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + if (config.isAutoRepairEnabled(repairType)) + AutoRepairService.instance.checkCanRun(repairType); + + repairExecutors.get(repairType).scheduleWithFixedDelay( + () -> repair(repairType), + config.getInitialSchedulerDelay(repairType).toSeconds(), + config.getRepairCheckInterval().toSeconds(), + TimeUnit.SECONDS); + } + isSetupDone = true; } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index 6137983b3ea7..8c4ff01233c0 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -67,9 +67,30 @@ public void testSetup() assertEquals(RepairType.values().length, instance.repairExecutors.size()); for (RepairType repairType : instance.repairExecutors.keySet()) { - assertEquals(String.format("Expected 1 task in queue for %s", repairType), - 1, instance.repairExecutors.get(repairType).getPendingTaskCount() - + instance.repairExecutors.get(repairType).getActiveTaskCount()); + int expectedTasks = instance.repairExecutors.get(repairType).getPendingTaskCount() + + instance.repairExecutors.get(repairType).getActiveTaskCount(); + assertEquals(String.format("Expected 1 task in queue for %s but was %s", repairType, expectedTasks), + 1, expectedTasks); + } + } + + @Test + public void testSafeGuardSetupCall() + { + AutoRepair instance = new AutoRepair(); + + // only one should be setup, and rest should be ignored + instance.setup(); + instance.setup(); + instance.setup(); + + assertEquals(RepairType.values().length, instance.repairExecutors.size()); + for (RepairType repairType : instance.repairExecutors.keySet()) + { + int expectedTasks = instance.repairExecutors.get(repairType).getPendingTaskCount() + + instance.repairExecutors.get(repairType).getActiveTaskCount(); + assertEquals(String.format("Expected 1 task in queue for %s but was %s", repairType, expectedTasks), + 1, expectedTasks); } } From 37017c5cddea4bdae8e5591a12bc4fd90ccc0b68 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 30 Sep 2024 20:54:49 -0700 Subject: [PATCH 031/199] Fix InJvm test AutoRepairSchedulerTest --- .../test/repair/AutoRepairSchedulerTest.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index 49b4300dc50b..67ece5de995d 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -22,9 +22,11 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.concurrent.TimeUnit; +import java.util.UUID; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Uninterruptibles; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -48,6 +50,9 @@ public class AutoRepairSchedulerTest extends TestBaseImpl @BeforeClass public static void init() throws IOException { + System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); + System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); + // Define the expected date format pattern String pattern = "EEE MMM dd HH:mm:ss z yyyy"; // Create SimpleDateFormat object with the given pattern @@ -78,6 +83,13 @@ public static void init() throws IOException cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck text, v1 int, v2 int, PRIMARY KEY (pk, ck)) WITH read_repair='NONE'")); } + @AfterClass + public static void afterClass() + { + System.clearProperty("cassandra.streaming.requires_view_build_during_repair"); + System.clearProperty("cassandra.streaming.requires_cdc_replay"); + } + @Test public void testScheduler() throws ParseException { @@ -112,7 +124,7 @@ private void validate(String repairType) throws ParseException // repair_type Assert.assertEquals(repairType, row[0].toString()); // host_id - Assert.assertEquals(String.format("00000000-0000-4000-8000-%012d", node + 1), row[1].toString()); + UUID.fromString(row[1].toString()); // ensure there is a legit repair_start_ts and repair_finish_ts sdf.parse(row[2].toString()); sdf.parse(row[3].toString()); From d4c894dab4fe6ba8d7bd07d87a8ad907d0ed298b Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 1 Oct 2024 09:35:28 -0700 Subject: [PATCH 032/199] Unit test failures AutoRepairTest.java --- .../repair/autorepair/AutoRepairTest.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index 8c4ff01233c0..ae9e6ca16936 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -21,11 +21,12 @@ import java.util.HashMap; import java.util.Map; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; - import org.junit.Assert; + import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.cql3.CQLTester; @@ -53,6 +54,15 @@ public static void setupClass() throws Exception public void setup() { AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); + System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); + System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.full, true); + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.incremental, true); + } + + @After + public void after() + { System.clearProperty("cassandra.streaming.requires_view_build_during_repair"); System.clearProperty("cassandra.streaming.requires_cdc_replay"); } @@ -61,7 +71,6 @@ public void setup() public void testSetup() { AutoRepair instance = new AutoRepair(); - instance.setup(); assertEquals(RepairType.values().length, instance.repairExecutors.size()); @@ -87,8 +96,7 @@ public void testSafeGuardSetupCall() assertEquals(RepairType.values().length, instance.repairExecutors.size()); for (RepairType repairType : instance.repairExecutors.keySet()) { - int expectedTasks = instance.repairExecutors.get(repairType).getPendingTaskCount() - + instance.repairExecutors.get(repairType).getActiveTaskCount(); + int expectedTasks = instance.repairExecutors.get(repairType).getCorePoolSize(); assertEquals(String.format("Expected 1 task in queue for %s but was %s", repairType, expectedTasks), 1, expectedTasks); } From be4f7f6257f32e2c6f82004856492dbbf1299c6f Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 8 Oct 2024 10:15:37 -0700 Subject: [PATCH 033/199] Fix flaky AutoRepairTest::testSetup and AutoRepairTest::testSafeGuardSetupCall --- .../cassandra/repair/autorepair/AutoRepairTest.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index ae9e6ca16936..6987cba87a8b 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -40,6 +40,7 @@ import static org.apache.cassandra.Util.setAutoRepairEnabled; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class AutoRepairTest extends CQLTester { @@ -78,8 +79,8 @@ public void testSetup() { int expectedTasks = instance.repairExecutors.get(repairType).getPendingTaskCount() + instance.repairExecutors.get(repairType).getActiveTaskCount(); - assertEquals(String.format("Expected 1 task in queue for %s but was %s", repairType, expectedTasks), - 1, expectedTasks); + assertTrue(String.format("Expected > 0 task in queue for %s but was %s", repairType, expectedTasks), + expectedTasks > 0); } } @@ -96,9 +97,10 @@ public void testSafeGuardSetupCall() assertEquals(RepairType.values().length, instance.repairExecutors.size()); for (RepairType repairType : instance.repairExecutors.keySet()) { - int expectedTasks = instance.repairExecutors.get(repairType).getCorePoolSize(); - assertEquals(String.format("Expected 1 task in queue for %s but was %s", repairType, expectedTasks), - 1, expectedTasks); + int expectedTasks = instance.repairExecutors.get(repairType).getPendingTaskCount() + + instance.repairExecutors.get(repairType).getActiveTaskCount(); + assertTrue(String.format("Expected > 0 task in queue for %s but was %s", repairType, expectedTasks), + expectedTasks > 0); } } From 662ee2394549fdafff867243234417d27d6a5392 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 8 Oct 2024 14:28:44 -0700 Subject: [PATCH 034/199] Fix a test failure inside pylib/cqlshlib/test/test_cqlsh_output.py --- pylib/cqlshlib/test/test_cqlsh_output.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index 52f564ab29a2..500d0f46c5b4 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -690,7 +690,9 @@ def test_describe_columnfamily_output(self): AND memtable_flush_period_in_ms = 0 AND min_index_interval = 128 AND read_repair = 'BLOCKING' - AND speculative_retry = '99p';""" % quote_name(get_keyspace())) + AND speculative_retry = '99p' + AND automated_repair_full = {'enabled': 'true'} + AND automated_repair_incremental = {'enabled': 'true'};""" % quote_name(get_keyspace())) with testrun_cqlsh(tty=True, env=self.default_env) as c: for cmdword in ('describe table', 'desc columnfamily'): From 919bb13fa650c6bb4898d23b96b840b13486f71d Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 8 Oct 2024 21:10:48 -0700 Subject: [PATCH 035/199] AutoRepair nodetool throws an exception if the framework is disabled --- .../apache/cassandra/repair/autorepair/AutoRepair.java | 1 - .../org/apache/cassandra/service/AutoRepairService.java | 2 +- src/java/org/apache/cassandra/service/StorageService.java | 1 + .../apache/cassandra/tools/nodetool/AutoRepairStatus.java | 8 ++++++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 87d318c7e76f..630e7185425b 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -119,7 +119,6 @@ public void setup() return; } AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); - AutoRepairService.setup(); AutoRepairUtils.setup(); for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index c0b305f2974d..62393d66ebab 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -190,7 +190,7 @@ public Set getOnGoingRepairHostIds(RepairType rType) List histories = AutoRepairUtils.getAutoRepairHistory(rType); if (histories == null) { - return null; + return hostIds; } AutoRepairUtils.CurrentRepairStatus currentRepairStatus = new AutoRepairUtils.CurrentRepairStatus(histories, AutoRepairUtils.getPriorityHostIds(rType)); for (UUID id : currentRepairStatus.hostIdsWithOnGoingRepair) diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index f35861ae50f8..cff111ef8c99 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -1339,6 +1339,7 @@ public void doAuthSetup(boolean setUpSchema) private void doAutoRepairSetup() { + AutoRepairService.setup(); if (DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) { logger.info("Enable auto-repair scheduling"); diff --git a/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java index 882f2a4d3cda..7b96102c6698 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java +++ b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java @@ -44,6 +44,14 @@ public void execute(NodeProbe probe) { checkArgument(repairType != null, "--repair-type is required."); PrintStream out = probe.output().out; + + AutoRepairConfig config = probe.getAutoRepairConfig(); + if (config == null || !config.isAutoRepairSchedulingEnabled()) + { + out.println("Auto-repair is not enabled"); + return; + } + TableBuilder table = new TableBuilder(); table.add("Active Repairs"); Set ongoingRepairHostIds = probe.getOnGoingRepairHostIds(repairType); From 2ca0adf55f7036b08313cd101a8358780aadaade Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 9 Oct 2024 09:37:23 -0700 Subject: [PATCH 036/199] Run repair on bootstrapped if configured to do so --- .../org/apache/cassandra/service/StorageService.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index cff111ef8c99..c4f79915c481 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -86,6 +86,7 @@ import com.google.common.util.concurrent.Uninterruptibles; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.repair.autorepair.AutoRepairKeyspace; import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.commons.lang3.StringUtils; @@ -174,6 +175,7 @@ import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.repair.RepairRunnable; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils; import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; import org.apache.cassandra.schema.KeyspaceMetadata; @@ -1211,6 +1213,14 @@ public void joinTokenRing(boolean finishJoiningRing, if (dataAvailable) { finishJoiningRing(shouldBootstrap, bootstrapTokens); + AutoRepairConfig repairConfig = DatabaseDescriptor.getAutoRepairConfig(); + // this node might have just bootstrapped; check if we should run repair immediately + if (shouldBootstrap && repairConfig.isAutoRepairSchedulingEnabled()) + { + for (AutoRepairConfig.RepairType rType : AutoRepairConfig.RepairType.values()) + if (repairConfig.isAutoRepairEnabled(rType) && repairConfig.getForceRepairNewNode(rType)) + AutoRepairUtils.setForceRepairNewNode(rType); + } // remove the existing info about the replaced node. if (!current.isEmpty()) { From 2854bcf5a9a65c61776b23af9f43a278c7d528df Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 9 Oct 2024 11:00:06 -0700 Subject: [PATCH 037/199] Fix broken AutoRepairSchedulerTest.java & AutoRepairStatusTest.java --- .../test/repair/AutoRepairSchedulerTest.java | 2 ++ .../tools/nodetool/AutoRepairStatusTest.java | 14 +++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index 67ece5de995d..d6e2c06602fd 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -37,6 +37,7 @@ import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.repair.autorepair.AutoRepairKeyspace; +import org.apache.cassandra.service.AutoRepairService; import static org.apache.cassandra.schema.SchemaConstants.DISTRIBUTED_KEYSPACE_NAME; import static org.junit.Assert.assertEquals; @@ -100,6 +101,7 @@ public void testScheduler() throws ParseException cluster.forEach(i -> i.runOnInstance(() -> { try { + AutoRepairService.instance.setup(); AutoRepair.instance.setup(); } catch (Exception e) diff --git a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java index a92f9ec960fe..16b12ee2350e 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java @@ -29,12 +29,14 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.Output; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import static org.apache.cassandra.Util.setAutoRepairEnabled; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.when; @@ -46,9 +48,6 @@ public class AutoRepairStatusTest private ByteArrayOutputStream cmdOutput; - @Mock - private static AutoRepairConfig config; - private static AutoRepairStatus cmd; @Parameterized.Parameter() @@ -61,14 +60,19 @@ public static Collection repairTypes() } @Before - public void setUp() + public void setUp() throws Exception { MockitoAnnotations.initMocks(this); cmdOutput = new ByteArrayOutputStream(); PrintStream out = new PrintStream(cmdOutput); when(probe.output()).thenReturn(new Output(out, out)); - when(probe.getAutoRepairConfig()).thenReturn(config); cmd = new AutoRepairStatus(); + DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.loadConfig(); + setAutoRepairEnabled(true); + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(AutoRepairConfig.RepairType.full, true); + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + when(probe.getAutoRepairConfig()).thenReturn(DatabaseDescriptor.getAutoRepairConfig()); } @Test(expected = IllegalArgumentException.class) From dbfec67458d3ccff2d4ff649f22a09be2f2764a5 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 9 Oct 2024 14:05:12 -0700 Subject: [PATCH 038/199] Comments from Chris Lohfink: move meta tables inside SystemDistributedKeyspace.java --- .../repair/autorepair/AutoRepairKeyspace.java | 84 ------------------- .../repair/autorepair/AutoRepairUtils.java | 25 +++--- .../schema/SystemDistributedKeyspace.java | 32 ++++++- .../cassandra/service/StorageService.java | 5 -- .../test/repair/AutoRepairSchedulerTest.java | 6 +- .../autorepair/AutoRepairKeyspaceTest.java | 30 +++---- .../AutoRepairParameterizedTest.java | 7 +- .../autorepair/AutoRepairUtilsTest.java | 53 ++++++------ .../service/AutoRepairServiceTest.java | 8 +- 9 files changed, 96 insertions(+), 154 deletions(-) delete mode 100644 src/java/org/apache/cassandra/repair/autorepair/AutoRepairKeyspace.java diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairKeyspace.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairKeyspace.java deleted file mode 100644 index 14cba17f6095..000000000000 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairKeyspace.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.repair.autorepair; - -import java.util.concurrent.TimeUnit; - -import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; -import org.apache.cassandra.schema.KeyspaceMetadata; -import org.apache.cassandra.schema.KeyspaceParams; -import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.Tables; - -public class AutoRepairKeyspace -{ - private AutoRepairKeyspace() - { - } - - /** - * Generation is used as a timestamp for automatic table creation on startup. - * If you make any changes to the tables below, make sure to increment the - * generation and document your change here. - */ - public static final long GENERATION = 0; - - public static final String AUTO_REPAIR_HISTORY = "auto_repair_history"; - - public static final String AUTO_REPAIR_PRIORITY = "auto_repair_priority"; - - private static final TableMetadata AutoRepairHistory = - parse(AUTO_REPAIR_HISTORY, - "Auto repair history for each node", - "CREATE TABLE %s (" - + "host_id uuid," - + "repair_type text," - + "repair_turn text," - + "repair_start_ts timestamp," - + "repair_finish_ts timestamp," - + "delete_hosts set," - + "delete_hosts_update_time timestamp," - + "force_repair boolean," - + "PRIMARY KEY (repair_type, host_id))"); - - private static final TableMetadata AutoRepairPriority = - parse(AUTO_REPAIR_PRIORITY, - "Auto repair priority for each group", - "CREATE TABLE %s (" - + "repair_type text," - + "repair_priority set," - + "PRIMARY KEY (repair_type))"); - - private static TableMetadata parse(String name, String description, String schema) - { - return CreateTableStatement.parse(String.format(schema, name), SchemaConstants.DISTRIBUTED_KEYSPACE_NAME) - .id(TableId.forSystemTable(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, name)) - .comment(description) - .gcGraceSeconds((int) TimeUnit.DAYS.toSeconds(90)) - .build(); - } - - public static KeyspaceMetadata metadata() - { - Tables tables = Tables.of(AutoRepairHistory, AutoRepairPriority); - return KeyspaceMetadata.create(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, KeyspaceParams.simple(1), tables); - } -} diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index fa2c0c91ac12..b10193e9b91a 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -53,6 +53,7 @@ import org.apache.cassandra.locator.NetworkTopologyStrategy; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.ViewMetadata; import org.apache.cassandra.serializers.SetSerializer; @@ -93,56 +94,56 @@ public class AutoRepairUtils final static String SELECT_REPAIR_HISTORY = String.format( "SELECT * FROM %s.%s WHERE %s = ?", SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, - AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE); + SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE); final static String SELECT_REPAIR_PRIORITY = String.format( "SELECT * FROM %s.%s WHERE %s = ?", SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, - AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_TYPE); + SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_TYPE); final static String DEL_REPAIR_PRIORITY = String.format( "DELETE %s[?] FROM %s.%s WHERE %s = ?", COL_REPAIR_PRIORITY, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, - AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_TYPE); + SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_TYPE); final static String ADD_PRIORITY_HOST = String.format( "UPDATE %s.%s SET %s = %s + ? WHERE %s = ?", SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, - AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_PRIORITY, COL_REPAIR_PRIORITY, COL_REPAIR_TYPE); + SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_PRIORITY, COL_REPAIR_PRIORITY, COL_REPAIR_TYPE); final static String INSERT_NEW_REPAIR_HISTORY = String.format( "INSERT INTO %s.%s (%s, %s, %s, %s, %s, %s) values (?, ? ,?, ?, {}, ?) IF NOT EXISTS", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, COL_HOST_ID, COL_REPAIR_START_TS, COL_REPAIR_FINISH_TS, COL_DELETE_HOSTS, COL_DELETE_HOSTS_UPDATE_TIME); final static String ADD_HOST_ID_TO_DELETE_HOSTS = String.format( "UPDATE %s.%s SET %s = %s + ?, %s = ? WHERE %s = ? AND %s = ? IF EXISTS" - , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_DELETE_HOSTS, + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_DELETE_HOSTS, COL_DELETE_HOSTS, COL_DELETE_HOSTS_UPDATE_TIME, COL_REPAIR_TYPE, COL_HOST_ID); final static String DEL_AUTO_REPAIR_HISTORY = String.format( "DELETE FROM %s.%s WHERE %s = ? AND %s = ?" - , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, COL_HOST_ID); final static String RECORD_START_REPAIR_HISTORY = String.format( "UPDATE %s.%s SET %s= ?, repair_turn = ? WHERE %s = ? AND %s = ?" - , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_START_TS, + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_START_TS, COL_REPAIR_TYPE, COL_HOST_ID); final static String RECORD_FINISH_REPAIR_HISTORY = String.format( "UPDATE %s.%s SET %s= ?, %s=false WHERE %s = ? AND %s = ?" - , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_FINISH_TS, + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_FINISH_TS, COL_FORCE_REPAIR, COL_REPAIR_TYPE, COL_HOST_ID); final static String CLEAR_DELETE_HOSTS = String.format( "UPDATE %s.%s SET %s= {} WHERE %s = ? AND %s = ?" - , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_DELETE_HOSTS, + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_DELETE_HOSTS, COL_REPAIR_TYPE, COL_HOST_ID); final static String SET_FORCE_REPAIR = String.format( "UPDATE %s.%s SET %s=true WHERE %s = ? AND %s = ?" - , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_FORCE_REPAIR, + , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_FORCE_REPAIR, COL_REPAIR_TYPE, COL_HOST_ID); final static String SELECT_LAST_REPAIR_TIME_FOR_NODE = String.format( "SELECT %s FROM %s.%s WHERE %s = ? AND %s = ?", COL_REPAIR_FINISH_TS, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, - AutoRepairKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, COL_HOST_ID); + SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, COL_HOST_ID); static ModificationStatement delStatementRepairHistory; static SelectStatement selectStatementRepairHistory; diff --git a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java index d63bbace79cf..02cb9f228e62 100644 --- a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java @@ -82,8 +82,9 @@ private SystemDistributedKeyspace() * gen 4: compression chunk length reduced to 16KiB, memtable_flush_period_in_ms now unset on all tables in 4.0 * gen 5: add ttl and TWCS to repair_history tables * gen 6: add denylist table + * gen 7: add AutoRepair related tables */ - public static final long GENERATION = 6; + public static final long GENERATION = 7; public static final String REPAIR_HISTORY = "repair_history"; @@ -93,6 +94,11 @@ private SystemDistributedKeyspace() public static final String PARTITION_DENYLIST_TABLE = "partition_denylist"; + public static final String AUTO_REPAIR_HISTORY = "auto_repair_history"; + + public static final String AUTO_REPAIR_PRIORITY = "auto_repair_priority"; + + private static final TableMetadata RepairHistory = parse(REPAIR_HISTORY, "Repair history", @@ -158,6 +164,28 @@ private SystemDistributedKeyspace() + "PRIMARY KEY ((ks_name, table_name), key))") .build(); + public static final TableMetadata AutoRepairHistory = + parse(AUTO_REPAIR_HISTORY, + "Auto repair history for each node", + "CREATE TABLE %s (" + + "host_id uuid," + + "repair_type text," + + "repair_turn text," + + "repair_start_ts timestamp," + + "repair_finish_ts timestamp," + + "delete_hosts set," + + "delete_hosts_update_time timestamp," + + "force_repair boolean," + + "PRIMARY KEY (repair_type, host_id))").build(); + public static final TableMetadata AutoRepairPriority = + parse(AUTO_REPAIR_PRIORITY, + "Auto repair priority for each group", + "CREATE TABLE %s (" + + "repair_type text," + + "repair_priority set," + + "PRIMARY KEY (repair_type))").build(); + + private static TableMetadata.Builder parse(String table, String description, String cql) { return CreateTableStatement.parse(format(cql, table), SchemaConstants.DISTRIBUTED_KEYSPACE_NAME) @@ -167,7 +195,7 @@ private static TableMetadata.Builder parse(String table, String description, Str public static KeyspaceMetadata metadata() { - return KeyspaceMetadata.create(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, KeyspaceParams.simple(Math.max(DEFAULT_RF, DatabaseDescriptor.getDefaultKeyspaceRF())), Tables.of(RepairHistory, ParentRepairHistory, ViewBuildStatus, PartitionDenylistTable)); + return KeyspaceMetadata.create(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, KeyspaceParams.simple(Math.max(DEFAULT_RF, DatabaseDescriptor.getDefaultKeyspaceRF())), Tables.of(RepairHistory, ParentRepairHistory, ViewBuildStatus, PartitionDenylistTable, AutoRepairHistory, AutoRepairPriority)); } public static void startParentRepair(TimeUUID parent_id, String keyspaceName, String[] cfnames, RepairOption options) diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index c4f79915c481..90df7854f513 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -87,7 +87,6 @@ import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; -import org.apache.cassandra.repair.autorepair.AutoRepairKeyspace; import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -1377,10 +1376,6 @@ public void setUpDistributedSystemKeyspaces() Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(TraceKeyspace.metadata(), TraceKeyspace.GENERATION)); Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(SystemDistributedKeyspace.metadata(), SystemDistributedKeyspace.GENERATION)); Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(AuthKeyspace.metadata(), AuthKeyspace.GENERATION)); - if (DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) - { - Schema.instance.transform(SchemaTransformations.updateSystemKeyspace(AutoRepairKeyspace.metadata(), AutoRepairKeyspace.GENERATION)); - } } public boolean isJoined() diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index d6e2c06602fd..6321523f7fe2 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -36,7 +36,7 @@ import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; -import org.apache.cassandra.repair.autorepair.AutoRepairKeyspace; +import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.service.AutoRepairService; import static org.apache.cassandra.schema.SchemaConstants.DISTRIBUTED_KEYSPACE_NAME; @@ -95,7 +95,7 @@ public static void afterClass() public void testScheduler() throws ParseException { // ensure there was no history of previous repair runs through the scheduler - Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s", DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY), ConsistencyLevel.QUORUM); + Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s", DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY), ConsistencyLevel.QUORUM); assertEquals(0, rows.length); cluster.forEach(i -> i.runOnInstance(() -> { @@ -118,7 +118,7 @@ public void testScheduler() throws ParseException private void validate(String repairType) throws ParseException { - Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s where repair_type='%s'", DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, repairType), ConsistencyLevel.QUORUM); + Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s where repair_type='%s'", DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType), ConsistencyLevel.QUORUM); assertEquals(3, rows.length); for (int node = 0; node < rows.length; node++) { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java index 43b8e4a5f4e8..f7ca33077661 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java @@ -18,26 +18,27 @@ package org.apache.cassandra.repair.autorepair; -import java.util.Optional; +import java.util.HashSet; +import java.util.Iterator; import java.util.Set; import com.google.common.collect.ImmutableSet; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableMetadata; -import static org.apache.cassandra.Util.setAutoRepairEnabled; - public class AutoRepairKeyspaceTest { private static final Set tables = ImmutableSet.of( - AutoRepairKeyspace.AUTO_REPAIR_HISTORY, - AutoRepairKeyspace.AUTO_REPAIR_PRIORITY + SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, + SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY ); @BeforeClass @@ -49,18 +50,17 @@ public static void setupDatabaseDescriptor() @Test - public void testMetadataCanParseSchemas() throws Exception + public void testEnsureAutoRepairTablesArePresent() { - setAutoRepairEnabled(true); - KeyspaceMetadata keyspaceMetadata = AutoRepairKeyspace.metadata(); - - assert keyspaceMetadata.tables.size() == tables.size() : "Expected " + tables.size() + " tables, got " + keyspaceMetadata.tables.size(); - - for (String table : tables) + KeyspaceMetadata keyspaceMetadata = SystemDistributedKeyspace.metadata(); + Iterator iter = keyspaceMetadata.tables.iterator(); + Set actualDistributedTablesIter = new HashSet<>(); + while (iter.hasNext()) { - Optional tableMetadata = keyspaceMetadata.tables.get(table); - - assert tableMetadata.isPresent() : "Table " + table + " not found in metadata"; + actualDistributedTablesIter.add(iter.next().name); } + + Assert.assertTrue(actualDistributedTablesIter.contains(SystemDistributedKeyspace.AUTO_REPAIR_HISTORY)); + Assert.assertTrue(actualDistributedTablesIter.contains(SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY)); } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 797fd1a9b05d..2a5669685d78 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -38,6 +38,7 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.schema.AutoRepairParams; +import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableParams; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.Pair; @@ -170,8 +171,8 @@ public void setup() Keyspace.open(KEYSPACE).getColumnFamilyStore(MV).truncateBlocking(); Keyspace.open(KEYSPACE).getColumnFamilyStore(MV).disableAutoCompaction(); - Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(AutoRepairKeyspace.AUTO_REPAIR_PRIORITY).truncateBlocking(); - Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(AutoRepairKeyspace.AUTO_REPAIR_HISTORY).truncateBlocking(); + Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY).truncateBlocking(); + Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(SystemDistributedKeyspace.AUTO_REPAIR_HISTORY).truncateBlocking(); AutoRepair.instance = new AutoRepair(); @@ -224,7 +225,7 @@ private void executeCQL() QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, s) VALUES ('k', 's')"); QueryProcessor.executeInternal("SELECT s FROM ks.tbl WHERE k='k'"); Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME) - .getColumnFamilyStore(AutoRepairKeyspace.AUTO_REPAIR_PRIORITY) + .getColumnFamilyStore(SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY) .forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index 09685c5f5110..afdbe0a6d825 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -47,6 +47,7 @@ import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.FBUtilities; import org.mockito.Mock; @@ -105,10 +106,10 @@ public void setup() DatabaseDescriptor.setEndpointSnitch(defaultSnitch); QueryProcessor.executeInternal(String.format( "TRUNCATE %s.%s", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY)); + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY)); QueryProcessor.executeInternal(String.format( "TRUNCATE %s.%s", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY)); + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY)); } @Test @@ -116,14 +117,14 @@ public void testSetForceRepair() { QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id, force_repair) VALUES ('%s', %s, false)", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); AutoRepairUtils.setForceRepair(repairType, ImmutableSet.of(localEndpoint)); UntypedResultSet result = QueryProcessor.executeInternal(String.format( "SELECT force_repair FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); assertNotNull(result); assertEquals(1, result.size()); @@ -137,7 +138,7 @@ public void testSetForceRepairNewNode() UntypedResultSet result = QueryProcessor.executeInternal(String.format( "SELECT force_repair FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); assertNotNull(result); assertEquals(1, result.size()); @@ -150,14 +151,14 @@ public void testClearDeleteHosts() { QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id, delete_hosts, delete_hosts_update_time) VALUES ('%s', %s, { %s }, toTimestamp(now()))", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId, hostId)); AutoRepairUtils.clearDeleteHosts(repairType, hostId); UntypedResultSet result = QueryProcessor.executeInternal(String.format( "SELECT delete_hosts FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); assertNotNull(result); assertEquals(1, result.size()); @@ -170,7 +171,7 @@ public void testGetAutoRepairHistoryForLocalGroup() { QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id, force_repair) VALUES ('%s', %s, false)", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); List history = AutoRepairUtils.getAutoRepairHistory(repairType); @@ -194,19 +195,19 @@ public void testGetCurrentRepairStatus() UUID regularRepair = UUID.randomUUID(); QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id, force_repair, repair_start_ts) VALUES ('%s', %s, true, toTimestamp(now()))", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), forceRepair)); QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id, repair_start_ts) VALUES ('%s', %s, toTimestamp(now()))", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), regularRepair)); QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, repair_priority) VALUES ('%s', { %s })", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, repairType.toString(), regularRepair)); CurrentRepairStatus status = AutoRepairUtils.getCurrentRepairStatus(repairType); @@ -256,11 +257,11 @@ public void testGetHostWithLongestUnrepairTime() UUID otherHostId = UUID.randomUUID(); QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id, repair_finish_ts) VALUES ('%s', %s, toTimestamp(now()))", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), otherHostId)); AutoRepairHistory history = AutoRepairUtils.getHostWithLongestUnrepairTime(repairType); @@ -296,14 +297,14 @@ public void testDeleteAutoRepairHistory() { QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); AutoRepairUtils.deleteAutoRepairHistory(repairType, hostId); UntypedResultSet result = QueryProcessor.executeInternal(String.format( "SELECT * FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); assertNotNull(result); assertEquals(0, result.size()); @@ -314,14 +315,14 @@ public void testUpdateStartAutoRepairHistory() { QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); AutoRepairUtils.updateStartAutoRepairHistory(repairType, hostId, 123, AutoRepairUtils.RepairTurn.MY_TURN); UntypedResultSet result = QueryProcessor.executeInternal(String.format( "SELECT repair_start_ts, repair_turn FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); assertNotNull(result); assertEquals(1, result.size()); @@ -335,14 +336,14 @@ public void testUpdateFinishAutoRepairHistory() { QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); AutoRepairUtils.updateFinishAutoRepairHistory(repairType, hostId, 123); UntypedResultSet result = QueryProcessor.executeInternal(String.format( "SELECT repair_finish_ts FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), hostId)); assertNotNull(result); assertEquals(1, result.size()); @@ -355,14 +356,14 @@ public void testAddHostIdToDeleteHosts() UUID otherHostId = UUID.randomUUID(); QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, host_id) VALUES ('%s', %s)", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), otherHostId)); AutoRepairUtils.addHostIdToDeleteHosts(repairType, hostId, otherHostId); UntypedResultSet result = QueryProcessor.executeInternal(String.format( "SELECT * FROM %s.%s WHERE repair_type = '%s' AND host_id = %s", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType.toString(), otherHostId)); assertNotNull(result); assertEquals(1, result.size()); @@ -379,7 +380,7 @@ public void testAddPriorityHost() UntypedResultSet result = QueryProcessor.executeInternal(String.format( "SELECT * FROM %s.%s WHERE repair_type = '%s'", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, repairType.toString())); assertNotNull(result); assertEquals(1, result.size()); @@ -394,14 +395,14 @@ public void testRemovePriorityStatus() { QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, repair_priority) VALUES ('%s', { %s })", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, repairType.toString(), hostId)); AutoRepairUtils.removePriorityStatus(repairType, hostId); UntypedResultSet result = QueryProcessor.executeInternal(String.format( "SELECT * FROM %s.%s WHERE repair_type = '%s'", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, repairType.toString())); assertNotNull(result); assertEquals(1, result.size()); @@ -414,7 +415,7 @@ public void testGetPriorityHosts() { QueryProcessor.executeInternal(String.format( "INSERT INTO %s.%s (repair_type, repair_priority) VALUES ('%s', { %s })", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY, + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, repairType.toString(), hostId)); Set hosts = AutoRepairUtils.getPriorityHosts(repairType); diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index d3befd024f21..874ec75862e9 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -44,9 +44,9 @@ import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; -import org.apache.cassandra.repair.autorepair.AutoRepairKeyspace; import org.apache.cassandra.repair.autorepair.AutoRepairUtils; import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SystemDistributedKeyspace; import static org.apache.cassandra.Util.setAutoRepairEnabled; import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; @@ -256,7 +256,7 @@ private static Set isLocalHostForceRepair(AutoRepairConfig.R UUID hostId = Gossiper.instance.getHostId(InetAddressAndPort.getLocalHost()); UntypedResultSet resultSet = QueryProcessor.executeInternal(String.format( "SELECT force_repair FROM %s.%s WHERE host_id = %s and repair_type = '%s'", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY, hostId, type)); + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, hostId, type)); if (!resultSet.isEmpty() && resultSet.one().getBoolean("force_repair")) { @@ -292,10 +292,10 @@ public void prepare() { QueryProcessor.executeInternal(String.format( "TRUNCATE %s.%s", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_HISTORY)); + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY)); QueryProcessor.executeInternal(String.format( "TRUNCATE %s.%s", - SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, AutoRepairKeyspace.AUTO_REPAIR_PRIORITY)); + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY)); } @Test From 37cb56283b8e5a648d341dddaf2f5ec6d2d6e8a3 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 9 Oct 2024 14:47:56 -0700 Subject: [PATCH 039/199] Comments from Chris Lohfink: nix automated_ and a typo --- pylib/cqlshlib/test/test_cqlsh_output.py | 4 ++-- .../cql3/statements/schema/TableAttributes.java | 8 ++++---- .../apache/cassandra/schema/SchemaKeyspace.java | 16 ++++++++-------- .../org/apache/cassandra/schema/TableParams.java | 12 ++++++------ .../tools/nodetool/GetAutoRepairConfig.java | 2 +- .../cql3/statements/DescribeStatementTest.java | 8 ++++---- .../apache/cassandra/db/SchemaCQLHelperTest.java | 4 ++-- .../autorepair/AutoRepairParameterizedTest.java | 4 ++-- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index 500d0f46c5b4..3cb3482d100a 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -691,8 +691,8 @@ def test_describe_columnfamily_output(self): AND min_index_interval = 128 AND read_repair = 'BLOCKING' AND speculative_retry = '99p' - AND automated_repair_full = {'enabled': 'true'} - AND automated_repair_incremental = {'enabled': 'true'};""" % quote_name(get_keyspace())) + AND repair_full = {'enabled': 'true'} + AND repair_incremental = {'enabled': 'true'};""" % quote_name(get_keyspace())) with testrun_cqlsh(tty=True, env=self.default_env) as c: for cmdword in ('describe table', 'desc columnfamily'): diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index 1e290869bd73..baa9849707dd 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -157,11 +157,11 @@ private TableParams build(TableParams.Builder builder) if (hasOption(Option.READ_REPAIR)) builder.readRepair(ReadRepairStrategy.fromString(getString(Option.READ_REPAIR))); - if (hasOption(Option.AUTOMATED_REPAIR_FULL)) - builder.automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.full, getMap(Option.AUTOMATED_REPAIR_FULL))); + if (hasOption(Option.REPAIR_FULL)) + builder.automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.full, getMap(Option.REPAIR_FULL))); - if (hasOption(Option.AUTOMATED_REPAIR_INCREMENTAL)) - builder.automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.incremental, getMap(Option.AUTOMATED_REPAIR_INCREMENTAL))); + if (hasOption(Option.REPAIR_INCREMENTAL)) + builder.automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.incremental, getMap(Option.REPAIR_INCREMENTAL))); return builder.build(); } diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index 5cd08901d93d..b5c908a18436 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -122,8 +122,8 @@ private SchemaKeyspace() + "additional_write_policy text," + "cdc boolean," + "read_repair text," - + "automated_repair_full frozen>," - + "automated_repair_incremental frozen>," + + "repair_full frozen>," + + "repair_incremental frozen>," + "PRIMARY KEY ((keyspace_name), table_name))"); private static final TableMetadata Columns = @@ -192,8 +192,8 @@ private SchemaKeyspace() + "additional_write_policy text," + "cdc boolean," + "read_repair text," - + "automated_repair_full frozen>," - + "automated_repair_incremental frozen>," + + "repair_full frozen>," + + "repair_incremental frozen>," + "PRIMARY KEY ((keyspace_name), view_name))"); private static final TableMetadata Indexes = @@ -558,8 +558,8 @@ private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBui .add("compression", params.compression.asMap()) .add("read_repair", params.readRepair.toString()) .add("extensions", params.extensions) - .add("automated_repair_full", params.automatedRepair.get(AutoRepairConfig.RepairType.full).asMap()) - .add("automated_repair_incremental", params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()); + .add("repair_full", params.automatedRepair.get(AutoRepairConfig.RepairType.full).asMap()) + .add("repair_incremental", params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()); // Only add CDC-enabled flag to schema if it's enabled on the node. This is to work around RTE's post-8099 if a 3.8+ @@ -984,8 +984,8 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row) SpeculativeRetryPolicy.fromString("99PERCENTILE")) .cdc(row.has("cdc") && row.getBoolean("cdc")) .readRepair(getReadRepairStrategy(row)) - .automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.full, row.getFrozenTextMap("automated_repair_full"))) - .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.incremental, row.getFrozenTextMap("automated_repair_incremental"))) + .automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.full, row.getFrozenTextMap("repair_full"))) + .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.incremental, row.getFrozenTextMap("repair_incremental"))) .build(); } diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index fa9145df8ead..1132a9ae461a 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -60,8 +60,8 @@ public enum Option CRC_CHECK_CHANCE, CDC, READ_REPAIR, - AUTOMATED_REPAIR_FULL, - AUTOMATED_REPAIR_INCREMENTAL, + REPAIR_FULL, + REPAIR_INCREMENTAL, ; @Override @@ -286,8 +286,8 @@ public String toString() .add(Option.EXTENSIONS.toString(), extensions) .add(Option.CDC.toString(), cdc) .add(Option.READ_REPAIR.toString(), readRepair) - .add(Option.AUTOMATED_REPAIR_FULL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.full)) - .add(Option.AUTOMATED_REPAIR_INCREMENTAL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.incremental)) + .add(Option.REPAIR_FULL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.full)) + .add(Option.REPAIR_INCREMENTAL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.incremental)) .toString(); } @@ -337,9 +337,9 @@ public void appendCqlTo(CqlBuilder builder, boolean isView) .newLine() .append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString()) .newLine() - .append("AND automated_repair_full = ").append(automatedRepair.get(AutoRepairConfig.RepairType.full).asMap()) + .append("AND repair_full = ").append(automatedRepair.get(AutoRepairConfig.RepairType.full).asMap()) .newLine() - .append("AND automated_repair_incremental = ").append(automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()); + .append("AND repair_incremental = ").append(automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()); } public static final class Builder diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java index c62063b13b7f..5d0d53819ea1 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java @@ -74,7 +74,7 @@ private String formatRepairTypeConfig(NodeProbe probe, RepairType repairType, Au sb.append("\n\tpercentage of parallel repairs within group: " + config.getParallelRepairPercentage(repairType)); sb.append("\n\tmv repair enabled: " + config.getMVRepairEnabled(repairType)); sb.append("\n\tinitial scheduler delay: " + config.getInitialSchedulerDelay(repairType)); - sb.append("\n\trepair setssion timeout: " + config.getRepairSessionTimeout(repairType)); + sb.append("\n\trepair setsession timeout: " + config.getRepairSessionTimeout(repairType)); return sb.toString(); } diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index 4d2c1b105058..2c910b321c07 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -1005,8 +1005,8 @@ private static String tableParametersCql() " AND min_index_interval = 128\n" + " AND read_repair = 'BLOCKING'\n" + " AND speculative_retry = '99p'\n" + - " AND automated_repair_full = {'enabled': 'true'}\n" + - " AND automated_repair_incremental = {'enabled': 'true'};"; + " AND repair_full = {'enabled': 'true'}\n" + + " AND repair_incremental = {'enabled': 'true'};"; } private static String mvParametersCql() @@ -1027,8 +1027,8 @@ private static String mvParametersCql() " AND min_index_interval = 128\n" + " AND read_repair = 'BLOCKING'\n" + " AND speculative_retry = '99p'\n" + - " AND automated_repair_full = {'enabled': 'true'}\n" + - " AND automated_repair_incremental = {'enabled': 'true'};"; + " AND repair_full = {'enabled': 'true'}\n" + + " AND repair_incremental = {'enabled': 'true'};"; } private static String keyspaceOutput() diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java index 39b2449a0d66..152b4112a3c0 100644 --- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java @@ -313,8 +313,8 @@ public void testCfmOptionsCQL() " AND min_index_interval = 6\n" + " AND read_repair = 'BLOCKING'\n" + " AND speculative_retry = 'ALWAYS'\n" + - " AND automated_repair_full = {'enabled': 'true'}\n" + - " AND automated_repair_incremental = {'enabled': 'true'};" + " AND repair_full = {'enabled': 'true'}\n" + + " AND repair_incremental = {'enabled': 'true'};" )); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 2a5669685d78..3cd57f68b079 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -586,8 +586,8 @@ public void testTokenRangesSplit() @Test public void testTableAttribute() { - assertTrue(TableAttributes.validKeywords().contains("automated_repair_full")); - assertTrue(TableAttributes.validKeywords().contains("automated_repair_incremental")); + assertTrue(TableAttributes.validKeywords().contains("repair_full")); + assertTrue(TableAttributes.validKeywords().contains("repair_incremental")); } @Test From 8e965b6f8c9483d09c320f532537502eb522415d Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 9 Oct 2024 16:09:57 -0700 Subject: [PATCH 040/199] Comments from Chris Lohfink: System::currentTimeMillis --> Clock.Global::currentTimeMillis --- .../org/apache/cassandra/repair/autorepair/AutoRepair.java | 3 ++- .../apache/cassandra/repair/autorepair/AutoRepairState.java | 3 ++- .../apache/cassandra/tools/nodetool/GetAutoRepairConfig.java | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 630e7185425b..709299def1e9 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -38,6 +38,7 @@ import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.RepairRunnable; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.Pair; import org.slf4j.Logger; @@ -67,7 +68,7 @@ public class AutoRepair private static final Logger logger = LoggerFactory.getLogger(AutoRepair.class); @VisibleForTesting - protected static Supplier timeFunc = System::currentTimeMillis; + protected static Supplier timeFunc = Clock.Global::currentTimeMillis; public static AutoRepair instance = new AutoRepair(); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index a4869c77425b..3653bae2c018 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -36,6 +36,7 @@ import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.concurrent.Condition; import org.apache.cassandra.utils.progress.ProgressEvent; import org.apache.cassandra.utils.progress.ProgressEventType; @@ -60,7 +61,7 @@ public abstract class AutoRepairState implements ProgressListener protected static final Logger logger = LoggerFactory.getLogger(AutoRepairState.class); private final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); @VisibleForTesting - protected static Supplier timeFunc = System::currentTimeMillis; + protected static Supplier timeFunc = Clock.Global::currentTimeMillis; @VisibleForTesting protected final RepairType repairType; diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java index 5d0d53819ea1..48f8d54de3f6 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java @@ -74,7 +74,7 @@ private String formatRepairTypeConfig(NodeProbe probe, RepairType repairType, Au sb.append("\n\tpercentage of parallel repairs within group: " + config.getParallelRepairPercentage(repairType)); sb.append("\n\tmv repair enabled: " + config.getMVRepairEnabled(repairType)); sb.append("\n\tinitial scheduler delay: " + config.getInitialSchedulerDelay(repairType)); - sb.append("\n\trepair setsession timeout: " + config.getRepairSessionTimeout(repairType)); + sb.append("\n\trepair session timeout: " + config.getRepairSessionTimeout(repairType)); return sb.toString(); } From e6c8937864f92d1ef29042341d8dfe3fdc4e2291 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Fri, 11 Oct 2024 16:03:32 -0700 Subject: [PATCH 041/199] Implement preview repaired auto-repair type --- .../repair/autorepair/AutoRepairConfig.java | 3 ++- .../repair/autorepair/AutoRepairState.java | 20 +++++++++++++++++++ .../AutoRepairStateFactoryTest.java | 4 ++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index ac9394d9cfe1..6fc669da642f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -57,7 +57,8 @@ public class AutoRepairConfig implements Serializable public enum RepairType implements Serializable { full, - incremental; + incremental, + previewRepaired; public static AutoRepairState getAutoRepairState(RepairType repairType) { diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index 3653bae2c018..aace983e968a 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -286,6 +286,26 @@ public void resetWaitCondition() } } +class PreviewRepairedState extends AutoRepairState +{ + public PreviewRepairedState() + { + super(RepairType.preview); + } + + @Override + public RepairRunnable getRepairRunnable(String keyspace, List tables, Set> ranges, boolean primaryRangeOnly) + { + RepairOption option = new RepairOption(RepairParallelism.PARALLEL, primaryRangeOnly, false, false, + AutoRepairService.instance.getAutoRepairConfig().getRepairThreads(repairType), ranges, + !ranges.isEmpty(), false, false, PreviewKind.REPAIRED, false, true, false, false); + + option.getColumnFamilies().addAll(tables); + + return getRepairRunnable(keyspace, option); + } +} + class IncrementalRepairState extends AutoRepairState { public IncrementalRepairState() diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java index a0e5bdc45294..3d8b3e7609ad 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java @@ -36,6 +36,10 @@ public void testGetRepairState() { state = RepairType.getAutoRepairState(RepairType.incremental); assert state instanceof IncrementalRepairState; + + state = RepairType.getAutoRepairState(RepairType.previewRepaired); + + assert state instanceof PreviewRepairedState; } @Test From 65192931166032b8904ee26b025298778e41a328 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Wed, 25 Sep 2024 18:05:30 -0500 Subject: [PATCH 042/199] Update IAutoRepairTokenRangeSplitter API to be more flexible Updates IAutoRepairTokenRangeSplitter API to take in a set of tables instead of a single table and return "RepairAssignments". This allows the API more control over how the repairs are issued by the AutoRepair framework, by allowing grouping of repairs by specific tables or a single repair at the keyspace level. Also updates IAutoRepairTokenRangeSplitter to be a ParameterizedClass, so custom splitters can have their own configuration. Update AutoRepair to account for these changes. Update DefaultAutoRepairTokenSplitter to account for the change, and to also evaluate on getRepairByKeyspace option. This also improves the filtering of tables that should not be repaired where previously getRepairByKeyspace would not factor this in. Move logic to split token ranges evenly into AutoRepairUtils so it can be reutilized by other splitters. --- .../cassandra/config/DatabaseDescriptor.java | 6 + .../cassandra/config/ParameterizedClass.java | 3 +- .../cassandra/metrics/AutoRepairMetrics.java | 11 +- .../repair/autorepair/AutoRepair.java | 265 +++++++++--------- .../repair/autorepair/AutoRepairConfig.java | 10 +- .../repair/autorepair/AutoRepairState.java | 14 + .../repair/autorepair/AutoRepairUtils.java | 30 ++ .../DefaultAutoRepairTokenSplitter.java | 65 ++--- .../IAutoRepairTokenRangeSplitter.java | 76 ++++- .../schema/SystemDistributedKeyspace.java | 2 +- .../apache/cassandra/utils/FBUtilities.java | 29 +- .../autorepair/AutoRepairConfigTest.java | 8 +- ...DefaultTokenSplitterParameterizedTest.java | 157 +++++++++++ .../AutoRepairParameterizedTest.java | 47 ++-- 14 files changed, 506 insertions(+), 217 deletions(-) create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 94e7771cc4eb..d7663b2aa1a7 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -4531,4 +4531,10 @@ public static void setIncrementalRepairDiskHeadroomRejectRatio(double value) { conf.incremental_repair_disk_headroom_reject_ratio = value; } + + @VisibleForTesting + public static void setInitialTokens(String initial_token) + { + conf.initial_token = initial_token; + } } diff --git a/src/java/org/apache/cassandra/config/ParameterizedClass.java b/src/java/org/apache/cassandra/config/ParameterizedClass.java index 9b001786e35b..a101caa92ca5 100644 --- a/src/java/org/apache/cassandra/config/ParameterizedClass.java +++ b/src/java/org/apache/cassandra/config/ParameterizedClass.java @@ -17,6 +17,7 @@ */ package org.apache.cassandra.config; +import java.io.Serializable; import java.util.List; import java.util.Map; @@ -27,7 +28,7 @@ import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; @Shared(scope = SIMULATION) -public class ParameterizedClass +public class ParameterizedClass implements Serializable { public static final String CLASS_NAME = "class_name"; public static final String PARAMETERS = "parameters"; diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java index d765899631b7..8da0ba3523d5 100644 --- a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java @@ -38,7 +38,7 @@ public class AutoRepairMetrics public Gauge succeededTokenRangesCount; public Gauge failedTokenRangesCount; public Gauge skippedTokenRangesCount; - + public Gauge skippedTablesCount; public Counter repairTurnMyTurn; public Counter repairTurnMyTurnDueToPriority; public Counter repairTurnMyTurnForceRepair; @@ -81,6 +81,15 @@ public Integer getValue() } }); + skippedTablesCount = Metrics.register(factory.createMetricName("SkippedTablesCount"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).getSkippedTablesCount(); + } + }); + + longestUnrepairedSec = Metrics.register(factory.createMetricName("LongestUnrepairedSec"), new Gauge() { public Integer getValue() diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 709299def1e9..1f20e8af00a9 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -32,14 +32,14 @@ import java.util.function.Supplier; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.Uninterruptibles; import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.utils.Clock; -import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter.RepairAssignment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -128,10 +128,10 @@ public void setup() AutoRepairService.instance.checkCanRun(repairType); repairExecutors.get(repairType).scheduleWithFixedDelay( - () -> repair(repairType), - config.getInitialSchedulerDelay(repairType).toSeconds(), - config.getRepairCheckInterval().toSeconds(), - TimeUnit.SECONDS); + () -> repair(repairType), + config.getInitialSchedulerDelay(repairType).toSeconds(), + config.getRepairCheckInterval().toSeconds(), + TimeUnit.SECONDS); } isSetupDone = true; } @@ -197,9 +197,7 @@ public void repair(AutoRepairConfig.RepairType repairType) repairState.setTotalTablesConsideredForRepair(0); repairState.setTotalMVTablesConsideredForRepair(0); - int failedTokenRanges = 0; - int succeededTokenRanges = 0; - int skippedTokenRanges = 0; + CollectectedRepairStats collectectedRepairStats = new CollectectedRepairStats(); List keyspaces = new ArrayList<>(); Keyspace.all().forEach(keyspaces::add); @@ -215,149 +213,116 @@ public void repair(AutoRepairConfig.RepairType repairType) } repairState.setRepairKeyspaceCount(repairState.getRepairKeyspaceCount() + 1); - List tablesToBeRepaired = retrieveTablesToBeRepaired(keyspace, repairType, repairState); - shuffleFunc.accept(tablesToBeRepaired); - for (String tableName : tablesToBeRepaired) + List tablesToBeRepairedList = retrieveTablesToBeRepaired(keyspace, config, repairType, repairState, collectectedRepairStats); + shuffleFunc.accept(tablesToBeRepairedList); + String keyspaceName = keyspace.getName(); + List repairAssignments = tokenRangeSplitters.get(repairType).getRepairAssignments(repairType, primaryRangeOnly, keyspaceName, tablesToBeRepairedList); + + int totalRepairAssignments = repairAssignments.size(); + long keyspaceStartTime = timeFunc.get(); + RepairAssignment previousAssignment = null; + long tableStartTime = timeFunc.get(); + int totalProcessedAssignments = 0; + Set> ranges = new HashSet<>(); + for (RepairAssignment curRepairAssignment : repairAssignments) { - String keyspaceName = keyspace.getName(); try { - List> subRangesToBeRepaired = tokenRangeSplitters.get(repairType).getRange(repairType, primaryRangeOnly, keyspaceName, tableName); - int totalSubRanges = subRangesToBeRepaired.size(); - - ColumnFamilyStore columnFamilyStore = keyspace.getColumnFamilyStore(tableName); - if (!columnFamilyStore.metadata().params.automatedRepair.get(repairType).repairEnabled()) + totalProcessedAssignments++; + boolean repairOneTableAtATime = !config.getRepairByKeyspace(repairType); + if (previousAssignment != null && repairOneTableAtATime && !previousAssignment.tableNames.equals(curRepairAssignment.tableNames)) { - logger.info("Repair is disabled for keyspace {} for tables: {}", keyspaceName, tableName); - repairState.setTotalDisabledTablesRepairCount(repairState.getTotalDisabledTablesRepairCount() + 1); - continue; + // In the repair assignment, all the tables are appended sequnetially. + // Check if we have a different table, and if so, we should reset the table start time. + tableStartTime = timeFunc.get(); } - // this is done to make autorepair safe as running repair on table with more sstables - // may have its own challenges - int size = columnFamilyStore.getLiveSSTables().size(); - if (size > config.getRepairSSTableCountHigherThreshold(repairType)) + previousAssignment = curRepairAssignment; + if (!config.isAutoRepairEnabled(repairType)) { - logger.info("Too many SSTables for repair, not doing repair on table {}.{} " + - "totalSSTables {}", keyspaceName, tableName, columnFamilyStore.getLiveSSTables().size()); - skippedTokenRanges += totalSubRanges; - continue; + logger.error("Auto-repair for type {} is disabled hence not running repair", repairType); + repairState.setRepairInProgress(false); + return; } - - if (config.getRepairByKeyspace(repairType)) + if (AutoRepairUtils.keyspaceMaxRepairTimeExceeded(repairType, keyspaceStartTime, tablesToBeRepairedList.size())) { - logger.info("Repair keyspace {} for tables: {}", keyspaceName, tablesToBeRepaired); + collectectedRepairStats.skippedTokenRanges += totalRepairAssignments - totalProcessedAssignments; + logger.info("Keyspace took too much time to repair hence skipping it {}", + keyspaceName); + break; } - else + if (repairOneTableAtATime && AutoRepairUtils.tableMaxRepairTimeExceeded(repairType, tableStartTime)) { - logger.info("Repair table {}.{}", keyspaceName, tableName); + collectectedRepairStats.skippedTokenRanges += 1; + logger.info("Table took too much time to repair hence skipping it table name {}.{}, token range {}", + keyspaceName, curRepairAssignment.tableNames, curRepairAssignment.tokenRange); + continue; } - long tableStartTime = timeFunc.get(); - Set> ranges = new HashSet<>(); - int totalProcessedSubRanges = 0; - for (Pair token : subRangesToBeRepaired) - { - if (!config.isAutoRepairEnabled(repairType)) - { - logger.error("Auto-repair for type {} is disabled hence not running repair", repairType); - repairState.setRepairInProgress(false); - return; - } - if (config.getRepairByKeyspace(repairType)) + Range tokenRange = curRepairAssignment.getTokenRange(); + logger.debug("Current Token Left side {}, right side {}", + tokenRange.left.toString(), + tokenRange.right.toString()); + + ranges.add(curRepairAssignment.getTokenRange()); + if ((totalProcessedAssignments % config.getRepairThreads(repairType) == 0) || + (totalProcessedAssignments == totalRepairAssignments)) + { + int retryCount = 0; + Future f = null; + while (retryCount <= config.getRepairMaxRetries()) { - if (AutoRepairUtils.keyspaceMaxRepairTimeExceeded(repairType, tableStartTime, tablesToBeRepaired.size())) + RepairRunnable task = repairState.getRepairRunnable(keyspaceName, + Lists.newArrayList(curRepairAssignment.getTableNames()), + ranges, primaryRangeOnly); + repairState.resetWaitCondition(); + f = repairRunnableExecutors.get(repairType).submit(task); + try { - skippedTokenRanges += totalSubRanges - totalProcessedSubRanges; - logger.info("Keyspace took too much time to repair hence skipping it {}", - keyspaceName); - break; + repairState.waitForRepairToComplete(config.getRepairSessionTimeout(repairType)); } - } - else - { - if (AutoRepairUtils.tableMaxRepairTimeExceeded(repairType, tableStartTime)) + catch (InterruptedException e) { - skippedTokenRanges += totalSubRanges - totalProcessedSubRanges; - logger.info("Table took too much time to repair hence skipping it {}.{}", - keyspaceName, tableName); - break; + logger.error("Exception in cond await:", e); } - } - Token childStartToken = token.left; - Token childEndToken = token.right; - logger.debug("Current Token Left side {}, right side {}", childStartToken - .toString(), childEndToken.toString()); - - ranges.add(new Range<>(childStartToken, childEndToken)); - totalProcessedSubRanges++; - if ((totalProcessedSubRanges % config.getRepairThreads(repairType) == 0) || - (totalProcessedSubRanges == totalSubRanges)) - { - int retryCount = 0; - Future f = null; - while (retryCount <= config.getRepairMaxRetries()) + if (repairState.isSuccess()) { - RepairRunnable task = repairState.getRepairRunnable(keyspaceName, - config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : ImmutableList.of(tableName), - ranges, primaryRangeOnly); - repairState.resetWaitCondition(); - f = repairRunnableExecutors.get(repairType).submit(task); - try - { - repairState.waitForRepairToComplete(config.getRepairSessionTimeout(repairType)); - } - catch (InterruptedException e) - { - logger.error("Exception in cond await:", e); - } - if (repairState.isSuccess()) - { - break; - } - else if (retryCount < config.getRepairMaxRetries()) - { - boolean cancellationStatus = f.cancel(true); - logger.warn("Repair failed for range {}-{} for {}.{} with cancellationStatus: {} retrying after {} seconds...", - childStartToken, childEndToken, - keyspaceName, config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, - cancellationStatus, config.getRepairRetryBackoff().toSeconds()); - sleepFunc.accept(config.getRepairRetryBackoff().toSeconds(), TimeUnit.SECONDS); - } - retryCount++; + break; } - //check repair status - if (repairState.isSuccess()) + else if (retryCount < config.getRepairMaxRetries()) { - logger.info("Repair completed for range {}-{} for {}.{}, total subranges: {}," + - "processed subranges: {}", childStartToken, childEndToken, - keyspaceName, config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, totalSubRanges, totalProcessedSubRanges); - succeededTokenRanges += ranges.size(); + boolean cancellationStatus = f.cancel(true); + logger.warn("Repair failed for range {}-{} for {} tables {} with cancellationStatus: {} retrying after {} seconds...", + tokenRange.left, tokenRange.right, + keyspaceName, curRepairAssignment.getTableNames(), + cancellationStatus, config.getRepairRetryBackoff().toSeconds()); + sleepFunc.accept(config.getRepairRetryBackoff().toSeconds(), TimeUnit.SECONDS); } - else + retryCount++; + } + //check repair status + if (repairState.isSuccess()) + { + logger.info("Repair completed for range {}-{} for {} tables {}, total assignments: {}," + + "processed assignments: {}", tokenRange.left, tokenRange.right, + keyspaceName, curRepairAssignment.getTableNames(), totalRepairAssignments, totalProcessedAssignments); + collectectedRepairStats.succeededTokenRanges += ranges.size(); + } + else + { + boolean cancellationStatus = true; + if (f != null) { - boolean cancellationStatus = true; - if (f != null) - { - cancellationStatus = f.cancel(true); - } - //in the future we can add retry, etc. - logger.error("Repair failed for range {}-{} for {}.{} after {} retries, total subranges: {}," + - "processed subranges: {}, cancellationStatus: {}", childStartToken.toString(), childEndToken.toString(), keyspaceName, - config.getRepairByKeyspace(repairType) ? tablesToBeRepaired : tableName, retryCount, totalSubRanges, totalProcessedSubRanges, cancellationStatus); - failedTokenRanges += ranges.size(); + cancellationStatus = f.cancel(true); } - ranges.clear(); + //in the future we can add retry, etc. + logger.error("Repair failed for range {}-{} for {} tables {} after {} retries, total assignments: {}," + + "processed assignments: {}, cancellationStatus: {}", tokenRange.left, tokenRange.right, keyspaceName, + curRepairAssignment.getTableNames(), retryCount, totalRepairAssignments, totalProcessedAssignments, cancellationStatus); + collectectedRepairStats.failedTokenRanges += ranges.size(); } + ranges.clear(); } - if (config.getRepairByKeyspace(repairType)) - { - logger.info("Repair completed for keyspace {}, tables: {}", keyspaceName, tablesToBeRepaired); - break; - } - else - { - logger.info("Repair completed for {}.{}", keyspaceName, tableName); - } + logger.info("Repair completed for {} tables {}, range {}", keyspaceName, curRepairAssignment.getTableNames(), curRepairAssignment.getTokenRange()); } catch (Exception e) { @@ -365,7 +330,7 @@ else if (retryCount < config.getRepairMaxRetries()) } } } - cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, failedTokenRanges, succeededTokenRanges, skippedTokenRanges); + cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, collectectedRepairStats); } else { @@ -400,7 +365,7 @@ private boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoR return false; } - private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairConfig.RepairType repairType, AutoRepairState repairState) + private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairConfig config, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, CollectectedRepairStats collectectedRepairStats) { Tables tables = keyspace.getMetadata().tables; List tablesToBeRepaired = new ArrayList<>(); @@ -410,6 +375,27 @@ private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairCon repairState.setTotalTablesConsideredForRepair(repairState.getTotalTablesConsideredForRepair() + 1); TableMetadata tableMetadata = iter.next(); String tableName = tableMetadata.name; + + ColumnFamilyStore columnFamilyStore = keyspace.getColumnFamilyStore(tableName); + if (!columnFamilyStore.metadata().params.automatedRepair.get(repairType).repairEnabled()) + { + logger.info("Repair is disabled for keyspace {} for tables: {}", keyspace.getName(), tableName); + repairState.setTotalDisabledTablesRepairCount(repairState.getTotalDisabledTablesRepairCount() + 1); + collectectedRepairStats.skippedTables++; + continue; + } + + // this is done to make autorepair safe as running repair on table with more sstables + // may have its own challenges + int totalSSTables = columnFamilyStore.getLiveSSTables().size(); + if (totalSSTables > config.getRepairSSTableCountHigherThreshold(repairType)) + { + logger.info("Too many SSTables for repair for table {}.{}" + + "totalSSTables {}", keyspace.getName(), tableName, totalSSTables); + collectectedRepairStats.skippedTables++; + continue; + } + tablesToBeRepaired.add(tableName); // See if we should repair MVs as well that are associated with this given table @@ -424,7 +410,7 @@ private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairCon } private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, UUID myId, - long startTime, int failedTokenRanges, int succeededTokenRanges, int skippedTokenRanges) throws InterruptedException + long startTime, CollectectedRepairStats collectectedRepairStats) throws InterruptedException { //if it was due to priority then remove it now if (turn == MY_TURN_DUE_TO_PRIORITY) @@ -433,16 +419,17 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType AutoRepairUtils.removePriorityStatus(repairType, myId); } - repairState.setFailedTokenRangesCount(failedTokenRanges); - repairState.setSucceededTokenRangesCount(succeededTokenRanges); - repairState.setSkippedTokenRangesCount(skippedTokenRanges); + repairState.setFailedTokenRangesCount(collectectedRepairStats.failedTokenRanges); + repairState.setSucceededTokenRangesCount(collectectedRepairStats.succeededTokenRanges); + repairState.setSkippedTokenRangesCount(collectectedRepairStats.skippedTokenRanges); + repairState.setSkippedTablesCount(collectectedRepairStats.skippedTables); repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - startTime)); long timeInHours = TimeUnit.SECONDS.toHours(repairState.getNodeRepairTimeInSec()); logger.info("Local {} repair time {} hour(s), stats: repairKeyspaceCount {}, " + "repairTokenRangesSuccessCount {}, repairTokenRangesFailureCount {}, " + - "repairTokenRangesSkipCount {}", repairType, timeInHours, repairState.getRepairKeyspaceCount(), + "repairTokenRangesSkipCount {}, repairTablesSkipCount {}", repairType, timeInHours, repairState.getRepairKeyspaceCount(), repairState.getSucceededTokenRangesCount(), repairState.getFailedTokenRangesCount(), - repairState.getSkippedTokenRangesCount()); + repairState.getSkippedTokenRangesCount(), repairState.getSkippedTablesCount()); if (repairState.getLastRepairTime() != 0) { repairState.setClusterRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - @@ -466,4 +453,12 @@ public AutoRepairState getRepairState(AutoRepairConfig.RepairType repairType) { return repairStates.get(repairType); } + + static class CollectectedRepairStats + { + int failedTokenRanges = 0; + int succeededTokenRanges = 0; + int skippedTokenRanges = 0; + int skippedTables = 0; + } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index ac9394d9cfe1..30433a1ad7ff 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -20,6 +20,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumMap; import java.util.HashSet; import java.util.Map; @@ -31,6 +32,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.config.ParameterizedClass; public class AutoRepairConfig implements Serializable { @@ -274,7 +276,7 @@ public boolean getForceRepairNewNode(RepairType repairType) return applyOverrides(repairType, opt -> opt.force_repair_new_node); } - public String getTokenRangeSplitter(RepairType repairType) + public ParameterizedClass getTokenRangeSplitter(RepairType repairType) { return applyOverrides(repairType, opt -> opt.token_range_splitter); } @@ -331,7 +333,7 @@ protected static Options getDefaultOptions() opts.force_repair_new_node = false; opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h"); opts.mv_repair_enabled = false; - opts.token_range_splitter = DefaultAutoRepairTokenSplitter.class.getName(); + opts.token_range_splitter = new ParameterizedClass(DefaultAutoRepairTokenSplitter.class.getName(), Collections.emptyMap()); opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("5m"); // 5 minutes opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h"); // 3 hours @@ -395,9 +397,9 @@ protected static Options getDefaultOptions() // the default is 'true'. // This flag determines whether the auto-repair framework needs to run anti-entropy, a.k.a, repair on the MV table or not. public volatile Boolean mv_repair_enabled; - // the default is DefaultAutoRepairTokenSplitter.class.getName(). The class should implement IAutoRepairTokenRangeSplitter. + // the default is DefaultAutoRepairTokenSplitter. The class should implement IAutoRepairTokenRangeSplitter. // The default implementation splits the tokens based on the token ranges owned by this node divided by the number of 'number_of_subranges' - public volatile String token_range_splitter; + public volatile ParameterizedClass token_range_splitter; // the minimum delay after a node starts before the scheduler starts running repair public volatile DurationSpec.IntSecondsBound initial_scheduler_delay; // repair session timeout - this is applicable for each repair session diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index 3653bae2c018..029690c876a8 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -88,6 +88,10 @@ public abstract class AutoRepairState implements ProgressListener protected int succeededTokenRangesCount = 0; @VisibleForTesting protected int skippedTokenRangesCount = 0; + + @VisibleForTesting + protected int skippedTablesCount = 0; + @VisibleForTesting protected AutoRepairHistory longestUnrepairedNode; @VisibleForTesting @@ -260,6 +264,16 @@ public int getSkippedTokenRangesCount() return skippedTokenRangesCount; } + public void setSkippedTablesCount(int count) + { + skippedTablesCount = count; + } + + public int getSkippedTablesCount() + { + return skippedTablesCount; + } + public boolean isSuccess() { return success; diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index b10193e9b91a..a05bb0a04e32 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -32,6 +32,8 @@ import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.LocalStrategy; import org.slf4j.Logger; @@ -828,4 +830,32 @@ public static List getAllMVs(RepairType repairType, Keyspace keyspace, T } return allMvs; } + + public static List> splitEvenly(Range tokenRange, int numberOfSplits) + { + List> splitRanges = new ArrayList<>(); + long left = (Long) tokenRange.left.getTokenValue(); + long right = (Long) tokenRange.right.getTokenValue(); + long repairTokenWidth = (right - left) / numberOfSplits; + for (int i = 0; i < numberOfSplits; i++) + { + long curLeft = left + (i * repairTokenWidth); + long curRight = curLeft + repairTokenWidth; + + if ((i + 1) == numberOfSplits) + { + curRight = right; + } + + Token childStartToken = StorageService.instance.getTokenMetadata() + .partitioner.getTokenFactory().fromString("" + curLeft); + Token childEndToken = StorageService.instance.getTokenMetadata() + .partitioner.getTokenFactory().fromString("" + curRight); + logger.debug("Current Token Left side {}, right side {}", childStartToken + .toString(), childEndToken.toString()); + Range splitRange = new Range<>(childStartToken, childEndToken); + splitRanges.add(splitRange); + } + return splitRanges; + } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java index 7c254fd358d0..e3f02f6da6f0 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java @@ -20,28 +20,24 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import org.apache.cassandra.service.AutoRepairService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.Pair; + +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.splitEvenly; public class DefaultAutoRepairTokenSplitter implements IAutoRepairTokenRangeSplitter { - private static final Logger logger = LoggerFactory.getLogger(DefaultAutoRepairTokenSplitter.class); - - @Override - public List> getRange(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, String tableName) + public List getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames) { - List> range = new ArrayList<>(); + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + List repairAssignments = new ArrayList<>(); Collection> tokens = StorageService.instance.getPrimaryRanges(keyspaceName); if (!primaryRangeOnly) @@ -49,41 +45,36 @@ public List> getRange(AutoRepairConfig.RepairType repairType, // if we need to repair non-primary token ranges, then change the tokens accrodingly tokens = StorageService.instance.getLocalReplicas(keyspaceName).ranges(); } - AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); int numberOfSubranges = config.getRepairSubRangeNum(repairType); + + boolean byKeyspace = config.getRepairByKeyspace(repairType); + + // collect all token ranges. + List> allRanges = new ArrayList<>(); for (Range token : tokens) { - Murmur3Partitioner.LongToken l = (Murmur3Partitioner.LongToken) (token.left); - Murmur3Partitioner.LongToken r = (Murmur3Partitioner.LongToken) (token.right); - Token parentStartToken = StorageService.instance.getTokenMetadata() - .partitioner.getTokenFactory().fromString("" + l.getTokenValue()); - Token parentEndToken = StorageService.instance.getTokenMetadata() - .partitioner.getTokenFactory().fromString("" + r.getTokenValue()); - logger.debug("Parent Token Left side {}, right side {}", parentStartToken.toString(), - parentEndToken.toString()); + allRanges.addAll(splitEvenly(token, numberOfSubranges)); + } - long left = (Long) l.getTokenValue(); - long right = (Long) r.getTokenValue(); - long repairTokenWidth = (right - left) / numberOfSubranges; - for (int i = 0; i < numberOfSubranges; i++) + if (byKeyspace) + { + for (Range splitRange : allRanges) { - long curLeft = left + (i * repairTokenWidth); - long curRight = curLeft + repairTokenWidth; - - if ((i + 1) == numberOfSubranges) + // add repair assignment for each range entire keyspace's tables + repairAssignments.add(new RepairAssignment(splitRange, keyspaceName, tableNames)); + } + } + else + { + // add repair assignment per table + for (String tableName : tableNames) + { + for (Range splitRange : allRanges) { - curRight = right; + repairAssignments.add(new RepairAssignment(splitRange, keyspaceName, Collections.singletonList(tableName))); } - - Token childStartToken = StorageService.instance.getTokenMetadata() - .partitioner.getTokenFactory().fromString("" + curLeft); - Token childEndToken = StorageService.instance.getTokenMetadata() - .partitioner.getTokenFactory().fromString("" + curRight); - logger.debug("Current Token Left side {}, right side {}", childStartToken - .toString(), childEndToken.toString()); - range.add(Pair.create(childStartToken, childEndToken)); } } - return range; + return repairAssignments; } } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java index 2b69898a360d..dd02c4d4f7b6 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java @@ -19,13 +19,81 @@ import java.util.List; +import java.util.Objects; +import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.utils.Pair; public interface IAutoRepairTokenRangeSplitter { - // split the token range you wish to repair into multiple subranges - // the autorepair framework will repair the list of returned subrange in a sequence - List> getRange(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, String tableName); + + /** + * Split the token range you wish to repair into multiple assignments. + * The autorepair framework will repair the list of returned subrange in a sequence. + * @param repairType The type of repair being executed + * @param primaryRangeOnly Whether to repair only this node's primary ranges or all of its ranges. + * @param keyspaceName The keyspace being repaired + * @param tableNames The tables to repair + * @return repair assignments broken up by range, keyspace and tables. + */ + List getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames); + + /** + * Defines a repair assignment to be issued by the autorepair framework. + */ + class RepairAssignment + { + final Range tokenRange; + + final String keyspaceName; + + final List tableNames; + + public RepairAssignment(Range tokenRange, String keyspaceName, List tableNames) + { + this.tokenRange = tokenRange; + this.keyspaceName = keyspaceName; + this.tableNames = tableNames; + } + + public Range getTokenRange() + { + return tokenRange; + } + + public String getKeyspaceName() + { + return keyspaceName; + } + + public List getTableNames() + { + return tableNames; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RepairAssignment that = (RepairAssignment) o; + return Objects.equals(tokenRange, that.tokenRange) && Objects.equals(keyspaceName, that.keyspaceName) && Objects.equals(tableNames, that.tableNames); + } + + @Override + public int hashCode() + { + return Objects.hash(tokenRange, keyspaceName, tableNames); + } + + @Override + public String toString() + { + return "RepairAssignment{" + + "tokenRange=" + tokenRange + + ", keyspaceName='" + keyspaceName + '\'' + + ", tableNames=" + tableNames + + '}'; + } + } } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java index 02cb9f228e62..a0840ab66cd3 100644 --- a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java @@ -434,4 +434,4 @@ private enum BuildStatus { UNKNOWN, STARTED, SUCCESS } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index d3764778da9e..c9b8233c6ffc 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -50,6 +50,7 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileInputStreamPlus; import org.apache.cassandra.io.util.FileOutputStreamPlus; @@ -686,11 +687,31 @@ public static ISslContextFactory newSslContextFactory(String className, Map tokenRangeSplitterClass = Class.forName(className); + try + { + Map parameters = parameterizedClass.parameters != null ? parameterizedClass.parameters : Collections.emptyMap(); + // first attempt to initialize with Map arguments. + return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor(Map.class).newInstance(parameters); + } + catch (NoSuchMethodException nsme) + { + // fall back on no argument constructor. + return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor().newInstance(); + } + } + catch (Exception ex) + { + throw new ConfigurationException("Unable to create instance of IAutoRepairTokenRangeSplitter for " + className, ex); + } } /** diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index c42959eec32b..4f1dd029ef5e 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -20,6 +20,7 @@ import java.util.EnumMap; import java.util.Objects; +import java.util.Collections; import java.util.Set; import com.google.common.collect.ImmutableSet; @@ -31,6 +32,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.Options; @@ -384,14 +386,16 @@ public void testGetDefaultOptionsTokenRangeSplitter() { Options defaultOptions = Options.getDefaultOptions(); - assertEquals(DefaultAutoRepairTokenSplitter.class.getName(),defaultOptions.token_range_splitter); + ParameterizedClass expectedDefault = new ParameterizedClass(DefaultAutoRepairTokenSplitter.class.getName(), Collections.emptyMap()); + + assertEquals(expectedDefault, defaultOptions.token_range_splitter); assertEquals(DefaultAutoRepairTokenSplitter.class.getName(), FBUtilities.newAutoRepairTokenRangeSplitter(defaultOptions.token_range_splitter).getClass().getName()); } @Test(expected = ConfigurationException.class) public void testInvalidTokenRangeSplitter() { - assertEquals(DefaultAutoRepairTokenSplitter.class.getName(), FBUtilities.newAutoRepairTokenRangeSplitter("invalid-class").getClass().getName()); + FBUtilities.newAutoRepairTokenRangeSplitter(new ParameterizedClass("invalid-class", Collections.emptyMap())); } @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java new file mode 100644 index 000000000000..b460a750770f --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter.RepairAssignment; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.service.StorageService; + +import static org.apache.cassandra.Util.setAutoRepairEnabled; +import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; +import static org.junit.Assert.assertEquals; + +@RunWith(Parameterized.class) +public class AutoRepairDefaultTokenSplitterParameterizedTest extends CQLTester +{ + private static final String KEYSPACE = "ks"; + private static final String TABLE1 = "tbl1"; + private static final String TABLE2 = "tbl2"; + private static final String TABLE3 = "tbl3"; + + @Parameterized.Parameter() + public AutoRepairConfig.RepairType repairType; + + @Parameterized.Parameters(name = "repairType={0}") + public static Collection repairTypes() + { + return Arrays.asList(AutoRepairConfig.RepairType.values()); + } + + @BeforeClass + public static void setupClass() throws Exception + { + AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); + SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); + setAutoRepairEnabled(true); + DatabaseDescriptor.setInitialTokens("0,256,1024"); + requireNetwork(); + AutoRepairUtils.setup(); + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1)); + } + + private static void appendExpectedTokens(long left, long right, int numberOfSplits, List> expectedToken) + { + long repairTokenWidth = (right - left) / numberOfSplits; + for (int i = 0; i < numberOfSplits; i++) + { + long curLeft = left + (i * repairTokenWidth); + long curRight = curLeft + repairTokenWidth; + if ((i + 1) == numberOfSplits) + { + curRight = right; + } + Token childStartToken = StorageService.instance.getTokenMetadata() + .partitioner.getTokenFactory().fromString("" + curLeft); + Token childEndToken = StorageService.instance.getTokenMetadata() + .partitioner.getTokenFactory().fromString("" + curRight); + expectedToken.add(new Range<>(childStartToken, childEndToken)); + } + } + + @Test + public void testTokenRangesSplitByTable() + { + AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(repairType, false); + int totalTokenRanges = 3; + Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); + assertEquals(totalTokenRanges, tokens.size()); + int numberOfSplits = 4; + List tables = Arrays.asList(TABLE1, TABLE2, TABLE3); + List> expectedToken = new ArrayList<>(); + for (int i = 0; i assignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, KEYSPACE, tables); + assertEquals(totalTokenRanges*numberOfSplits*tables.size(), assignments.size()); + assertEquals(expectedToken.size(), assignments.size()); + + int expectedTableIndex = -1; + for (int i = 0; i> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); + assertEquals(totalTokenRanges, tokens.size()); + int numberOfSplits = 4; + List tables = Arrays.asList(TABLE1, TABLE2, TABLE3); + List> expectedToken = new ArrayList<>(); + appendExpectedTokens(1024, 0, numberOfSplits, expectedToken); + appendExpectedTokens(0, 256, numberOfSplits, expectedToken); + appendExpectedTokens(256, 1024, numberOfSplits, expectedToken); + + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setRepairSubRangeNum(repairType, numberOfSplits); + List assignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, KEYSPACE, tables); + assertEquals(totalTokenRanges*numberOfSplits, assignments.size()); + assertEquals(expectedToken.size(), assignments.size()); + + for (int i = 0; i> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); assertEquals(1, tokens.size()); - List> expectedToken = new ArrayList<>(); - expectedToken.addAll(tokens); + List> expectedToken = new ArrayList<>(tokens); - List> ranges = new DefaultAutoRepairTokenSplitter().getRange(repairType, true, KEYSPACE, TABLE); - assertEquals(1, ranges.size()); - assertEquals(expectedToken.get(0).left, ranges.get(0).left); - assertEquals(expectedToken.get(0).right, ranges.get(0).right); - } - - @Test - public void testTokenRangesSplit() - { - Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); - assertEquals(1, tokens.size()); - List> expectedToken = new ArrayList<>(); - expectedToken.addAll(tokens); - - AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - config.setRepairSubRangeNum(repairType, 4); - List> ranges = new DefaultAutoRepairTokenSplitter().getRange(repairType, true, KEYSPACE, TABLE); - assertEquals(4, ranges.size()); + List assignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, KEYSPACE, Collections.singletonList(TABLE)); + assertEquals(1, assignments.size()); + assertEquals(expectedToken.get(0).left, assignments.get(0).getTokenRange().left); + assertEquals(expectedToken.get(0).right, assignments.get(0).getTokenRange().right); } @Test From 5bbd5a5562e01e2462ca832fca3faa0ce01905ed Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sat, 12 Oct 2024 09:45:10 -0700 Subject: [PATCH 043/199] Fix a corner-case bug NPE org.apache.cassandra.repair.autorepair.AutoRepairUtils.myTurnToRunRepair --- .../repair/autorepair/AutoRepairUtils.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index a05bb0a04e32..14e61003236e 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -575,12 +575,8 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) } } - // get the longest unrepaired node from the nodes which are not running repair - AutoRepairHistory defaultNodeToBeRepaired = getHostWithLongestUnrepairTime(currentRepairStatus.historiesWithoutOnGoingRepair); - //check who is next, which is helpful for debugging - logger.info("Next node to be repaired for repair type {} by default: {}", repairType, defaultNodeToBeRepaired); UUID priorityHostId = null; - if (currentRepairStatus.priority != null) + if (currentRepairStatus != null && currentRepairStatus.priority != null) { for (UUID priorityID : currentRepairStatus.priority) { @@ -611,8 +607,14 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) return MY_TURN_DUE_TO_PRIORITY; } - if (defaultNodeToBeRepaired.hostId.equals(myId)) + // get the longest unrepaired node from the nodes which are not running repair + AutoRepairHistory defaultNodeToBeRepaired = getHostWithLongestUnrepairTime(currentRepairStatus.historiesWithoutOnGoingRepair); + //check who is next, which is helpful for debugging + logger.info("Next node to be repaired for repair type {} by default: {}", repairType, defaultNodeToBeRepaired); + if (defaultNodeToBeRepaired != null && defaultNodeToBeRepaired.hostId.equals(myId)) + { return MY_TURN; + } } else if (currentRepairStatus.hostIdsWithOnGoingForceRepair.contains(myId)) { From 69989ee6b3768e5208cc0aa0ab12b946af7c911d Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 14 Oct 2024 13:50:25 -0700 Subject: [PATCH 044/199] Update documentation for the newly added metric --- doc/modules/cassandra/pages/operating/metrics.adoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/modules/cassandra/pages/operating/metrics.adoc b/doc/modules/cassandra/pages/operating/metrics.adoc index 69948a00c90a..1728dbf8d9ee 100644 --- a/doc/modules/cassandra/pages/operating/metrics.adoc +++ b/doc/modules/cassandra/pages/operating/metrics.adoc @@ -978,6 +978,9 @@ ran on the node in seconds |SkippedTokenRangesCount |Gauge |Number of token ranges skipped on the node +|SkippedTablesCount |Gauge |Number of tables skipped +on the node + |TotalMVTablesConsideredForRepair |Gauge |Number of materialized views considered on the node From 0a758abac7ce3996253a1635a6df8689ed440edb Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 15 Oct 2024 14:50:14 -0700 Subject: [PATCH 045/199] Formatting: Add a new line towards the end SystemDistributedKeyspace.java --- .../org/apache/cassandra/schema/SystemDistributedKeyspace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java index a0840ab66cd3..02cb9f228e62 100644 --- a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java @@ -434,4 +434,4 @@ private enum BuildStatus { UNKNOWN, STARTED, SUCCESS } -} \ No newline at end of file +} From 1bc506836a9290293239219ef74ea2fd8bcdce60 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 17 Oct 2024 16:51:37 -0700 Subject: [PATCH 046/199] Revert .circleci/config.yaml settings --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d90de438d53d..6e48d91b5031 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,7 +24,7 @@ jobs: resource_class: medium working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 4 + parallelism: 1 steps: - attach_workspace: at: /home/cassandra From a28679028c495815e7a3c989d32d5b425c3bff86 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Sat, 26 Oct 2024 15:12:19 -0700 Subject: [PATCH 047/199] Implement previewRepaired --- .../cassandra/repair/autorepair/AutoRepairState.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index aace983e968a..c8e4228dfb2c 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -290,7 +290,7 @@ class PreviewRepairedState extends AutoRepairState { public PreviewRepairedState() { - super(RepairType.preview); + super(RepairType.previewRepaired); } @Override @@ -304,6 +304,16 @@ public RepairRunnable getRepairRunnable(String keyspace, List tables, Se return getRepairRunnable(keyspace, option); } + + @Override + public void progress(String tag, ProgressEvent event) { + if (event.getType() == ProgressEventType.COMPLETE) + { + + } + + super.progress(tag, event); + } } class IncrementalRepairState extends AutoRepairState From 74b2d83fa0601f2c3f856e78cc025b5f59d3a7ef Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 27 Oct 2024 15:58:02 -0700 Subject: [PATCH 048/199] Code formatting: Remove Group from naming --- .../apache/cassandra/service/AutoRepairService.java | 8 ++++---- .../cassandra/service/AutoRepairServiceMBean.java | 4 ++-- src/java/org/apache/cassandra/tools/NodeProbe.java | 8 ++++---- .../cassandra/tools/nodetool/SetAutoRepairConfig.java | 10 +++++----- .../cassandra/service/AutoRepairServiceTest.java | 4 ++-- .../tools/nodetool/SetAutoRepairConfigTest.java | 4 ++-- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 62393d66ebab..04aaba4fa3ba 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -162,15 +162,15 @@ public void setPrimaryTokenRangeOnly(RepairType repairType, boolean primaryToken } @Override - public void setParallelRepairPercentageInGroup(RepairType repairType, int percentageInGroup) + public void setParallelRepairPercentage(RepairType repairType, int percentage) { - config.setParallelRepairPercentage(repairType, percentageInGroup); + config.setParallelRepairPercentage(repairType, percentage); } @Override - public void setParallelRepairCountInGroup(RepairType repairType, int countInGroup) + public void setParallelRepairCount(RepairType repairType, int count) { - config.setParallelRepairCount(repairType, countInGroup); + config.setParallelRepairCount(repairType, count); } public void setMVRepairEnabled(RepairType repairType, boolean enabled) diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index 5e62e5630043..b6f08715d3e4 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -56,9 +56,9 @@ public interface AutoRepairServiceMBean public void setPrimaryTokenRangeOnly(RepairType repairType, boolean primaryTokenRangeOnly); - public void setParallelRepairPercentageInGroup(RepairType repairType, int percentageInGroup); + public void setParallelRepairPercentage(RepairType repairType, int percentage); - public void setParallelRepairCountInGroup(RepairType repairType, int countInGroup); + public void setParallelRepairCount(RepairType repairType, int count); public void setMVRepairEnabled(RepairType repairType, boolean enabled); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index e89294c5c341..2d04b76fe3d9 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2215,12 +2215,12 @@ public void setAutoRepairIgnoreDCs(AutoRepairConfig.RepairType repairType, Set validateLocalGroupHosts(String paramVal) + private Set retrieveHosts(String paramVal) { Set hosts = new HashSet<>(); for (String host : Splitter.on(',').split(paramVal)) diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index 874ec75862e9..5da91ecef409 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -243,8 +243,8 @@ public static Collection testCases() forEachRepairType(400, AutoRepairService.instance::setRepairSSTableCountHigherThreshold, config::getRepairSSTableCountHigherThreshold), forEachRepairType(ImmutableSet.of("dc1", "dc2"), AutoRepairService.instance::setIgnoreDCs, config::getIgnoreDCs), forEachRepairType(true, AutoRepairService.instance::setPrimaryTokenRangeOnly, config::getRepairPrimaryTokenRangeOnly), - forEachRepairType(600, AutoRepairService.instance::setParallelRepairPercentageInGroup, config::getParallelRepairPercentage), - forEachRepairType(700, AutoRepairService.instance::setParallelRepairCountInGroup, config::getParallelRepairCount), + forEachRepairType(600, AutoRepairService.instance::setParallelRepairPercentage, config::getParallelRepairPercentage), + forEachRepairType(700, AutoRepairService.instance::setParallelRepairCount, config::getParallelRepairCount), forEachRepairType(true, AutoRepairService.instance::setMVRepairEnabled, config::getMVRepairEnabled), forEachRepairType(ImmutableSet.of(InetAddressAndPort.getLocalHost()), AutoRepairService.instance::setRepairPriorityForHosts, AutoRepairUtils::getPriorityHosts), forEachRepairType(ImmutableSet.of(InetAddressAndPort.getLocalHost()), AutoRepairService.instance::setForceRepairForHosts, SetterTests::isLocalHostForceRepair) diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index 2a69cefcace9..5245062ad815 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -259,8 +259,8 @@ public static Collection testCases() forEachRepairType("sstable_upper_threshold", "4", (type) -> verify(probe, times(1)).setRepairSSTableCountHigherThreshold(type, 4)), forEachRepairType("table_max_repair_time", "5s", (type) -> verify(probe, times(1)).setAutoRepairTableMaxRepairTime(type, "5s")), forEachRepairType("repair_primary_token_range_only", "true", (type) -> verify(probe, times(1)).setPrimaryTokenRangeOnly(type, true)), - forEachRepairType("parallel_repair_count", "6", (type) -> verify(probe, times(1)).setParallelRepairCountInGroup(type, 6)), - forEachRepairType("parallel_repair_percentage", "7", (type) -> verify(probe, times(1)).setParallelRepairPercentageInGroup(type, 7)), + forEachRepairType("parallel_repair_count", "6", (type) -> verify(probe, times(1)).setParallelRepairCount(type, 6)), + forEachRepairType("parallel_repair_percentage", "7", (type) -> verify(probe, times(1)).setParallelRepairPercentage(type, 7)), forEachRepairType("mv_repair_enabled", "true", (type) -> verify(probe, times(1)).setMVRepairEnabled(type, true)), forEachRepairType("ignore_dcs", "dc1,dc2", (type) -> verify(probe, times(1)).setAutoRepairIgnoreDCs(type, ImmutableSet.of("dc1", "dc2"))) ).flatMap(Function.identity()).collect(Collectors.toList()); From ecc15a175b8ba31a537f10a0e2c8081afd34daf1 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 27 Oct 2024 20:57:39 -0700 Subject: [PATCH 049/199] Enable repair scheduler through nodetool --- .../repair/autorepair/AutoRepairConfig.java | 8 +++++++- .../cassandra/service/AutoRepairService.java | 6 ++++++ .../service/AutoRepairServiceMBean.java | 2 ++ .../org/apache/cassandra/tools/NodeProbe.java | 5 +++++ .../tools/nodetool/SetAutoRepairConfig.java | 10 ++++++++-- .../tools/nodetool/SetAutoRepairConfigTest.java | 16 ++++++++++++++++ 6 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 30433a1ad7ff..a80a875125a9 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -39,7 +39,7 @@ public class AutoRepairConfig implements Serializable // enable/disable auto repair globally, overrides all other settings. Cannot be modified dynamically. // if it is set to false, then no repair will be scheduled, including full and incremental repairs by this framework. // if it is set to true, then this repair scheduler will consult another config available for each RepairType, and based on that config, it will schedule repairs. - public final Boolean enabled; + public volatile Boolean enabled; // the interval between successive checks for repair scheduler to check if either the ongoing repair is completed or if // none is going, then check if it's time to schedule or wait public final DurationSpec.IntSecondsBound repair_check_interval = new DurationSpec.IntSecondsBound("5m"); @@ -108,6 +108,12 @@ public DurationSpec.IntSecondsBound getAutoRepairHistoryClearDeleteHostsBufferIn return history_clear_delete_hosts_buffer_interval; } + public void startScheduler() + { + enabled = true; + AutoRepair.instance.setup(); + } + public void setAutoRepairHistoryClearDeleteHostsBufferInterval(String duration) { history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound(duration); diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 04aaba4fa3ba..43d27ebbb248 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -120,6 +120,12 @@ public void setRepairMinInterval(RepairType repairType, String minRepairInterval config.setRepairMinInterval(repairType, minRepairInterval); } + @Override + public void startScheduler() + { + config.startScheduler(); + } + public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration) { config.setAutoRepairHistoryClearDeleteHostsBufferInterval(duration); diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index b6f08715d3e4..121c9a480303 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -42,6 +42,8 @@ public interface AutoRepairServiceMBean public void setRepairMinInterval(RepairType repairType, String minRepairInterval); + void startScheduler(); + public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration); public void setAutoRepairMaxRetriesCount(int retries); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 2d04b76fe3d9..54092ae18705 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2190,6 +2190,11 @@ public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration) autoRepairProxy.setAutoRepairHistoryClearDeleteHostsBufferDuration(duration); } + public void startScheduler() + { + autoRepairProxy.startScheduler(); + } + public void setAutoRepairMaxRetriesCount(int retries) { autoRepairProxy.setAutoRepairMaxRetriesCount(retries); diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index cdd3ca58a8fc..2929c944442a 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -43,7 +43,7 @@ public class SetAutoRepairConfig extends NodeToolCmd @VisibleForTesting @Arguments(title = " ", usage = " ", description = "autorepair param and value.\nPossible autorepair parameters are as following: " + - "[number_of_repair_threads|number_of_subranges|min_repair_interval|sstable_upper_threshold" + + "[start_scheduler|number_of_repair_threads|number_of_subranges|min_repair_interval|sstable_upper_threshold" + "|enabled|table_max_repair_time|priority_hosts|forcerepair_hosts|ignore_dcs" + "|history_clear_delete_hosts_buffer_interval|repair_primary_token_range_only" + "|parallel_repair_count|parallel_repair_percentage|mv_repair_enabled|repair_max_retries|repair_retry_backoff|repair_session_timeout]", @@ -64,7 +64,7 @@ public void execute(NodeProbe probe) String paramType = args.get(0); String paramVal = args.get(1); - if (!probe.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + if (!probe.getAutoRepairConfig().isAutoRepairSchedulingEnabled() && !paramType.equalsIgnoreCase("start_scheduler")) { out.println("Auto-repair is not enabled"); return; @@ -73,6 +73,12 @@ public void execute(NodeProbe probe) // options that do not require --repair-type option switch (paramType) { + case "start_scheduler": + if (Boolean.parseBoolean(paramVal)) + { + probe.startScheduler(); + } + return; case "history_clear_delete_hosts_buffer_interval": probe.setAutoRepairHistoryClearDeleteHostsBufferDuration(paramVal); return; diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index 5245062ad815..6e2d79def026 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -115,6 +115,22 @@ public void testRetryBackoffInSec() verify(probe, times(1)).setAutoRepairRetryBackoff("3s"); } + + @Test + public void testStartScheduler() + { + cmd.args = ImmutableList.of("start_scheduler", "false"); + + cmd.execute(probe); + + verify(probe, times(0)).startScheduler(); + + cmd.args = ImmutableList.of("start_scheduler", "true"); + + cmd.execute(probe); + + verify(probe, times(1)).startScheduler(); + } } @RunWith(Parameterized.class) From 00f0dff34f603d37affabe63bdae0111143d9b3b Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 3 Nov 2024 13:10:20 -0600 Subject: [PATCH 050/199] UnrepairedBytesBasedTokenRangeSplitter prototype Adds prototype UnrepairedBytesBasedTokenRangeSplitter that demonstrates consuming splitter-based properties to limit the repair assignments returned to be bound to the requested amount of work. --- .../sstable/format/big/BigTableScanner.java | 47 ++- ...nrepairedBytesBasedTokenRangeSplitter.java | 336 ++++++++++++++++++ 2 files changed, 377 insertions(+), 6 deletions(-) create mode 100644 src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java index 235b9b1a7173..451cdd3a84d6 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java @@ -21,6 +21,8 @@ import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; + import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.AbstractIterator; @@ -117,7 +119,7 @@ private BigTableScanner(SSTableReader sstable, this.listener = listener; } - private static List> makeBounds(SSTableReader sstable, Collection> tokenRanges) + public static List> makeBounds(SSTableReader sstable, Collection> tokenRanges) { List> boundsList = new ArrayList<>(tokenRanges.size()); for (Range range : Range.normalize(tokenRanges)) @@ -177,9 +179,12 @@ private static void addRange(SSTableReader sstable, AbstractBounds range, + boolean seekDataFile) throws CorruptSSTableException { - long indexPosition = sstable.getIndexScanPosition(currentRange.left); + long indexPosition = sstable.getIndexScanPosition(position); ifile.seek(indexPosition); try { @@ -188,13 +193,21 @@ private void seekToCurrentRangeStart() { indexPosition = ifile.getFilePointer(); DecoratedKey indexDecoratedKey = sstable.decorateKey(ByteBufferUtil.readWithShortLength(ifile)); - if (indexDecoratedKey.compareTo(currentRange.left) > 0 || currentRange.contains(indexDecoratedKey)) + + // Whether the position or range is present in the SSTable. + boolean isFound = (range == null && indexDecoratedKey.compareTo(position) > 0) + || (range != null && (indexDecoratedKey.compareTo(range.left) > 0 || range.contains(indexDecoratedKey))); + if (isFound) { // Found, just read the dataPosition and seek into index and data files long dataPosition = RowIndexEntry.Serializer.readPosition(ifile); + // seek the index file position as we will presumably seek further when going to the next position. ifile.seek(indexPosition); - dfile.seek(dataPosition); - break; + if (seekDataFile) + { + dfile.seek(dataPosition); + } + return dataPosition; } else { @@ -207,6 +220,28 @@ private void seekToCurrentRangeStart() sstable.markSuspect(); throw new CorruptSSTableException(e, sstable.getFilename()); } + // If for whatever reason we don't find position in file, just return 0 + return 0L; + } + + /** + * Seeks to the start of the current range and updates both the index and data file positions. + */ + private void seekToCurrentRangeStart() throws CorruptSSTableException + { + seekAndProcess(currentRange.left, currentRange, true); + } + + /** + * Gets the position in the data file, but does not seek to it. This does seek the index to find the data position + * but does not actually seek the data file. + * @param position position to find in data file. + * @return offset in data file where position exists. + * @throws CorruptSSTableException if SSTable was malformed + */ + public long getDataPosition(PartitionPosition position) throws CorruptSSTableException + { + return seekAndProcess(position, null, false); } public void close() diff --git a/src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java new file mode 100644 index 000000000000..c51a37d69779 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java @@ -0,0 +1,336 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DataStorageSpec; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.lifecycle.SSTableIntervalTree; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigTableScanner; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.concurrent.Refs; + +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.splitEvenly; + +public class UnrepairedBytesBasedTokenRangeSplitter implements IAutoRepairTokenRangeSplitter +{ + private static final Logger logger = LoggerFactory.getLogger(UnrepairedBytesBasedTokenRangeSplitter.class); + + static final String SUBRANGE_SIZE = "subrange_size"; + static final String MAX_BYTES_PER_SCHEDULE = "max_bytes_per_schedule"; + + // target bytes per subrange + private final DataStorageSpec.LongBytesBound subrangeSize; + + // maximum target bytes to repair + private final DataStorageSpec.LongBytesBound maxBytesPerSchedule; + + private final long subrangeBytes; + + private final long maxBytesPerScheduleBytes; + + private static final DataStorageSpec.LongBytesBound DEFAULT_SUBRANGE_SIZE = new DataStorageSpec.LongBytesBound("100GiB"); + private static final DataStorageSpec.LongBytesBound DEFAULT_MAX_BYTES_PER_SCHEDULE = new DataStorageSpec.LongBytesBound("500GiB"); + + public UnrepairedBytesBasedTokenRangeSplitter(Map parameters) + { + // Demonstrates parameterizing a range splitter so we can have splitter specific options. + if (parameters.containsKey(SUBRANGE_SIZE)) + { + subrangeSize = new DataStorageSpec.LongBytesBound(parameters.get(SUBRANGE_SIZE)); + } + else + { + subrangeSize = DEFAULT_SUBRANGE_SIZE; + } + subrangeBytes = subrangeSize.toBytes(); + + if (parameters.containsKey(MAX_BYTES_PER_SCHEDULE)) + { + maxBytesPerSchedule = new DataStorageSpec.LongBytesBound(parameters.get(MAX_BYTES_PER_SCHEDULE)); + } + else + { + maxBytesPerSchedule = DEFAULT_MAX_BYTES_PER_SCHEDULE; + } + maxBytesPerScheduleBytes = maxBytesPerSchedule.toBytes(); + + logger.info("Configured {} with {}={}, {}={}", UnrepairedBytesBasedTokenRangeSplitter.class.getName(), + SUBRANGE_SIZE, subrangeSize, MAX_BYTES_PER_SCHEDULE, maxBytesPerSchedule); + } + + @Override + public List getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames) + { + List repairAssignments = new ArrayList<>(); + + logger.info("Calculating token range splits for repairType={} primaryRangeOnly={} keyspaceName={} tableNames={}", repairType, primaryRangeOnly, keyspaceName, tableNames); + if (repairType != AutoRepairConfig.RepairType.incremental) + { + throw new IllegalArgumentException(this.getClass().getName() + " only supports " + AutoRepairConfig.RepairType.incremental + " repair"); + } + + // TODO: create a custom repair assignment that indicates number of bytes in repair and join tables by byte size. + Collection> tokenRanges = getTokenRanges(primaryRangeOnly, keyspaceName); + for (String tableName : tableNames) + { + repairAssignments.addAll(getRepairAssignmentsForTable(keyspaceName, tableName, tokenRanges)); + } + return repairAssignments; + } + + public List getRepairAssignmentsForTable(String keyspaceName, String tableName, Collection> tokenRanges) + { + List repairAssignments = new ArrayList<>(); + + long targetBytesSoFar = 0; + + for (Range tokenRange : tokenRanges) + { + logger.info("Calculating unrepaired bytes for {}.{} for range {}", keyspaceName, tableName, tokenRange); + // Capture the amount of unrepaired bytes for range + long approximateUnrepairedBytesForRange = 0L; + // Capture the total bytes in read sstables, this will be useful for calculating the ratio + // of data in SSTables including this range and also useful to know how much anticompaction there will be. + long totalBytesInUnrepairedSSTables = 0L; + try (Refs refs = getSSTableReaderRefs(keyspaceName, tableName, tokenRange)) + { + for (SSTableReader reader : refs) + { + // Only evaluate unrepaired SSTables. + if (!reader.isRepaired()) + { + long sstableSize = reader.bytesOnDisk(); + totalBytesInUnrepairedSSTables += sstableSize; + // get the bounds of the sstable for this range using the index file but do not actually read it. + List> bounds = BigTableScanner.makeBounds(reader, Collections.singleton(tokenRange)); + try (BigTableScanner scanner = (BigTableScanner) BigTableScanner.getScanner(reader, Collections.singleton(tokenRange))) + { + assert bounds.size() == 1; + + AbstractBounds bound = bounds.get(0); + long startPosition = scanner.getDataPosition(bound.left); + long endPosition = scanner.getDataPosition(bound.right); + // If end position is 0 we can assume the sstable ended before that token, bound at size of file + if (endPosition == 0) + { + endPosition = sstableSize; + } + + long approximateRangeBytesInSSTable = Math.max(0, endPosition - startPosition); + // get the fraction of the sstable belonging to the range. + approximateUnrepairedBytesForRange += Math.min(approximateRangeBytesInSSTable, sstableSize); + double ratio = approximateRangeBytesInSSTable / (double) sstableSize; + logger.info("Calculations for {}.{} {}: sstableSize={}, rangeBytesInSSTable={}, startPosition={}, endPosition={}, ratio={}", + keyspaceName, tableName, reader.descriptor.baseFilename(), + FileUtils.stringifyFileSize(sstableSize), FileUtils.stringifyFileSize(approximateRangeBytesInSSTable), startPosition, endPosition, ratio); + } + } + else + { + logger.info("Skipping over {}.{} {} ({}) because it is repaired", keyspaceName, tableName, reader.descriptor.baseFilename(), FileUtils.stringifyFileSize(reader.bytesOnDisk())); + } + } + } + + // Only consider token range if it had unrepaired sstables or live data in memtables. + if (totalBytesInUnrepairedSSTables > 0L) + { + // TODO: Possibly some anticompaction configuration we want here, where if we detect a large amount of anticompaction we want to reduce the work we do. + double ratio = approximateUnrepairedBytesForRange / (double) totalBytesInUnrepairedSSTables; + logger.info("Calculated unrepaired bytes for {}.{} for range {}: sstableSize={}, rangeBytesInSSTables={}, ratio={}", keyspaceName, tableName, tokenRange, + FileUtils.stringifyFileSize(totalBytesInUnrepairedSSTables), FileUtils.stringifyFileSize(approximateUnrepairedBytesForRange), ratio); + + // TODO: split on byte size here, this is currently a bit naive in assuming that data is evenly distributed among the range which may not be the + // right assumption. May want to consider when splitting on these ranges to reevaluate how much data is in the range, but for this + // exists as a demonstration. + if (approximateUnrepairedBytesForRange < subrangeBytes) + { + // accept range as is if less than bytes. + logger.info("Using 1 repair assignment for {}.{} for range {} as {} is less than {}", keyspaceName, tableName, tokenRange, + FileUtils.stringifyFileSize(approximateUnrepairedBytesForRange), subrangeSize); + // TODO: this is a bit repetitive see if can reduce more. + RepairAssignment assignment = new BytesBasedRepairAssignment(tokenRange, keyspaceName, Collections.singletonList(tableName), approximateUnrepairedBytesForRange); + if (canAddAssignment(assignment, targetBytesSoFar, approximateUnrepairedBytesForRange)) + { + repairAssignments.add(assignment); + targetBytesSoFar += approximateUnrepairedBytesForRange; + } + else + return repairAssignments; + } + else + { + long targetRanges = approximateUnrepairedBytesForRange / subrangeBytes; + // TODO: approximation per range, this is a bit lossy since targetRanges rounds down. + long approximateBytesPerSplit = approximateUnrepairedBytesForRange / targetRanges; + logger.info("Splitting {}.{} for range {} into {} sub ranges, approximateBytesPerSplit={}", keyspaceName, tableName, tokenRange, targetRanges, FileUtils.stringifyFileSize(approximateBytesPerSplit)); + List> splitRanges = splitEvenly(tokenRange, (int) targetRanges); + int splitRangeCount = 0; + for (Range splitRange : splitRanges) + { + RepairAssignment assignment = new BytesBasedRepairAssignment(splitRange, keyspaceName, Collections.singletonList(tableName), approximateBytesPerSplit); + if (canAddAssignment(assignment, targetBytesSoFar, approximateBytesPerSplit)) + { + logger.info("Added repair assignment for {}.{} for subrange {} (#{}/{}) with approximateBytes={}", + keyspaceName, tableName, splitRange, ++splitRangeCount, splitRanges.size(), FileUtils.stringifyFileSize(approximateBytesPerSplit)); + repairAssignments.add(assignment); + targetBytesSoFar += approximateBytesPerSplit; + } + else + return repairAssignments; + } + } + } + else + { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); + long memtableSize = cfs.getTracker().getView().getCurrentMemtable().getLiveDataSize(); + if (memtableSize > 0L) + { + logger.info("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", keyspaceName, tableName, tokenRange, memtableSize); + RepairAssignment assignment = new BytesBasedRepairAssignment(tokenRange, keyspaceName, Collections.singletonList(tableName), memtableSize); + if (targetBytesSoFar >= maxBytesPerScheduleBytes) + { + return repairAssignments; + } + repairAssignments.add(assignment); + targetBytesSoFar += memtableSize; + } + else + { + logger.info("Skipping {}.{} for range {} because it had no unrepaired SSTables and no memtable data", keyspaceName, tableName, tokenRange); + } + } + } + return repairAssignments; + } + + private boolean canAddAssignment(RepairAssignment repairAssignment, long targetBytesSoFar, long bytesToBeAdded) + { + if (targetBytesSoFar + bytesToBeAdded < maxBytesPerScheduleBytes) + { + return true; + } + logger.warn("Refusing to add {} with a target size of {} because it would increase total repair bytes to {} which is greater than {}={}", + repairAssignment, FileUtils.stringifyFileSize(bytesToBeAdded), FileUtils.stringifyFileSize(targetBytesSoFar + bytesToBeAdded), MAX_BYTES_PER_SCHEDULE, maxBytesPerSchedule); + return false; + } + + public Collection> getTokenRanges(boolean primaryRangeOnly, String keyspaceName) + { + // Collect all applicable token ranges + Collection> wrappedRanges; + if (primaryRangeOnly) + { + wrappedRanges = StorageService.instance.getPrimaryRanges(keyspaceName); + } + else + { + wrappedRanges = StorageService.instance.getLocalRanges(keyspaceName); + } + + // Unwrap each range as we need to account for ranges that overlap the ring + Collection> ranges = new ArrayList<>(); + for (Range wrappedRange : wrappedRanges) + { + ranges.addAll(wrappedRange.unwrap()); + } + + return ranges; + } + + public Refs getSSTableReaderRefs(String keyspaceName, String tableName, Range tokenRange) + { + final ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); + + if (cfs == null) + { + throw new IllegalArgumentException(String.format("Could not resolve ColumnFamilyStore from %s.%s", keyspaceName, tableName)); + } + + Iterable sstables = cfs.getTracker().getView().select(SSTableSet.CANONICAL); + SSTableIntervalTree tree = SSTableIntervalTree.build(sstables); + Range r = Range.makeRowRange(tokenRange); + Iterable canonicalSSTables = View.sstablesInBounds(r.left, r.right, tree); + + // TODO: may need to reason about this not working. + return Refs.ref(canonicalSSTables); + } + + public static class BytesBasedRepairAssignment extends RepairAssignment + { + private final long approximateBytes; + + public BytesBasedRepairAssignment(Range tokenRange, String keyspaceName, List tableNames, long approximateBytes) + { + super(tokenRange, keyspaceName, tableNames); + this.approximateBytes = approximateBytes; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + BytesBasedRepairAssignment that = (BytesBasedRepairAssignment) o; + return approximateBytes == that.approximateBytes; + } + + @Override + public int hashCode() + { + return Objects.hash(super.hashCode(), approximateBytes); + } + + @Override + public String toString() + { + return "BytesBasedRepairAssignment{" + + "keyspaceName='" + keyspaceName + '\'' + + ", approximateBytes=" + approximateBytes + + ", tokenRange=" + tokenRange + + ", tableNames=" + tableNames + + '}'; + } + + public long getApproximateBytes() + { + return approximateBytes; + } + } +} From a63d70e3dd5fc1f8e9e054809576ebf96cad1a37 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 4 Nov 2024 10:05:39 -0800 Subject: [PATCH 051/199] Backport CASSANDRA-17666: Option to disable write path during streaming for CDC enabled tables --- conf/cassandra.yaml | 12 +++ .../org/apache/cassandra/config/Config.java | 3 + .../cassandra/config/DatabaseDescriptor.java | 10 ++ .../cassandra/db/commitlog/CommitLog.java | 28 +++++- .../db/commitlog/CommitLogMBean.java | 6 ++ .../db/streaming/CassandraStreamReceiver.java | 16 ++- .../cdc/ToggleCDCOnRepairEnabledTest.java | 97 +++++++++++++++++++ 7 files changed, 163 insertions(+), 9 deletions(-) create mode 100644 test/distributed/org/apache/cassandra/distributed/test/cdc/ToggleCDCOnRepairEnabledTest.java diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index be1439c80ef5..8f9ef88b27e8 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -303,6 +303,18 @@ partitioner: org.apache.cassandra.dht.Murmur3Partitioner # containing a CDC-enabled table if at space limit in cdc_raw_directory). cdc_enabled: false +# Specify whether writes to the CDC-enabled tables should be blocked when CDC data on disk has reached to the limit. +# When setting to false, the writes will not be blocked and the oldest CDC data on disk will be deleted to +# ensure the size constraint. The default is true. +# cdc_block_writes: true + +# Specify whether CDC mutations are replayed through the write path on streaming, e.g. repair. +# When enabled, CDC data streamed to the destination node will be written into commit log first. When setting to false, +# the streamed CDC data is written into SSTables just the same as normal streaming. The default is true. +# If this is set to false, streaming will be considerably faster however it's possible that, in extreme situations +# (losing > quorum # nodes in a replica set), you may have data in your SSTables that never makes it to the CDC log. +# cdc_on_repair_enabled: true + # CommitLogSegments are moved to this directory on flush if cdc_enabled: true and the # segment contains mutations for a CDC-enabled table. This should be placed on a # separate spindle than the data directories. If not set, the default directory is diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 81c5e9e8e8f2..b5763ff99e20 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -388,6 +388,9 @@ public MemtableOptions() // When true, new CDC mutations are rejected/blocked when reaching max CDC storage. // When false, new CDC mutations can always be added. But it will remove the oldest CDC commit log segment on full. public volatile boolean cdc_block_writes = true; + // When true, CDC data in SSTable go through commit logs during internodes streaming, e.g. repair + // When false, it behaves the same as normal streaming. + public volatile boolean cdc_on_repair_enabled = true; public String cdc_raw_directory; @Replaces(oldName = "cdc_total_space_in_mb", converter = Converters.MEBIBYTES_DATA_STORAGE_INT, deprecated = true) public DataStorageSpec.IntMebibytesBound cdc_total_space = new DataStorageSpec.IntMebibytesBound("0MiB"); diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index d7663b2aa1a7..dc4d48978baf 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -3738,6 +3738,16 @@ public static void setCDCBlockWrites(boolean val) conf.cdc_block_writes = val; } + public static boolean isCDCOnRepairEnabled() + { + return conf.cdc_on_repair_enabled; + } + + public static void setCDCOnRepairEnabled(boolean val) + { + conf.cdc_on_repair_enabled = val; + } + public static String getCDCLogLocation() { return conf.cdc_raw_directory; diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java index 6195b1b4ca02..ef7ab7a41c6f 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java @@ -453,11 +453,7 @@ public boolean getCDCBlockWrites() @Override public void setCDCBlockWrites(boolean val) { - Preconditions.checkState(DatabaseDescriptor.isCDCEnabled(), - "Unable to set block_writes (%s): CDC is not enabled.", val); - Preconditions.checkState(segmentManager instanceof CommitLogSegmentManagerCDC, - "CDC is enabled but we have the wrong CommitLogSegmentManager type: %s. " + - "Please report this as bug.", segmentManager.getClass().getName()); + ensureCDCEnabled("Unable to set block_writes."); boolean oldVal = DatabaseDescriptor.getCDCBlockWrites(); CommitLogSegment currentSegment = segmentManager.allocatingFrom(); // Update the current segment CDC state to PERMITTED if block_writes is disabled now, and it was in FORBIDDEN state @@ -467,6 +463,28 @@ public void setCDCBlockWrites(boolean val) logger.info("Updated CDC block_writes from {} to {}", oldVal, val); } + @Override + public boolean isCDCOnRepairEnabled() + { + return DatabaseDescriptor.isCDCOnRepairEnabled(); + } + + @Override + public void setCDCOnRepairEnabled(boolean value) + { + ensureCDCEnabled("Unable to set cdc_on_repair_enabled."); + DatabaseDescriptor.setCDCOnRepairEnabled(value); + logger.info("Set cdc_on_repair_enabled to {}", value); + } + + private void ensureCDCEnabled(String hint) + { + Preconditions.checkState(DatabaseDescriptor.isCDCEnabled(), "CDC is not enabled. %s", hint); + Preconditions.checkState(segmentManager instanceof CommitLogSegmentManagerCDC, + "CDC is enabled but we have the wrong CommitLogSegmentManager type: %s. " + + "Please report this as bug.", segmentManager.getClass().getName()); + } + /** * Shuts down the threads used by the commit log, blocking until completion. * TODO this should accept a timeout, and throw TimeoutException diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogMBean.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogMBean.java index 7e8deca9b086..189916c66e52 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogMBean.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogMBean.java @@ -88,4 +88,10 @@ public interface CommitLogMBean public boolean getCDCBlockWrites(); public void setCDCBlockWrites(boolean val); + + /** Returns true if internodes streaming of CDC data should go through write path */ + boolean isCDCOnRepairEnabled(); + + /** Set whether enable write path for CDC data during internodes streaming, e.g. repair */ + void setCDCOnRepairEnabled(boolean value); } diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index 95ede080738c..1ca5afd61abe 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -27,6 +27,7 @@ import com.google.common.collect.Iterables; import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.io.sstable.SSTable; @@ -173,6 +174,12 @@ private boolean hasCDC(ColumnFamilyStore cfs) return cfs.metadata().params.cdc; } + // returns true iif it is a cdc table and cdc on repair is enabled. + private boolean cdcRequiresWriteCommitLog(ColumnFamilyStore cfs) + { + return DatabaseDescriptor.isCDCOnRepairEnabled() && hasCDC(cfs); + } + /* * We have a special path for views and for CDC. * @@ -184,13 +191,14 @@ private boolean hasCDC(ColumnFamilyStore cfs) */ public boolean requiresWritePath(ColumnFamilyStore cfs) { - // TODO: Deprecate STREAMING_REQUIRES_CDC_REPLAY property in trunk (or 5.0) as a new config has been added to control this https://issues.apache.org/jira/browse/CASSANDRA-17666 - return (hasCDC(cfs) && CassandraRelevantProperties.STREAMING_REQUIRES_CDC_REPLAY.getBoolean()) || cfs.streamToMemtable() || (session.streamOperation().requiresViewBuild() && hasViews(cfs)); + return cdcRequiresWriteCommitLog(cfs) + || cfs.streamToMemtable() + || (session.streamOperation().requiresViewBuild() && hasViews(cfs)); } private void sendThroughWritePath(ColumnFamilyStore cfs, Collection readers) { - boolean hasCdc = hasCDC(cfs); + boolean writeCDCCommitLog = cdcRequiresWriteCommitLog(cfs); ColumnFilter filter = ColumnFilter.all(cfs.metadata()); for (SSTableReader reader : readers) { @@ -208,7 +216,7 @@ private void sendThroughWritePath(ColumnFamilyStore cfs, Collection { + cluster.get(2).runOnInstance(() -> { + boolean containCDCInLog = CommitLog.instance.segmentManager + .getActiveSegments() + .stream() + .anyMatch(s -> s.getCDCState() == CommitLogSegment.CDCState.CONTAINS); + assertTrue("Mutation should be added to commit log when cdc_on_repair_enabled is true", + containCDCInLog); + }); + }); + } + + @Test + public void testCDCOnRepairIsDisabled() throws Exception + { + testCDCOnRepairEnabled(false, cluster -> { + cluster.get(2).runOnInstance(() -> { + boolean containCDCInLog = CommitLog.instance.segmentManager + .getActiveSegments() + .stream() + .allMatch(s -> s.getCDCState() != CommitLogSegment.CDCState.CONTAINS); + assertTrue("No mutation should be added to commit log when cdc_on_repair_enabled is false", + containCDCInLog); + }); + }); + } + + // test helper to repair data between nodes when cdc_on_repair_enabled is on or off. + private void testCDCOnRepairEnabled(boolean enabled, Consumer assertion) throws Exception + { + try (Cluster cluster = init(Cluster.build(2) + .withConfig(c -> c.set("cdc_enabled", true) + .set("cdc_on_repair_enabled", enabled) + .with(Feature.NETWORK) + .with(Feature.GOSSIP)) + .start())) + { + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k INT PRIMARY KEY, v INT) WITH cdc=true")); + + // Data only in node1 + cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.tbl (k, v) VALUES (1, 1)")); + Object[][] result = cluster.get(1).executeInternal(withKeyspace("SELECT * FROM %s.tbl WHERE k = 1")); + assertRows(result, row(1, 1)); + result = cluster.get(2).executeInternal(withKeyspace("SELECT * FROM %s.tbl WHERE k = 1")); + assertRows(result); + + // repair + cluster.get(1).flush(KEYSPACE); + cluster.get(2).nodetool("repair", KEYSPACE, "tbl"); + + // verify node2 now have data + result = cluster.get(2).executeInternal(withKeyspace("SELECT * FROM %s.tbl WHERE k = 1")); + assertRows(result, row(1, 1)); + + assertion.accept(cluster); + } + } +} From a119fee37cf660efd339cfa79a6d2e0867c1a9f6 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 4 Nov 2024 10:18:47 -0800 Subject: [PATCH 052/199] Option to disable write path during streaming for MV enabled tables; similar to CASSANDRA-17666 --- conf/cassandra.yaml | 7 +++++++ src/java/org/apache/cassandra/config/Config.java | 4 ++++ .../apache/cassandra/config/DatabaseDescriptor.java | 10 ++++++++++ .../db/streaming/CassandraStreamReceiver.java | 3 +-- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 8f9ef88b27e8..1df0bb587587 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1686,6 +1686,13 @@ report_unconfirmed_repaired_data_mismatches: false # Materialized views are considered experimental and are not recommended for production use. materialized_views_enabled: false +# Specify whether Materialized View mutations are replayed through the write path on streaming, e.g. repair. +# When enabled, Materialized View data streamed to the destination node will be written into commit log first. When setting to false, +# the streamed Materialized View data is written into SSTables just the same as normal streaming. The default is true. +# If this is set to false, streaming will be considerably faster however it's possible that, in extreme situations +# (losing > quorum # nodes in a replica set), you may have data in your SSTables that never makes it to the Materialized View. +# materialized_views_on_repair_enabled: true + # Enables SASI index creation on this node. # SASI indexes are considered experimental and are not recommended for production use. sasi_indexes_enabled: false diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index b5763ff99e20..965318466726 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -567,6 +567,10 @@ public MemtableOptions() @Replaces(oldName = "enable_materialized_views", converter = Converters.IDENTITY, deprecated = true) public boolean materialized_views_enabled = false; + // When true, materialized views data in SSTable go through commit logs during internodes streaming, e.g. repair + // When false, it behaves the same as normal streaming. + public volatile boolean materialized_views_on_repair_enabled = true; + @Replaces(oldName = "enable_transient_replication", converter = Converters.IDENTITY, deprecated = true) public boolean transient_replication_enabled = false; diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index dc4d48978baf..12964456e2e8 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -3641,6 +3641,16 @@ public static void setMaterializedViewsEnabled(boolean enableMaterializedViews) conf.materialized_views_enabled = enableMaterializedViews; } + public static boolean isMaterializedViewsOnRepairEnabled() + { + return conf.materialized_views_on_repair_enabled; + } + + public static void setMaterializedViewsOnRepairEnabled(boolean val) + { + conf.materialized_views_on_repair_enabled = val; + } + public static boolean getSASIIndexesEnabled() { return conf.sasi_indexes_enabled; diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index 1ca5afd61abe..eab07abf56dd 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -26,7 +26,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; -import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.io.sstable.SSTable; @@ -193,7 +192,7 @@ public boolean requiresWritePath(ColumnFamilyStore cfs) { return cdcRequiresWriteCommitLog(cfs) || cfs.streamToMemtable() - || (session.streamOperation().requiresViewBuild() && hasViews(cfs)); + || (session.streamOperation().requiresViewBuild() && hasViews(cfs) && DatabaseDescriptor.isMaterializedViewsOnRepairEnabled()); } private void sendThroughWritePath(ColumnFamilyStore cfs, Collection readers) From e6cf6317365a7d39fab024a7aeee79d3a00ae3c8 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 4 Nov 2024 11:00:18 -0800 Subject: [PATCH 053/199] cr from tolbertam - replace MV/CDC repair replay JVM properties to yaml configs --- .../config/CassandraRelevantProperties.java | 11 -------- .../cassandra/service/AutoRepairService.java | 9 +++---- .../cassandra/streaming/StreamOperation.java | 4 +-- .../test/repair/AutoRepairSchedulerTest.java | 14 +++------- .../AutoRepairParameterizedTest.java | 8 +++--- .../repair/autorepair/AutoRepairTest.java | 16 +++--------- .../service/AutoRepairServiceTest.java | 26 +++++++------------ 7 files changed, 25 insertions(+), 63 deletions(-) diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java index 380dfb0a3df2..4ee11ec591d4 100644 --- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java +++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java @@ -346,17 +346,6 @@ public enum CassandraRelevantProperties */ GOSSIP_SETTLE_POLL_SUCCESSES_REQUIRED("cassandra.gossip_settle_poll_success_required", "3"), - /** - * If set to true, mutations streamed during anti-entropy repair will be replayed via the regular write path for associated views. - */ - STREAMING_REQUIRES_VIEW_BUILD_DURING_REPAIR("cassandra.streaming.requires_view_build_during_repair", "true"), - - /** - * If set to true, streamed mutations via the regular write path for CDC. - * Deprecate this property in trunk (or 5.0) as a new config has been added to control this https://issues.apache.org/jira/browse/CASSANDRA-17666 - */ - STREAMING_REQUIRES_CDC_REPLAY("cassandra.streaming.requires_cdc_replay", "true"), - ; CassandraRelevantProperties(String key, String defaultVal) diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 43d27ebbb248..7a8342c2a7a4 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.service; -import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.InetAddressAndPort; @@ -65,11 +64,11 @@ public void checkCanRun(RepairType repairType) if (repairType != RepairType.incremental) return; - if (CassandraRelevantProperties.STREAMING_REQUIRES_VIEW_BUILD_DURING_REPAIR.getBoolean()) - throw new ConfigurationException("Cannot run incremental repair while materialized view replay is enabled."); + if (DatabaseDescriptor.isMaterializedViewsOnRepairEnabled()) + throw new ConfigurationException("Cannot run incremental repair while materialized view replay is enabled. Set materialized_views_on_repair_enabled to false."); - if (CassandraRelevantProperties.STREAMING_REQUIRES_CDC_REPLAY.getBoolean()) - throw new ConfigurationException("Cannot run incremental repair while CDC replay is enabled."); + if (DatabaseDescriptor.isCDCOnRepairEnabled()) + throw new ConfigurationException("Cannot run incremental repair while CDC replay is enabled. Set cdc_on_repair_enabled to false."); } @Override diff --git a/src/java/org/apache/cassandra/streaming/StreamOperation.java b/src/java/org/apache/cassandra/streaming/StreamOperation.java index 1e1be1d1be89..8151b47b22b1 100644 --- a/src/java/org/apache/cassandra/streaming/StreamOperation.java +++ b/src/java/org/apache/cassandra/streaming/StreamOperation.java @@ -17,8 +17,6 @@ */ package org.apache.cassandra.streaming; -import org.apache.cassandra.config.CassandraRelevantProperties; - public enum StreamOperation { OTHER("Other"), // Fallback to avoid null types when deserializing from string @@ -28,7 +26,7 @@ public enum StreamOperation BOOTSTRAP("Bootstrap", false), REBUILD("Rebuild", false), BULK_LOAD("Bulk Load"), - REPAIR("Repair", CassandraRelevantProperties.STREAMING_REQUIRES_VIEW_BUILD_DURING_REPAIR.getBoolean()); + REPAIR("Repair"); private final String description; private final boolean requiresViewBuild; diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index 6321523f7fe2..1c95c92225d2 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -26,11 +26,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Uninterruptibles; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.test.TestBaseImpl; @@ -51,9 +51,6 @@ public class AutoRepairSchedulerTest extends TestBaseImpl @BeforeClass public static void init() throws IOException { - System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); - System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); - // Define the expected date format pattern String pattern = "EEE MMM dd HH:mm:ss z yyyy"; // Create SimpleDateFormat object with the given pattern @@ -84,13 +81,6 @@ public static void init() throws IOException cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck text, v1 int, v2 int, PRIMARY KEY (pk, ck)) WITH read_repair='NONE'")); } - @AfterClass - public static void afterClass() - { - System.clearProperty("cassandra.streaming.requires_view_build_during_repair"); - System.clearProperty("cassandra.streaming.requires_cdc_replay"); - } - @Test public void testScheduler() throws ParseException { @@ -101,6 +91,8 @@ public void testScheduler() throws ParseException cluster.forEach(i -> i.runOnInstance(() -> { try { + DatabaseDescriptor.setCDCOnRepairEnabled(false); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); AutoRepairService.instance.setup(); AutoRepair.instance.setup(); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 931fa2dd971d..620a7050a122 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -162,8 +162,8 @@ public static void setupClass() throws Exception @Before public void setup() { - System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); - System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); + DatabaseDescriptor.setCDCOnRepairEnabled(false); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); MockitoAnnotations.initMocks(this); Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).truncateBlocking(); @@ -694,7 +694,7 @@ public void testRepairSuccessAfterRetry() public void testRepairThrowsForIRWithMVReplay() { AutoRepair.instance.setup(); - System.setProperty("cassandra.streaming.requires_view_build_during_repair", "true"); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); if (repairType == AutoRepairConfig.RepairType.incremental) { @@ -718,7 +718,7 @@ public void testRepairThrowsForIRWithMVReplay() public void testRepairThrowsForIRWithCDCReplay() { AutoRepair.instance.setup(); - System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); + DatabaseDescriptor.setCDCOnRepairEnabled(true); if (repairType == AutoRepairConfig.RepairType.incremental) { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index 6987cba87a8b..1e35f4bdbbac 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.Map; -import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -55,19 +54,12 @@ public static void setupClass() throws Exception public void setup() { AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); - System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); - System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); + DatabaseDescriptor.setCDCOnRepairEnabled(false); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.full, true); DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.incremental, true); } - @After - public void after() - { - System.clearProperty("cassandra.streaming.requires_view_build_during_repair"); - System.clearProperty("cassandra.streaming.requires_cdc_replay"); - } - @Test public void testSetup() { @@ -108,7 +100,7 @@ public void testSafeGuardSetupCall() public void testSetupFailsWhenIREnabledWithCDCReplay() { DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.incremental, true); - System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); + DatabaseDescriptor.setCDCOnRepairEnabled(true); DatabaseDescriptor.setCDCEnabled(true); AutoRepair instance = new AutoRepair(); @@ -119,7 +111,7 @@ public void testSetupFailsWhenIREnabledWithCDCReplay() public void testSetupFailsWhenIREnabledWithMVReplay() { DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.incremental, true); - System.setProperty("cassandra.streaming.requires_view_build_during_repair", "true"); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); AutoRepair instance = new AutoRepair(); instance.setup(); } diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index 5da91ecef409..46572e40b6f5 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -28,7 +28,6 @@ import java.util.stream.Stream; import com.google.common.collect.ImmutableSet; -import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -64,8 +63,8 @@ public static class BasicTests extends CQLTester @Before public void setUp() { - System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); - System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); + DatabaseDescriptor.setCDCOnRepairEnabled(false); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); DatabaseDescriptor.setMaterializedViewsEnabled(false); DatabaseDescriptor.setCDCEnabled(false); config = new AutoRepairConfig(); @@ -73,13 +72,6 @@ public void setUp() autoRepairService.config = config; } - @After - public void tearDown() - { - System.clearProperty("cassandra.streaming.requires_view_build_during_repair"); - System.clearProperty("cassandra.streaming.requires_cdc_replay"); - } - @Test public void testSetup() { @@ -134,7 +126,7 @@ public void testSetAutoRepairEnabledThrowsWithSchedulerDisabled() public void testSetAutoRepairEnabledThrowsForIRWithMVReplay() { autoRepairService.config = new AutoRepairConfig(true); - System.setProperty("cassandra.streaming.requires_view_build_during_repair", "true"); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); } @@ -144,8 +136,8 @@ public void testSetAutoRepairEnabledDoesNotThrowForIRWithMVReplayDisabled() { autoRepairService.config = new AutoRepairConfig(true); DatabaseDescriptor.setMaterializedViewsEnabled(true); - System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); - System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); + DatabaseDescriptor.setCDCOnRepairEnabled(false); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); } @@ -154,7 +146,7 @@ public void testSetAutoRepairEnabledDoesNotThrowForIRWithMVReplayDisabled() public void testSetAutoRepairEnabledThrowsForIRWithCDCReplay() { autoRepairService.config = new AutoRepairConfig(true); - System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); + DatabaseDescriptor.setCDCOnRepairEnabled(true); autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); } @@ -164,7 +156,7 @@ public void testSetAutoRepairEnabledDoesNotThrowForIRWithCDCReplayDisabled() { autoRepairService.config = new AutoRepairConfig(true); DatabaseDescriptor.setCDCEnabled(true); - System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); + DatabaseDescriptor.setCDCOnRepairEnabled(false); autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); } @@ -301,8 +293,8 @@ public void prepare() @Test public void testSetters() { - System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); - System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); + DatabaseDescriptor.setCDCOnRepairEnabled(false); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); setter.accept(repairType, arg); assertEquals(arg, getter.apply(repairType)); } From a0b81821a1126675396f97ed98e5657c4a842d00 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Thu, 7 Nov 2024 12:30:28 -0800 Subject: [PATCH 054/199] Implement preview repaired metrics --- .../apache/cassandra/metrics/KeyspaceMetrics.java | 7 +++++++ .../org/apache/cassandra/metrics/TableMetrics.java | 6 ++++++ .../org/apache/cassandra/repair/RepairJob.java | 12 +++++++++++- .../repair/autorepair/AutoRepairConfig.java | 4 +++- .../repair/autorepair/AutoRepairState.java | 2 +- .../repair/consistent/SyncStatSummary.java | 14 +++++++++++--- .../autorepair/AutoRepairStateFactoryTest.java | 2 +- 7 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index 776027e395a1..d85a8846df53 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -125,6 +125,11 @@ public class KeyspaceMetrics public final Histogram bytesValidated; /** histogram over the number of partitions we have validated */ public final Histogram partitionsValidated; + /** histogram over the number of desynchronized token ranges detected during preview repair */ + public final Histogram previewedDesynchronizedTokenRanges; + /** histogram over the number of desynchronized bytes detected during preview repair */ + public final Histogram previewedDesynchronizedBytes; + /* * Metrics for inconsistencies detected between repaired data sets across replicas. These @@ -244,6 +249,8 @@ public KeyspaceMetrics(final Keyspace ks) repairSyncTime = createKeyspaceTimer("RepairSyncTime"); partitionsValidated = createKeyspaceHistogram("PartitionsValidated", false); bytesValidated = createKeyspaceHistogram("BytesValidated", false); + previewedDesynchronizedTokenRanges = createKeyspaceHistogram("PreviewedDesynchronizedTokenRanges", false); + previewedDesynchronizedBytes = createKeyspaceHistogram("PreviewedDesynchronizedBytes", false); confirmedRepairedInconsistencies = createKeyspaceMeter("RepairedDataInconsistenciesConfirmed"); unconfirmedRepairedInconsistencies = createKeyspaceMeter("RepairedDataInconsistenciesUnconfirmed"); diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 24c4b16153e6..1356b60212ca 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -204,6 +204,10 @@ public class TableMetrics public final Counter bytesAnticompacted; /** number of bytes where the whole sstable was contained in a repairing range so that we only mutated the repair status */ public final Counter bytesMutatedAnticompaction; + /** number of desynchronized token ranges that were detected during preview repair */ + public final TableHistogram previewedDesynchronizedTokenRanges; + /** number of desynchronized bytes that were detected during preview repair */ + public final TableHistogram previewedDesynchronizedBytes; /** ratio of how much we anticompact vs how much we could mutate the repair status*/ public final Gauge mutatedAnticompactionGauge; @@ -900,6 +904,8 @@ protected double getDenominator() partitionsValidated = createTableHistogram("PartitionsValidated", cfs.keyspace.metric.partitionsValidated, false); bytesAnticompacted = createTableCounter("BytesAnticompacted"); bytesMutatedAnticompaction = createTableCounter("BytesMutatedAnticompaction"); + previewedDesynchronizedTokenRanges = createTableHistogram("PreviewedDesynchronizedTokenRanges", cfs.keyspace.metric.previewedDesynchronizedTokenRanges, false); + previewedDesynchronizedBytes = createTableHistogram("PreviewedDesynchronizedBytes", cfs.keyspace.metric.previewedDesynchronizedBytes, false); mutatedAnticompactionGauge = createTableGauge("MutatedAnticompactionGauge", () -> { double bytesMutated = bytesMutatedAnticompaction.getCount(); diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java index d6f8820e56f9..7ece7e049bac 100644 --- a/src/java/org/apache/cassandra/repair/RepairJob.java +++ b/src/java/org/apache/cassandra/repair/RepairJob.java @@ -29,9 +29,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.*; import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.repair.consistent.SyncStatSummary; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.repair.state.JobState; +import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.utils.concurrent.AsyncFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -228,7 +230,15 @@ public void onFailure(Throwable t) logger.warn("{} {}.{} sync failed", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily); SystemDistributedKeyspace.failedRepairJob(session.getId(), desc.keyspace, desc.columnFamily, t); } - cfs.metric.repairsCompleted.inc(); + } + else + { + SyncStatSummary.Table summary = new SyncStatSummary.Table(cfs.keyspace.getName(), cfs.getTableName()); + summary.consumeStats(stats); + cfs.metric.previewedDesynchronizedTokenRanges.update(summary.getRanges()); + cfs.metric.previewedDesynchronizedBytes.update(summary.getRanges()); + } + cfs.metric.repairsCompleted.inc(); tryFailure(t instanceof NoSuchRepairSessionExceptionWrapper ? ((NoSuchRepairSessionExceptionWrapper) t).wrapped : t); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 6fc669da642f..38c2d10f4975 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -58,7 +58,7 @@ public enum RepairType implements Serializable { full, incremental, - previewRepaired; + preview_repaired; public static AutoRepairState getAutoRepairState(RepairType repairType) { @@ -68,6 +68,8 @@ public static AutoRepairState getAutoRepairState(RepairType repairType) return new FullRepairState(); case incremental: return new IncrementalRepairState(); + case preview_repaired:: + return new PreviewRepairedState(); } throw new IllegalArgumentException("Invalid repair type: " + repairType); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index c8e4228dfb2c..f0c9fb0b01b6 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -290,7 +290,7 @@ class PreviewRepairedState extends AutoRepairState { public PreviewRepairedState() { - super(RepairType.previewRepaired); + super(RepairType.preview_repaired); } @Override diff --git a/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java index 3d217024c76b..e6fcbe73b418 100644 --- a/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java +++ b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java @@ -76,7 +76,7 @@ public String toString() } } - private static class Table + public static class Table { final String keyspace; @@ -89,7 +89,7 @@ private static class Table final Map, Session> sessions = new HashMap<>(); - Table(String keyspace, String table) + public Table(String keyspace, String table) { this.keyspace = keyspace; this.table = table; @@ -114,7 +114,7 @@ void consumeStat(SyncStat stat) } } - void consumeStats(List stats) + public void consumeStats(List stats) { filter(stats, s -> s.summaries != null).forEach(this::consumeStat); } @@ -154,6 +154,14 @@ public String toString() } return output.toString(); } + + public long getBytes() { + return this.bytes; + } + + public long getRanges() { + return this.ranges; + } } private Map, Table> summaries = new HashMap<>(); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java index 3d8b3e7609ad..c4d961f2a5fc 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java @@ -37,7 +37,7 @@ public void testGetRepairState() { assert state instanceof IncrementalRepairState; - state = RepairType.getAutoRepairState(RepairType.previewRepaired); + state = RepairType.getAutoRepairState(RepairType.preview_repaired); assert state instanceof PreviewRepairedState; } From e9f0ec7905ec5f6f014192a990776e73430f05b7 Mon Sep 17 00:00:00 2001 From: Chris Lohfink Date: Thu, 7 Nov 2024 15:38:16 -0600 Subject: [PATCH 055/199] Add Repair range splitter --- .../statements/schema/TableAttributes.java | 4 +- .../repair/autorepair/AutoRepairConfig.java | 8 +- .../repair/autorepair/AutoRepairState.java | 4 +- .../repair/autorepair/AutoRepairUtils.java | 38 +- .../DefaultAutoRepairTokenSplitter.java | 5 +- .../autorepair/RepairRangeSplitter.java | 396 ++++++++++++++++++ ...nrepairedBytesBasedTokenRangeSplitter.java | 8 +- .../cassandra/schema/AutoRepairParams.java | 4 +- .../cassandra/schema/SchemaKeyspace.java | 8 +- .../apache/cassandra/schema/TableParams.java | 20 +- .../cassandra/service/AutoRepairService.java | 2 +- .../test/repair/AutoRepairSchedulerTest.java | 8 +- .../config/YamlConfigurationLoaderTest.java | 4 +- .../AutoRepairParameterizedTest.java | 28 +- .../AutoRepairStateFactoryTest.java | 4 +- .../repair/autorepair/AutoRepairTest.java | 8 +- .../autorepair/AutoRepairUtilsTest.java | 2 +- .../autorepair/RepairRangeSplitterTest.java | 239 +++++++++++ .../service/AutoRepairServiceTest.java | 10 +- .../tools/nodetool/AutoRepairStatusTest.java | 4 +- 20 files changed, 715 insertions(+), 89 deletions(-) create mode 100644 src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index baa9849707dd..0577eb69cfd4 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -158,10 +158,10 @@ private TableParams build(TableParams.Builder builder) builder.readRepair(ReadRepairStrategy.fromString(getString(Option.READ_REPAIR))); if (hasOption(Option.REPAIR_FULL)) - builder.automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.full, getMap(Option.REPAIR_FULL))); + builder.automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.FULL, getMap(Option.REPAIR_FULL))); if (hasOption(Option.REPAIR_INCREMENTAL)) - builder.automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.incremental, getMap(Option.REPAIR_INCREMENTAL))); + builder.automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.INCREMENTAL, getMap(Option.REPAIR_INCREMENTAL))); return builder.build(); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index a80a875125a9..e1e2d5f2baf2 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -58,16 +58,16 @@ public class AutoRepairConfig implements Serializable public enum RepairType implements Serializable { - full, - incremental; + FULL, + INCREMENTAL; public static AutoRepairState getAutoRepairState(RepairType repairType) { switch (repairType) { - case full: + case FULL: return new FullRepairState(); - case incremental: + case INCREMENTAL: return new IncrementalRepairState(); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index 029690c876a8..d8e3ae6ef274 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -304,7 +304,7 @@ class IncrementalRepairState extends AutoRepairState { public IncrementalRepairState() { - super(RepairType.incremental); + super(RepairType.INCREMENTAL); } @Override @@ -349,7 +349,7 @@ class FullRepairState extends AutoRepairState { public FullRepairState() { - super(RepairType.full); + super(RepairType.FULL); } @Override diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index 14e61003236e..46eb1bcdc1c3 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -19,10 +19,12 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.UUID; @@ -33,6 +35,7 @@ import com.google.common.collect.Lists; import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Splitter; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.LocalStrategy; @@ -70,6 +73,7 @@ import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.utils.NoSpamLogger; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_DUE_TO_PRIORITY; @@ -833,31 +837,19 @@ public static List getAllMVs(RepairType repairType, Keyspace keyspace, T return allMvs; } - public static List> splitEvenly(Range tokenRange, int numberOfSplits) + public static Collection> split(Range tokenRange, int numberOfSplits) { - List> splitRanges = new ArrayList<>(); - long left = (Long) tokenRange.left.getTokenValue(); - long right = (Long) tokenRange.right.getTokenValue(); - long repairTokenWidth = (right - left) / numberOfSplits; - for (int i = 0; i < numberOfSplits; i++) + Collection> ranges; + Optional splitter = DatabaseDescriptor.getPartitioner().splitter(); + if (splitter.isEmpty()) { - long curLeft = left + (i * repairTokenWidth); - long curRight = curLeft + repairTokenWidth; - - if ((i + 1) == numberOfSplits) - { - curRight = right; - } - - Token childStartToken = StorageService.instance.getTokenMetadata() - .partitioner.getTokenFactory().fromString("" + curLeft); - Token childEndToken = StorageService.instance.getTokenMetadata() - .partitioner.getTokenFactory().fromString("" + curRight); - logger.debug("Current Token Left side {}, right side {}", childStartToken - .toString(), childEndToken.toString()); - Range splitRange = new Range<>(childStartToken, childEndToken); - splitRanges.add(splitRange); + NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 30, TimeUnit.MINUTES, "Partitioner {} does not support splitting, falling back to splitting by token range", DatabaseDescriptor.getPartitioner()); + ranges = Collections.singleton(tokenRange); + } + else + { + ranges = splitter.get().split(Collections.singleton(tokenRange), numberOfSplits); } - return splitRanges; + return ranges; } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java index e3f02f6da6f0..6c6b4fe0dad9 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java @@ -29,7 +29,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.service.StorageService; -import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.splitEvenly; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; public class DefaultAutoRepairTokenSplitter implements IAutoRepairTokenRangeSplitter { @@ -48,12 +48,11 @@ public List getRepairAssignments(AutoRepairConfig.RepairType r int numberOfSubranges = config.getRepairSubRangeNum(repairType); boolean byKeyspace = config.getRepairByKeyspace(repairType); - // collect all token ranges. List> allRanges = new ArrayList<>(); for (Range token : tokens) { - allRanges.addAll(splitEvenly(token, numberOfSubranges)); + allRanges.addAll(split(token, numberOfSubranges)); } if (byKeyspace) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java new file mode 100644 index 000000000000..27c4fd0bf31a --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -0,0 +1,396 @@ +package org.apache.cassandra.repair.autorepair; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Iterators; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.clearspring.analytics.stream.cardinality.CardinalityMergeException; +import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; +import com.clearspring.analytics.stream.cardinality.ICardinality; +import org.apache.cassandra.config.DataStorageSpec; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.lifecycle.SSTableIntervalTree; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigTableScanner; +import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.concurrent.Refs; + +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; + +/** + * In Apache Cassandra, tuning the repair ranges has three main goals: + *

+ * 1. **Create smaller, consistent repair times**: Long repairs, such as those lasting 15 hours, can be problematic. + * If a node fails 14 hours into the repair, the entire process must be restarted. The goal is to reduce the impact + * of disturbances or failures. However, making the repairs too short can lead to overhead from repair orchestration + * becoming the main bottleneck. + *

+ * 2. **Minimize the impact on hosts**: Repairs should not heavily affect the host systems. For incremental repairs, + * this might involve anti-compaction work. In full repairs, streaming large amounts of data—especially with wide + * partitions—can lead to issues with disk usage and higher compaction costs. + *

+ * 3. **Reduce overstreaming**: The Merkle tree, which represents data within each partition and range, has a maximum size. + * If a repair covers too many partitions, the tree’s leaves represent larger data ranges. Even a small change in a leaf + * can trigger excessive data streaming, making the process inefficient. + *

+ * Additionally, if there are many small tables, it's beneficial to batch these tables together under a single parent repair. + * This prevents the repair overhead from becoming a bottleneck, especially when dealing with hundreds of tables. Running + * individual repairs for each table can significantly impact performance and efficiency. + *

+ * To manage these issues, the strategy involves estimating the size and number of partitions within a range and splitting + * it accordingly to bound the size of the range splits. + */ + +public class RepairRangeSplitter implements IAutoRepairTokenRangeSplitter +{ + private static final Logger logger = LoggerFactory.getLogger(RepairRangeSplitter.class); + + static final String SUBRANGE_SIZE = "bytes_per_assignment"; + static final String PARTITION_COUNT = "partitions_per_assignment"; + static final String TABLE_BATCH_LIMIT = "max_tables_per_assignment"; + static final String MAX_BYTES_PER_SCHEDULE = "max_bytes_per_schedule"; + + private final int tablesPerAssignmentLimit; + private final long maxBytesPerScheduleBytes; + private final long bytesPerSubrange; + private final long partitionsPerSubrange; + + private static final DataStorageSpec.LongBytesBound DEFAULT_SUBRANGE_SIZE = new DataStorageSpec.LongBytesBound("100GiB"); + private static final long DEFAULT_MAX_BYTES_PER_SCHEDULE = Long.MAX_VALUE; + private static final long DEFAULT_PARTITION_LIMIT = (long) Math.pow(2, DatabaseDescriptor.getRepairSessionMaxTreeDepth()); + private static final int DEFAULT_TABLE_BATCH_LIMIT = 64; + + public RepairRangeSplitter(Map parameters) + { + // Demonstrates parameterizing a range splitter so we can have splitter specific options. + DataStorageSpec.LongBytesBound subrangeSize; + if (parameters.containsKey(SUBRANGE_SIZE)) + { + subrangeSize = new DataStorageSpec.LongBytesBound(parameters.get(SUBRANGE_SIZE)); + } + else + { + subrangeSize = DEFAULT_SUBRANGE_SIZE; + } + bytesPerSubrange = subrangeSize.toBytes(); + + if (parameters.containsKey(MAX_BYTES_PER_SCHEDULE)) + { + maxBytesPerScheduleBytes = new DataStorageSpec.LongBytesBound(parameters.get(MAX_BYTES_PER_SCHEDULE)).toBytes(); + } + else + { + maxBytesPerScheduleBytes = DEFAULT_MAX_BYTES_PER_SCHEDULE; + } + + if (parameters.containsKey(PARTITION_COUNT)) + { + partitionsPerSubrange = Long.parseLong(parameters.get(PARTITION_COUNT)); + } + else + { + partitionsPerSubrange = DEFAULT_PARTITION_LIMIT; + } + + if (parameters.containsKey(TABLE_BATCH_LIMIT)) + { + tablesPerAssignmentLimit = Integer.parseInt(parameters.get(TABLE_BATCH_LIMIT)); + } + else + { + tablesPerAssignmentLimit = DEFAULT_TABLE_BATCH_LIMIT; + } + + logger.info("Configured {} with {}={}, {}={}, {}={}, {}={}", RepairRangeSplitter.class.getName(), + SUBRANGE_SIZE, subrangeSize, MAX_BYTES_PER_SCHEDULE, maxBytesPerScheduleBytes, + PARTITION_COUNT, partitionsPerSubrange, TABLE_BATCH_LIMIT, tablesPerAssignmentLimit); + } + + @Override + public List getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames) + { + logger.debug("Calculating token range splits for repairType={} primaryRangeOnly={} keyspaceName={} tableNames={}", repairType, primaryRangeOnly, keyspaceName, tableNames); + Collection> tokenRanges = getTokenRanges(primaryRangeOnly, keyspaceName); + return getRepairAssignments(repairType, keyspaceName, tableNames, tokenRanges); + } + + @VisibleForTesting + List getRepairAssignments(AutoRepairConfig.RepairType repairType, String keyspaceName, List tableNames, Collection> tokenRanges) + { + List repairAssignments = new ArrayList<>(); + // this is used for batching minimal single assignment tables together + List currentAssignments = new ArrayList<>(); + + for (String tableName : tableNames) + { + List tableAssignments = getRepairAssignmentsForTable(repairType, keyspaceName, tableName, tokenRanges); + + if (tableAssignments.isEmpty()) + continue; + + // If the table assignments are for the same token range and we have room to add more tables to the current assignment + if (tableAssignments.size() == 1 && currentAssignments.size() < tablesPerAssignmentLimit && + (currentAssignments.isEmpty() || currentAssignments.get(0).getTokenRange().equals(tableAssignments.get(0).getTokenRange()))) + { + currentAssignments.addAll(tableAssignments); + } + else + { + if (!currentAssignments.isEmpty()) + { + repairAssignments.add(merge(currentAssignments)); + currentAssignments.clear(); + } + repairAssignments.addAll(tableAssignments); + } + } + if (!currentAssignments.isEmpty()) + repairAssignments.add(merge(currentAssignments)); + return repairAssignments; + } + + @VisibleForTesting + static RepairAssignment merge(List assignments) + { + if (assignments.isEmpty()) + throw new IllegalStateException("Cannot merge empty assignments"); + + Set mergedTableNames = new HashSet<>(); + Range referenceTokenRange = assignments.get(0).getTokenRange(); + String referenceKeyspaceName = assignments.get(0).getKeyspaceName(); + + for (RepairAssignment assignment : assignments) + { + // These checks _should_ be unnecessary but are here to ensure that the assignments are consistent + if (!assignment.getTokenRange().equals(referenceTokenRange)) + throw new IllegalStateException("All assignments must have the same token range"); + if (!assignment.getKeyspaceName().equals(referenceKeyspaceName)) + throw new IllegalStateException("All assignments must have the same keyspace name"); + + mergedTableNames.addAll(assignment.getTableNames()); + } + + return new RepairAssignment(referenceTokenRange, referenceKeyspaceName, new ArrayList<>(mergedTableNames)); + } + + public List getRepairAssignmentsForTable(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName, Collection> tokenRanges) + { + List repairAssignments = new ArrayList<>(); + + long targetBytesSoFar = 0; + + List sizeEstimates = getRangeSizeEstimate(repairType, keyspaceName, tableName, tokenRanges); + // since its possible for us to hit maxBytesPerScheduleBytes before seeing all ranges, shuffle so there is chance + // at least of hitting all the ranges _eventually_ for the worst case scenarios + Collections.shuffle(sizeEstimates); + for (SizeEstimate estimate : sizeEstimates) + { + if (estimate.sizeInRange == 0) + { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); + long memtableSize = cfs.getTracker().getView().getCurrentMemtable().getLiveDataSize(); + if (memtableSize > 0L) + { + logger.debug("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", keyspaceName, tableName, estimate.tokenRange, memtableSize); + RepairAssignment assignment = new RepairAssignment(estimate.tokenRange, keyspaceName, Collections.singletonList(tableName)); + repairAssignments.add(assignment); + } + else + { + logger.debug("Skipping {}.{} for range {} because it had no unrepaired SSTables and no memtable data", keyspaceName, tableName, estimate.tokenRange); + } + } + else if (targetBytesSoFar + estimate.sizeInRange < maxBytesPerScheduleBytes) + { + targetBytesSoFar += estimate.sizeInRange; + // Check if the estimate needs splitting based on the criteria + boolean needsSplitting = estimate.sizeInRange > bytesPerSubrange || estimate.partitions > partitionsPerSubrange; + + if (needsSplitting) + { + // Calculate the number of splits needed for size and partitions + int splitsForSize = (int) Math.ceil((double) estimate.sizeInRange / bytesPerSubrange); + int splitsForPartitions = (int) Math.ceil((double) estimate.partitions / partitionsPerSubrange); + + // Choose the larger of the two as the number of splits + int numberOfSplits = Math.max(splitsForSize, splitsForPartitions); + + // Split the token range into subranges + Collection> subranges = split(estimate.tokenRange, numberOfSplits); + for (Range subrange : subranges) + { + RepairAssignment assignment = new RepairAssignment(subrange, keyspaceName, Collections.singletonList(tableName)); + repairAssignments.add(assignment); + } + } + else + { + // No splitting needed, repair the entire range as-is + RepairAssignment assignment = new RepairAssignment(estimate.tokenRange, keyspaceName, Collections.singletonList(tableName)); + repairAssignments.add(assignment); + } + } + else + { + // Really this is "Ok" but it does mean we are relying on randomness to cover the other ranges + logger.info("Skipping range {} for {}.{} as it would exceed maxBytesPerScheduleBytes, consider increasing maxBytesPerScheduleBytes, reducing node density or monitoring to ensure all ranges do get repaired within gc_grace_seconds", estimate.tokenRange, keyspaceName, tableName); + } + } + return repairAssignments; + } + + public Collection> getTokenRanges(boolean primaryRangeOnly, String keyspaceName) + { + // Collect all applicable token ranges + Collection> wrappedRanges; + if (primaryRangeOnly) + { + wrappedRanges = StorageService.instance.getPrimaryRanges(keyspaceName); + } + else + { + wrappedRanges = StorageService.instance.getLocalRanges(keyspaceName); + } + + // Unwrap each range as we need to account for ranges that overlap the ring + List> ranges = new ArrayList<>(); + for (Range wrappedRange : wrappedRanges) + { + ranges.addAll(wrappedRange.unwrap()); + } + return ranges; + } + + private List getRangeSizeEstimate(AutoRepairConfig.RepairType repairType, String keyspace, String table, Collection> tokenRanges) + { + List sizeEstimates = new ArrayList<>(); + for (Range tokenRange : tokenRanges) + { + logger.debug("Calculating size estimate for {}.{} for range {}", keyspace, table, tokenRange); + try (Refs refs = getSSTableReaderRefs(repairType, keyspace, table, tokenRange)) + { + SizeEstimate estimate = getSizesForRangeOfSSTables(keyspace, table, tokenRange, refs); + logger.debug("Size estimate for {}.{} for range {} is {}", keyspace, table, tokenRange, estimate); + sizeEstimates.add(estimate); + } + } + return sizeEstimates; + } + + private static SizeEstimate getSizesForRangeOfSSTables(String keyspace, String table, Range tokenRange, Refs refs) + { + ICardinality cardinality = new HyperLogLogPlus(13, 25); + long approxBytesInRange = 0L; + long totalBytes = 0L; + + for (SSTableReader reader : refs) + { + try + { + if (reader.openReason == SSTableReader.OpenReason.EARLY) + continue; + CompactionMetadata metadata = (CompactionMetadata) reader.descriptor.getMetadataSerializer().deserialize(reader.descriptor, MetadataType.COMPACTION); + if (metadata != null) + cardinality = cardinality.merge(metadata.cardinalityEstimator); + + long sstableSize = reader.bytesOnDisk(); + totalBytes += sstableSize; + // TODO since reading the index file anyway may be able to get more accurate ratio for partition count, + // still better to use the cardinality estimator then the index since it wont count duplicates. + // get the bounds of the sstable for this range using the index file but do not actually read it. + List> bounds = BigTableScanner.makeBounds(reader, Collections.singleton(tokenRange)); + try (BigTableScanner scanner = (BigTableScanner) BigTableScanner.getScanner(reader, Collections.singleton(tokenRange))) + { + assert bounds.size() == 1; + + AbstractBounds bound = bounds.get(0); + long startPosition = scanner.getDataPosition(bound.left); + long endPosition = scanner.getDataPosition(bound.right); + // If end position is 0 we can assume the sstable ended before that token, bound at size of file + if (endPosition == 0) + { + endPosition = sstableSize; + } + + long approximateRangeBytesInSSTable = Math.max(0, endPosition - startPosition); + approxBytesInRange += Math.min(approximateRangeBytesInSSTable, sstableSize); + } + } + catch (IOException | CardinalityMergeException e) + { + logger.error("Error calculating size estimate for {}.{} for range {} on {}", keyspace, table, tokenRange, reader, e); + } + } + double ratio = approxBytesInRange / (double) totalBytes; + // use the ratio from size to estimate the partitions in the range as well + long partitions = (long) Math.max(1, Math.ceil(cardinality.cardinality() * ratio)); + return new SizeEstimate(keyspace, table, tokenRange, partitions, approxBytesInRange, totalBytes); + } + + private static Refs getSSTableReaderRefs(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName, Range tokenRange) + { + final ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); + + if (cfs == null) + { + throw new IllegalArgumentException(String.format("Could not resolve ColumnFamilyStore from %s.%s", keyspaceName, tableName)); + } + + Refs refs = null; + + while (refs == null) + { + Iterable sstables = cfs.getTracker().getView().select(SSTableSet.CANONICAL); + SSTableIntervalTree tree = SSTableIntervalTree.build(sstables); + Range r = Range.makeRowRange(tokenRange); + List canonicalSSTables = View.sstablesInBounds(r.left, r.right, tree); + if (repairType == AutoRepairConfig.RepairType.INCREMENTAL) + { + canonicalSSTables = canonicalSSTables.stream().filter(SSTableReader::isRepaired).collect(Collectors.toList()); + } + refs = Refs.tryRef(canonicalSSTables); + } + return refs; + } + + @VisibleForTesting + protected static class SizeEstimate + { + public final String keyspace; + public final String table; + public final Range tokenRange; + public final long partitions; + public final long sizeInRange; + public final long totalSize; + + public SizeEstimate(String keyspace, String table, Range tokenRange, long partitions, long sizeInRange, long totalSize) + { + this.keyspace = keyspace; + this.table = table; + this.tokenRange = tokenRange; + this.partitions = partitions; + this.sizeInRange = sizeInRange; + this.totalSize = totalSize; + } + } +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java index c51a37d69779..2ac0813b034e 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java @@ -43,7 +43,7 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.concurrent.Refs; -import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.splitEvenly; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; public class UnrepairedBytesBasedTokenRangeSplitter implements IAutoRepairTokenRangeSplitter { @@ -98,9 +98,9 @@ public List getRepairAssignments(AutoRepairConfig.RepairType r List repairAssignments = new ArrayList<>(); logger.info("Calculating token range splits for repairType={} primaryRangeOnly={} keyspaceName={} tableNames={}", repairType, primaryRangeOnly, keyspaceName, tableNames); - if (repairType != AutoRepairConfig.RepairType.incremental) + if (repairType != AutoRepairConfig.RepairType.INCREMENTAL) { - throw new IllegalArgumentException(this.getClass().getName() + " only supports " + AutoRepairConfig.RepairType.incremental + " repair"); + throw new IllegalArgumentException(this.getClass().getName() + " only supports " + AutoRepairConfig.RepairType.INCREMENTAL + " repair"); } // TODO: create a custom repair assignment that indicates number of bytes in repair and join tables by byte size. @@ -198,7 +198,7 @@ public List getRepairAssignmentsForTable(String keyspaceName, // TODO: approximation per range, this is a bit lossy since targetRanges rounds down. long approximateBytesPerSplit = approximateUnrepairedBytesForRange / targetRanges; logger.info("Splitting {}.{} for range {} into {} sub ranges, approximateBytesPerSplit={}", keyspaceName, tableName, tokenRange, targetRanges, FileUtils.stringifyFileSize(approximateBytesPerSplit)); - List> splitRanges = splitEvenly(tokenRange, (int) targetRanges); + Collection> splitRanges = split(tokenRange, (int) targetRanges); int splitRangeCount = 0; for (Range splitRange : splitRanges) { diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index ae97e562635f..8203411bf0e6 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -44,8 +44,8 @@ public String toString() } public static final Map> DEFAULT_OPTIONS = - ImmutableMap.of(AutoRepairConfig.RepairType.full, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true)), - AutoRepairConfig.RepairType.incremental, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true))); + ImmutableMap.of(AutoRepairConfig.RepairType.FULL, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true)), + AutoRepairConfig.RepairType.INCREMENTAL, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true))); public final AutoRepairConfig.RepairType type; diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index b5c908a18436..673164af9dc0 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -558,8 +558,8 @@ private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBui .add("compression", params.compression.asMap()) .add("read_repair", params.readRepair.toString()) .add("extensions", params.extensions) - .add("repair_full", params.automatedRepair.get(AutoRepairConfig.RepairType.full).asMap()) - .add("repair_incremental", params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()); + .add("repair_full", params.automatedRepair.get(AutoRepairConfig.RepairType.FULL).asMap()) + .add("repair_incremental", params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).asMap()); // Only add CDC-enabled flag to schema if it's enabled on the node. This is to work around RTE's post-8099 if a 3.8+ @@ -984,8 +984,8 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row) SpeculativeRetryPolicy.fromString("99PERCENTILE")) .cdc(row.has("cdc") && row.getBoolean("cdc")) .readRepair(getReadRepairStrategy(row)) - .automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.full, row.getFrozenTextMap("repair_full"))) - .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.incremental, row.getFrozenTextMap("repair_incremental"))) + .automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.FULL, row.getFrozenTextMap("repair_full"))) + .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.INCREMENTAL, row.getFrozenTextMap("repair_incremental"))) .build(); } diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 1132a9ae461a..ac58b9c1a08f 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -115,8 +115,8 @@ private TableParams(Builder builder) automatedRepair = new EnumMap(AutoRepairConfig.RepairType.class) { { - put(AutoRepairConfig.RepairType.full, builder.automatedRepairFull); - put(AutoRepairConfig.RepairType.incremental, builder.automatedRepairIncremental); + put(AutoRepairConfig.RepairType.FULL, builder.automatedRepairFull); + put(AutoRepairConfig.RepairType.INCREMENTAL, builder.automatedRepairIncremental); } }; } @@ -145,8 +145,8 @@ public static Builder builder(TableParams params) .extensions(params.extensions) .cdc(params.cdc) .readRepair(params.readRepair) - .automatedRepairFull(params.automatedRepair.get(AutoRepairConfig.RepairType.full)) - .automatedRepairIncremental(params.automatedRepair.get(AutoRepairConfig.RepairType.incremental)) + .automatedRepairFull(params.automatedRepair.get(AutoRepairConfig.RepairType.FULL)) + .automatedRepairIncremental(params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL)) ; } @@ -286,8 +286,8 @@ public String toString() .add(Option.EXTENSIONS.toString(), extensions) .add(Option.CDC.toString(), cdc) .add(Option.READ_REPAIR.toString(), readRepair) - .add(Option.REPAIR_FULL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.full)) - .add(Option.REPAIR_INCREMENTAL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.incremental)) + .add(Option.REPAIR_FULL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.FULL)) + .add(Option.REPAIR_INCREMENTAL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL)) .toString(); } @@ -337,9 +337,9 @@ public void appendCqlTo(CqlBuilder builder, boolean isView) .newLine() .append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString()) .newLine() - .append("AND repair_full = ").append(automatedRepair.get(AutoRepairConfig.RepairType.full).asMap()) + .append("AND repair_full = ").append(automatedRepair.get(AutoRepairConfig.RepairType.FULL).asMap()) .newLine() - .append("AND repair_incremental = ").append(automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()); + .append("AND repair_incremental = ").append(automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).asMap()); } public static final class Builder @@ -362,8 +362,8 @@ public static final class Builder private boolean cdc; private ReadRepairStrategy readRepair = ReadRepairStrategy.BLOCKING; - private AutoRepairParams automatedRepairFull = new AutoRepairParams(AutoRepairConfig.RepairType.full); - private AutoRepairParams automatedRepairIncremental = new AutoRepairParams(AutoRepairConfig.RepairType.incremental); + private AutoRepairParams automatedRepairFull = new AutoRepairParams(AutoRepairConfig.RepairType.FULL); + private AutoRepairParams automatedRepairIncremental = new AutoRepairParams(AutoRepairConfig.RepairType.INCREMENTAL); public Builder() { diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 43d27ebbb248..029c026ac824 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -62,7 +62,7 @@ public void checkCanRun(RepairType repairType) if (!config.isAutoRepairSchedulingEnabled()) throw new ConfigurationException("Auto-repair scheduller is disabled."); - if (repairType != RepairType.incremental) + if (repairType != RepairType.INCREMENTAL) return; if (CassandraRelevantProperties.STREAMING_REQUIRES_VIEW_BUILD_DURING_REPAIR.getBoolean()) diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index 6321523f7fe2..03d7f124b5fb 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -63,14 +63,14 @@ public static void init() throws IOException .set("auto_repair", ImmutableMap.of( "repair_type_overrides", - ImmutableMap.of(AutoRepairConfig.RepairType.full.toString(), + ImmutableMap.of(AutoRepairConfig.RepairType.FULL.toString(), ImmutableMap.of( "initial_scheduler_delay", "5s", "enabled", "true", "parallel_repair_count", "1", "parallel_repair_percentage", "0", "min_repair_interval", "1s"), - AutoRepairConfig.RepairType.incremental.toString(), + AutoRepairConfig.RepairType.INCREMENTAL.toString(), ImmutableMap.of( "initial_scheduler_delay", "5s", "enabled", "true", @@ -112,8 +112,8 @@ public void testScheduler() throws ParseException // wait for a couple of minutes for repair to go through on all three nodes Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MINUTES); - validate(AutoRepairConfig.RepairType.full.toString()); - validate(AutoRepairConfig.RepairType.incremental.toString()); + validate(AutoRepairConfig.RepairType.FULL.toString()); + validate(AutoRepairConfig.RepairType.INCREMENTAL.toString()); } private void validate(String repairType) throws ParseException diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java index ff7e0d0210dc..ad914c261a78 100644 --- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java +++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java @@ -253,8 +253,8 @@ public void fromMapTest() assertEquals(new DataStorageSpec.IntBytesBound("5B"), config.internode_socket_send_buffer_size); // Check names backward compatibility (CASSANDRA-17141 and CASSANDRA-15234) assertEquals(new DataStorageSpec.IntBytesBound("5B"), config.internode_socket_receive_buffer_size); // Check names backward compatibility (CASSANDRA-17141 and CASSANDRA-15234) assertEquals(true, config.auto_repair.enabled); - assertEquals(new DurationSpec.IntSecondsBound("6h"), config.auto_repair.getAutoRepairTableMaxRepairTime(AutoRepairConfig.RepairType.incremental)); - config.auto_repair.setMVRepairEnabled(AutoRepairConfig.RepairType.incremental, false); + assertEquals(new DurationSpec.IntSecondsBound("6h"), config.auto_repair.getAutoRepairTableMaxRepairTime(AutoRepairConfig.RepairType.INCREMENTAL)); + config.auto_repair.setMVRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, false); } @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 931fa2dd971d..e3fb400bdf11 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -133,8 +133,8 @@ public static void setupClass() throws Exception .addStaticColumn("s", UTF8Type.instance) .addClusteringColumn("i", IntegerType.instance) .addRegularColumn("v", UTF8Type.instance) - .params(TableParams.builder().automatedRepairFull(AutoRepairParams.create(AutoRepairConfig.RepairType.full, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))). - automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.incremental, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))).build()) + .params(TableParams.builder().automatedRepairFull(AutoRepairParams.create(AutoRepairConfig.RepairType.FULL, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))). + automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.INCREMENTAL, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))).build()) .build(); cfmDisabledAutoRepair = TableMetadata.builder(KEYSPACE, TABLE_DISABLED_AUTO_REPAIR) @@ -142,8 +142,8 @@ public static void setupClass() throws Exception .addStaticColumn("s", UTF8Type.instance) .addClusteringColumn("i", IntegerType.instance) .addRegularColumn("v", UTF8Type.instance) - .params(TableParams.builder().automatedRepairFull(AutoRepairParams.create(AutoRepairConfig.RepairType.full, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))). - automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.incremental, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))).build()) + .params(TableParams.builder().automatedRepairFull(AutoRepairParams.create(AutoRepairConfig.RepairType.FULL, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))). + automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.INCREMENTAL, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))).build()) .build(); SchemaLoader.prepareServer(); @@ -544,10 +544,10 @@ public void testRepairWaitsForRepairToFinishBeforeSchedullingNewSession() throws @Test public void testDisabledAutoRepairForATableThroughTableLevelConfiguration() { - Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); - Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).repairEnabled()); - Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); - Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).repairEnabled()); + Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.FULL).repairEnabled()); + Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).repairEnabled()); + Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.FULL).repairEnabled()); + Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).repairEnabled()); AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setRepairMinInterval(repairType, "0s"); @@ -584,10 +584,10 @@ public void testTableAttribute() @Test public void testDefaultAutomatedRepair() { - Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); - Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).repairEnabled()); - Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); - Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).repairEnabled()); + Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.FULL).repairEnabled()); + Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).repairEnabled()); + Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.FULL).repairEnabled()); + Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).repairEnabled()); } @Test @@ -696,7 +696,7 @@ public void testRepairThrowsForIRWithMVReplay() AutoRepair.instance.setup(); System.setProperty("cassandra.streaming.requires_view_build_during_repair", "true"); - if (repairType == AutoRepairConfig.RepairType.incremental) + if (repairType == AutoRepairConfig.RepairType.INCREMENTAL) { try { @@ -720,7 +720,7 @@ public void testRepairThrowsForIRWithCDCReplay() AutoRepair.instance.setup(); System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); - if (repairType == AutoRepairConfig.RepairType.incremental) + if (repairType == AutoRepairConfig.RepairType.INCREMENTAL) { try { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java index a0e5bdc45294..1de6d5e78607 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java @@ -29,11 +29,11 @@ public class AutoRepairStateFactoryTest { @Test public void testGetRepairState() { - AutoRepairState state = RepairType.getAutoRepairState(RepairType.full); + AutoRepairState state = RepairType.getAutoRepairState(RepairType.FULL); assert state instanceof FullRepairState; - state = RepairType.getAutoRepairState(RepairType.incremental); + state = RepairType.getAutoRepairState(RepairType.INCREMENTAL); assert state instanceof IncrementalRepairState; } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index 6987cba87a8b..8623c4c4ef2f 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -57,8 +57,8 @@ public void setup() AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); - DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.full, true); - DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.incremental, true); + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.FULL, true); + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.INCREMENTAL, true); } @After @@ -107,7 +107,7 @@ public void testSafeGuardSetupCall() @Test(expected = ConfigurationException.class) public void testSetupFailsWhenIREnabledWithCDCReplay() { - DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.incremental, true); + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.INCREMENTAL, true); System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); DatabaseDescriptor.setCDCEnabled(true); @@ -118,7 +118,7 @@ public void testSetupFailsWhenIREnabledWithCDCReplay() @Test(expected = ConfigurationException.class) public void testSetupFailsWhenIREnabledWithMVReplay() { - DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.incremental, true); + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.INCREMENTAL, true); System.setProperty("cassandra.streaming.requires_view_build_during_repair", "true"); AutoRepair instance = new AutoRepair(); instance.setup(); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index afdbe0a6d825..38110e903c08 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -70,7 +70,7 @@ public class AutoRepairUtilsTest extends CQLTester { - static RepairType repairType = RepairType.incremental; + static RepairType repairType = RepairType.INCREMENTAL; static UUID hostId; static InetAddressAndPort localEndpoint; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java new file mode 100644 index 000000000000..f1367aad10f1 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter.RepairAssignment; + +import static org.apache.cassandra.repair.autorepair.RepairRangeSplitter.TABLE_BATCH_LIMIT; +import static org.junit.Assert.assertEquals; + +public class RepairRangeSplitterTest extends CQLTester +{ + private RepairRangeSplitter repairRangeSplitter; + private String tableName; + private static Range fullRange; + + @BeforeClass + public static void setUpClass() + { + CQLTester.setUpClass(); + fullRange = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken()); + } + + @Before + public void setUp() { + repairRangeSplitter = new RepairRangeSplitter(Collections.emptyMap()); + tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); + } + + @Test + public void testGetRepairAssignmentsForTable_NoSSTables() { + Collection> ranges = Collections.singleton(new Range<>(Murmur3Partitioner.instance.getMinimumToken(), Murmur3Partitioner.instance.getMaximumToken())); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(AutoRepairConfig.RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); + assertEquals(0, assignments.size()); + } + + @Test + public void testGetRepairAssignmentsForTable_Single() throws Throwable { + Collection> ranges = Collections.singleton(new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken())); + insertAndFlushSingleTable(tableName); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(AutoRepairConfig.RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); + assertEquals(1, assignments.size()); + } + + @Test + public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable { + repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "2")); + Collection> ranges = Collections.singleton(fullRange); + + List tableNames = createAndInsertTables(3); + List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + + // We expect two assignments, one with table1 and table2 batched, and one with table3 + assertEquals(2, assignments.size()); + assertEquals(2, assignments.get(0).getTableNames().size()); + assertEquals(1, assignments.get(1).getTableNames().size()); + } + + @Test + public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable { + repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "2")); + Collection> ranges = Collections.singleton(fullRange); + + List tableNames = createAndInsertTables(2); + List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + + // We expect one assignment, with two tables batched + assertEquals(1, assignments.size()); + assertEquals(2, assignments.get(0).getTableNames().size()); + } + + @Test + public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable { + repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "1")); + Collection> ranges = Collections.singleton(fullRange); + + List tableNames = createAndInsertTables(3); + List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + + assertEquals(3, assignments.size()); + } + + @Test + public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable { + repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "100")); + Collection> ranges = Collections.singleton(fullRange); + + List tableNames = createAndInsertTables(5); + List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + + assertEquals(1, assignments.size()); + } + + + + @Test(expected = IllegalStateException.class) + public void testMergeEmptyAssignments() { + // Test when the list of assignments is empty + List emptyAssignments = Collections.emptyList(); + RepairRangeSplitter.merge(emptyAssignments); + } + + @Test + public void testMergeSingleAssignment() { + // Test when there is only one assignment in the list + String keyspaceName = "testKeyspace"; + List tableNames = Arrays.asList("table1", "table2"); + + RepairAssignment assignment = new RepairAssignment(fullRange, keyspaceName, tableNames); + List assignments = Collections.singletonList(assignment); + + RepairAssignment result = RepairRangeSplitter.merge(assignments); + + assertEquals(fullRange, result.getTokenRange()); + assertEquals(keyspaceName, result.getKeyspaceName()); + assertEquals(new HashSet<>(tableNames), new HashSet<>(result.getTableNames())); + } + + @Test + public void testMergeMultipleAssignmentsWithSameTokenRangeAndKeyspace() { + // Test merging multiple assignments with the same token range and keyspace + String keyspaceName = "testKeyspace"; + List tableNames1 = Arrays.asList("table1", "table2"); + List tableNames2 = Arrays.asList("table2", "table3"); + + RepairAssignment assignment1 = new RepairAssignment(fullRange, keyspaceName, tableNames1); + RepairAssignment assignment2 = new RepairAssignment(fullRange, keyspaceName, tableNames2); + List assignments = Arrays.asList(assignment1, assignment2); + + RepairAssignment result = RepairRangeSplitter.merge(assignments); + + assertEquals(fullRange, result.getTokenRange()); + assertEquals(keyspaceName, result.getKeyspaceName()); + assertEquals(new HashSet<>(Arrays.asList("table1", "table2", "table3")), new HashSet<>(result.getTableNames())); + } + + @Test(expected = IllegalStateException.class) + public void testMergeDifferentTokenRange() { + // Test merging assignments with different token ranges + Iterator> range = AutoRepairUtils.split(fullRange, 2).iterator(); // Split the full range into two ranges ie (0-100, 100-200 + Range tokenRange1 = range.next(); + Range tokenRange2 = range.next(); + Assert.assertFalse(range.hasNext()); + + String keyspaceName = "testKeyspace"; + List tableNames = Arrays.asList("table1", "table2"); + + RepairAssignment assignment1 = new RepairAssignment(tokenRange1, keyspaceName, tableNames); + RepairAssignment assignment2 = new RepairAssignment(tokenRange2, keyspaceName, tableNames); + List assignments = Arrays.asList(assignment1, assignment2); + + RepairRangeSplitter.merge(assignments); // Should throw IllegalStateException + } + + @Test(expected = IllegalStateException.class) + public void testMergeDifferentKeyspaceName() { + // Test merging assignments with different keyspace names + List tableNames = Arrays.asList("table1", "table2"); + + RepairAssignment assignment1 = new RepairAssignment(fullRange, "keyspace1", tableNames); + RepairAssignment assignment2 = new RepairAssignment(fullRange, "keyspace2", tableNames); + List assignments = Arrays.asList(assignment1, assignment2); + + RepairRangeSplitter.merge(assignments); // Should throw IllegalStateException + } + + @Test + public void testMergeWithDuplicateTables() { + // Test merging assignments with duplicate table names + String keyspaceName = "testKeyspace"; + List tableNames1 = Arrays.asList("table1", "table2"); + List tableNames2 = Arrays.asList("table2", "table3"); + + RepairAssignment assignment1 = new RepairAssignment(fullRange, keyspaceName, tableNames1); + RepairAssignment assignment2 = new RepairAssignment(fullRange, keyspaceName, tableNames2); + List assignments = Arrays.asList(assignment1, assignment2); + + RepairAssignment result = RepairRangeSplitter.merge(assignments); + + // The merged result should contain all unique table names + assertEquals(new HashSet<>(Arrays.asList("table1", "table2", "table3")), new HashSet<>(result.getTableNames())); + } + + private void insertAndFlushSingleTable(String tableName) throws Throwable { + execute("INSERT INTO %s (k, v) values (?, ?)", 1, 1); + flush(); + } + + private List createAndInsertTables(int count) throws Throwable { + List tableNames = new ArrayList<>(); + for (int i = 0; i < count; i++) { + String tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); + tableNames.add(tableName); + insertAndFlushTable(tableName); + } + return tableNames; + } + + private void insertAndFlushTable(String tableName) throws Throwable { + executeFormattedQuery("INSERT INTO " + KEYSPACE + '.' + tableName + " (k, v) values (?, ?)", 1, 1); + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, tableName); + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + assertEquals(1, cfs.getLiveSSTables().size()); + } +} diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index 5da91ecef409..34c9af3427bf 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -127,7 +127,7 @@ public void testSetAutoRepairEnabledThrowsWithSchedulerDisabled() { autoRepairService.config = new AutoRepairConfig(false); - autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); } @Test(expected = ConfigurationException.class) @@ -136,7 +136,7 @@ public void testSetAutoRepairEnabledThrowsForIRWithMVReplay() autoRepairService.config = new AutoRepairConfig(true); System.setProperty("cassandra.streaming.requires_view_build_during_repair", "true"); - autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); } @Test @@ -147,7 +147,7 @@ public void testSetAutoRepairEnabledDoesNotThrowForIRWithMVReplayDisabled() System.setProperty("cassandra.streaming.requires_view_build_during_repair", "false"); System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); - autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); } @Test(expected = ConfigurationException.class) @@ -156,7 +156,7 @@ public void testSetAutoRepairEnabledThrowsForIRWithCDCReplay() autoRepairService.config = new AutoRepairConfig(true); System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); - autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); } @Test @@ -166,7 +166,7 @@ public void testSetAutoRepairEnabledDoesNotThrowForIRWithCDCReplayDisabled() DatabaseDescriptor.setCDCEnabled(true); System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); - autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); } } diff --git a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java index 16b12ee2350e..f1faabc0fc03 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java @@ -70,8 +70,8 @@ public void setUp() throws Exception DatabaseDescriptor.daemonInitialization(); DatabaseDescriptor.loadConfig(); setAutoRepairEnabled(true); - DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(AutoRepairConfig.RepairType.full, true); - DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(AutoRepairConfig.RepairType.incremental, true); + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(AutoRepairConfig.RepairType.FULL, true); + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); when(probe.getAutoRepairConfig()).thenReturn(DatabaseDescriptor.getAutoRepairConfig()); } From 06e13cae8f08d1bd631e3d371133284bc6834811 Mon Sep 17 00:00:00 2001 From: Chris Lohfink Date: Thu, 7 Nov 2024 21:01:45 -0600 Subject: [PATCH 056/199] add assignment priorirty to table options --- .../IAutoRepairTokenRangeSplitter.java | 33 +++++++ .../autorepair/RepairRangeSplitter.java | 14 ++- .../cassandra/schema/AutoRepairParams.java | 27 +++++- .../autorepair/RepairRangeSplitterTest.java | 91 +++++++++++++++++++ 4 files changed, 159 insertions(+), 6 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java index dd02c4d4f7b6..7e679b8dbc9d 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java @@ -18,11 +18,17 @@ package org.apache.cassandra.repair.autorepair; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.schema.AutoRepairParams; public interface IAutoRepairTokenRangeSplitter { @@ -38,6 +44,21 @@ public interface IAutoRepairTokenRangeSplitter */ List getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames); + /** + * Reorders the list of {@link RepairAssignment} objects based on their priority for a given repair type. + * The list is sorted in descending order, so higher priority assignments appear first. + * If two assignments have the same priority for the specified repair type, their original order is preserved. + * + * @param repairAssignments A list of {@link RepairAssignment} objects to be reordered. + * @param repairType The {@link AutoRepairConfig.RepairType} used to determine the priority of each assignment. + * The priority is determined using the {@link RepairAssignment#getPriority(AutoRepairConfig.RepairType)} method. + */ + @VisibleForTesting + default void reorderByPriority(List repairAssignments, AutoRepairConfig.RepairType repairType) + { + repairAssignments.sort(Comparator.comparingInt(a -> ((RepairAssignment) a).getPriority(repairType)).reversed()); + } + /** * Defines a repair assignment to be issued by the autorepair framework. */ @@ -71,6 +92,18 @@ public List getTableNames() return tableNames; } + public int getPriority(AutoRepairConfig.RepairType type) + { + int max = 0; + for (String table : tableNames) + { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, table); + if (cfs != null) + max = Math.max(max, cfs.metadata().params.automatedRepair.get(type).priority()); + } + return max; + } + @Override public boolean equals(Object o) { diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 27c4fd0bf31a..b0b1add41564 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -11,7 +12,6 @@ import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Iterators; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -141,6 +141,16 @@ List getRepairAssignments(AutoRepairConfig.RepairType repairTy // this is used for batching minimal single assignment tables together List currentAssignments = new ArrayList<>(); + // sort the tables by size so can batch the smallest ones together + tableNames.sort((t1, t2) -> { + ColumnFamilyStore cfs1 = ColumnFamilyStore.getIfExists(keyspaceName, t1); + ColumnFamilyStore cfs2 = ColumnFamilyStore.getIfExists(keyspaceName, t2); + if (cfs1 == null || cfs2 == null) + { + throw new IllegalArgumentException(String.format("Could not resolve ColumnFamilyStore from %s.%s", keyspaceName, t1)); + } + return Long.compare(cfs1.metric.totalDiskSpaceUsed.getCount(), cfs2.metric.totalDiskSpaceUsed.getCount()); + }); for (String tableName : tableNames) { List tableAssignments = getRepairAssignmentsForTable(repairType, keyspaceName, tableName, tokenRanges); @@ -166,6 +176,8 @@ List getRepairAssignments(AutoRepairConfig.RepairType repairTy } if (!currentAssignments.isEmpty()) repairAssignments.add(merge(currentAssignments)); + + reorderByPriority(repairAssignments, repairType); return repairAssignments; } diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index 8203411bf0e6..b56fb8ed4d83 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -17,7 +17,10 @@ */ package org.apache.cassandra.schema; +import java.util.Arrays; +import java.util.EnumMap; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Objects; @@ -34,7 +37,8 @@ public final class AutoRepairParams { public enum Option { - ENABLED; + ENABLED, + PRIORITY; @Override public String toString() @@ -43,9 +47,14 @@ public String toString() } } + public static final Map DEFAULT_SUB_OPTIONS = ImmutableMap.of( + Option.ENABLED.toString(), Boolean.toString(true), + Option.PRIORITY.toString(), "0" + ); + public static final Map> DEFAULT_OPTIONS = - ImmutableMap.of(AutoRepairConfig.RepairType.FULL, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true)), - AutoRepairConfig.RepairType.INCREMENTAL, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true))); + ImmutableMap.of(AutoRepairConfig.RepairType.FULL, DEFAULT_SUB_OPTIONS, + AutoRepairConfig.RepairType.INCREMENTAL, DEFAULT_SUB_OPTIONS); public final AutoRepairConfig.RepairType type; @@ -58,7 +67,7 @@ public String toString() public static AutoRepairParams create(AutoRepairConfig.RepairType repairType, Map options) { - Map> optionsMap = new HashMap<>(); + Map> optionsMap = new EnumMap<>(AutoRepairConfig.RepairType.class); for (Map.Entry> entry : DEFAULT_OPTIONS.entrySet()) { optionsMap.put(entry.getKey(), new HashMap<>(entry.getValue())); @@ -67,7 +76,7 @@ public static AutoRepairParams create(AutoRepairConfig.RepairType repairType, Ma { for (Map.Entry entry : options.entrySet()) { - if (!Option.ENABLED.toString().equals(entry.getKey().toLowerCase())) + if (Arrays.stream(Option.values()).noneMatch(option -> option.toString().equalsIgnoreCase(entry.getKey()))) { throw new ConfigurationException(format("Unknown property '%s'", entry.getKey())); } @@ -87,6 +96,14 @@ public boolean repairEnabled() : Boolean.parseBoolean(enabled); } + public int priority() + { + String priority = options.get(type).get(Option.PRIORITY.toString()); + return priority == null + ? Integer.parseInt(DEFAULT_OPTIONS.get(type).get(Option.PRIORITY.toString())) + : Integer.parseInt(priority); + } + public void validate() { String enabled = options.get(type).get(Option.ENABLED.toString()); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java index f1367aad10f1..3413c5171a74 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java @@ -41,6 +41,7 @@ import static org.apache.cassandra.repair.autorepair.RepairRangeSplitter.TABLE_BATCH_LIMIT; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class RepairRangeSplitterTest extends CQLTester { @@ -61,6 +62,95 @@ public void setUp() { tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); } + @Test + public void testReorderByPriorityWithDifferentPriorities() { + String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '3'}"); + String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); + + // Test reordering assignments with different priorities + RepairAssignment assignment1 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table1)); + RepairAssignment assignment2 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table2)); + RepairAssignment assignment3 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table3)); + + // Assume these priorities based on the repair type + List assignments = new ArrayList<>(Arrays.asList(assignment1, assignment2, assignment3)); + + repairRangeSplitter.reorderByPriority(assignments, AutoRepairConfig.RepairType.FULL); + + // Verify the order is by descending priority + assertEquals(assignment2, assignments.get(0)); + assertEquals(assignment1, assignments.get(1)); + assertEquals(assignment3, assignments.get(2)); + } + + @Test + public void testReorderByPriorityWithSamePriority() { + String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + + // Test reordering assignments with the same priority + RepairAssignment assignment1 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table1)); + RepairAssignment assignment2 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table2)); + RepairAssignment assignment3 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table3)); + + List assignments = new ArrayList<>(Arrays.asList(assignment1, assignment2, assignment3)); + + repairRangeSplitter.reorderByPriority(assignments, AutoRepairConfig.RepairType.FULL); + + // Verify the original order is preserved as all priorities are the same + assertEquals(assignment1, assignments.get(0)); + assertEquals(assignment2, assignments.get(1)); + assertEquals(assignment3, assignments.get(2)); + } + + @Test + public void testReorderByPriorityWithMixedPriorities() { + String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '3'}"); + String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table4 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); + + // Test reordering assignments with mixed priorities + RepairAssignment assignment1 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table1)); + RepairAssignment assignment2 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table2)); + RepairAssignment assignment3 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table3)); + RepairAssignment assignment4 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table4)); + + List assignments = new ArrayList<>(Arrays.asList(assignment1, assignment2, assignment3, assignment4)); + + repairRangeSplitter.reorderByPriority(assignments, AutoRepairConfig.RepairType.FULL); + + // Verify the order: highest priority first, then preserved order for same priority + assertEquals(assignment2, assignments.get(0)); // Priority 3 + assertEquals(assignment1, assignments.get(1)); // Priority 2 + assertEquals(assignment3, assignments.get(2)); // Priority 2 + assertEquals(assignment4, assignments.get(3)); // Priority 1 + } + + @Test + public void testReorderByPriorityWithEmptyList() { + // Test with an empty list (should remain empty) + List assignments = new ArrayList<>(); + repairRangeSplitter.reorderByPriority(assignments, AutoRepairConfig.RepairType.FULL); + assertTrue(assignments.isEmpty()); + } + + @Test + public void testReorderByPriorityWithOneElement() { + // Test with a single element (should remain unchanged) + String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '5'}"); + + RepairAssignment assignment1 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table1)); + + List assignments = new ArrayList<>(Collections.singletonList(assignment1)); + + repairRangeSplitter.reorderByPriority(assignments, AutoRepairConfig.RepairType.FULL); + + assertEquals(assignment1, assignments.get(0)); // Single element should remain in place + } + @Test public void testGetRepairAssignmentsForTable_NoSSTables() { Collection> ranges = Collections.singleton(new Range<>(Murmur3Partitioner.instance.getMinimumToken(), Murmur3Partitioner.instance.getMaximumToken())); @@ -215,6 +305,7 @@ public void testMergeWithDuplicateTables() { assertEquals(new HashSet<>(Arrays.asList("table1", "table2", "table3")), new HashSet<>(result.getTableNames())); } + private void insertAndFlushSingleTable(String tableName) throws Throwable { execute("INSERT INTO %s (k, v) values (?, ?)", 1, 1); flush(); From a8979c0917e0bdfce807e984f0c3e831a60ab848 Mon Sep 17 00:00:00 2001 From: Chris Lohfink Date: Thu, 7 Nov 2024 21:03:05 -0600 Subject: [PATCH 057/199] reorder with DefaultAutoRepairTokenSplitter --- .../repair/autorepair/DefaultAutoRepairTokenSplitter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java index 6c6b4fe0dad9..d34c02c2c710 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java @@ -74,6 +74,7 @@ public List getRepairAssignments(AutoRepairConfig.RepairType r } } } + reorderByPriority(repairAssignments, repairType); return repairAssignments; } } \ No newline at end of file From 8ff99745382f11b1de37f780e67f2541e86f7815 Mon Sep 17 00:00:00 2001 From: Chris Lohfink Date: Thu, 7 Nov 2024 22:48:30 -0600 Subject: [PATCH 058/199] add tests for partition count estimation --- .../autorepair/RepairRangeSplitter.java | 8 +- .../autorepair/RepairRangeSplitterTest.java | 98 +++++++++++++------ 2 files changed, 71 insertions(+), 35 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index b0b1add41564..0c66320cc6b8 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -146,9 +146,7 @@ List getRepairAssignments(AutoRepairConfig.RepairType repairTy ColumnFamilyStore cfs1 = ColumnFamilyStore.getIfExists(keyspaceName, t1); ColumnFamilyStore cfs2 = ColumnFamilyStore.getIfExists(keyspaceName, t2); if (cfs1 == null || cfs2 == null) - { throw new IllegalArgumentException(String.format("Could not resolve ColumnFamilyStore from %s.%s", keyspaceName, t1)); - } return Long.compare(cfs1.metric.totalDiskSpaceUsed.getCount(), cfs2.metric.totalDiskSpaceUsed.getCount()); }); for (String tableName : tableNames) @@ -309,7 +307,8 @@ private List getRangeSizeEstimate(AutoRepairConfig.RepairType repa return sizeEstimates; } - private static SizeEstimate getSizesForRangeOfSSTables(String keyspace, String table, Range tokenRange, Refs refs) + @VisibleForTesting + static SizeEstimate getSizesForRangeOfSSTables(String keyspace, String table, Range tokenRange, Refs refs) { ICardinality cardinality = new HyperLogLogPlus(13, 25); long approxBytesInRange = 0L; @@ -359,7 +358,8 @@ private static SizeEstimate getSizesForRangeOfSSTables(String keyspace, String t return new SizeEstimate(keyspace, table, tokenRange, partitions, approxBytesInRange, totalBytes); } - private static Refs getSSTableReaderRefs(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName, Range tokenRange) + @VisibleForTesting + static Refs getSSTableReaderRefs(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName, Range tokenRange) { final ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java index 3413c5171a74..aa81e75300cf 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java @@ -37,7 +37,9 @@ import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter.RepairAssignment; +import org.apache.cassandra.utils.concurrent.Refs; import static org.apache.cassandra.repair.autorepair.RepairRangeSplitter.TABLE_BATCH_LIMIT; import static org.junit.Assert.assertEquals; @@ -47,13 +49,13 @@ public class RepairRangeSplitterTest extends CQLTester { private RepairRangeSplitter repairRangeSplitter; private String tableName; - private static Range fullRange; + private static Range FULL_RANGE; @BeforeClass public static void setUpClass() { CQLTester.setUpClass(); - fullRange = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken()); + FULL_RANGE = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken()); } @Before @@ -62,6 +64,36 @@ public void setUp() { tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); } + @Test + public void testSizePartitionCount() throws Throwable + { + insertAndFlushTable(tableName, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10); + Refs sstables = RepairRangeSplitter.getSSTableReaderRefs(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, FULL_RANGE); + assertEquals(10, sstables.iterator().next().getEstimatedPartitionSize().count()); + RepairRangeSplitter.SizeEstimate sizes = RepairRangeSplitter.getSizesForRangeOfSSTables(KEYSPACE, tableName, FULL_RANGE, sstables); + assertEquals(10, sizes.partitions); + } + + @Test + public void testSizePartitionCountSplit() throws Throwable + { + int[] values = new int[10000]; + for (int i = 0; i < values.length; i++) + values[i] = i + 1; + insertAndFlushTable(tableName, values); + Iterator> range = AutoRepairUtils.split(FULL_RANGE, 2).iterator(); + Range tokenRange1 = range.next(); + Range tokenRange2 = range.next(); + Assert.assertFalse(range.hasNext()); + + Refs sstables1 = RepairRangeSplitter.getSSTableReaderRefs(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, tokenRange1); + Refs sstables2 = RepairRangeSplitter.getSSTableReaderRefs(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, tokenRange2); + RepairRangeSplitter.SizeEstimate sizes1 = RepairRangeSplitter.getSizesForRangeOfSSTables(KEYSPACE, tableName, tokenRange1, sstables1); + RepairRangeSplitter.SizeEstimate sizes2 = RepairRangeSplitter.getSizesForRangeOfSSTables(KEYSPACE, tableName, tokenRange2, sstables2); + // +-5% because HLL merge and the applying of range size approx ratio causes estimation errors + assertTrue(Math.abs(10000 - (sizes1.partitions + sizes2.partitions)) <= 60); + } + @Test public void testReorderByPriorityWithDifferentPriorities() { String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); @@ -69,9 +101,9 @@ public void testReorderByPriorityWithDifferentPriorities() { String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); // Test reordering assignments with different priorities - RepairAssignment assignment1 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table1)); - RepairAssignment assignment2 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table2)); - RepairAssignment assignment3 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table3)); + RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table1)); + RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table2)); + RepairAssignment assignment3 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table3)); // Assume these priorities based on the repair type List assignments = new ArrayList<>(Arrays.asList(assignment1, assignment2, assignment3)); @@ -91,9 +123,9 @@ public void testReorderByPriorityWithSamePriority() { String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); // Test reordering assignments with the same priority - RepairAssignment assignment1 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table1)); - RepairAssignment assignment2 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table2)); - RepairAssignment assignment3 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table3)); + RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table1)); + RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table2)); + RepairAssignment assignment3 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table3)); List assignments = new ArrayList<>(Arrays.asList(assignment1, assignment2, assignment3)); @@ -113,10 +145,10 @@ public void testReorderByPriorityWithMixedPriorities() { String table4 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); // Test reordering assignments with mixed priorities - RepairAssignment assignment1 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table1)); - RepairAssignment assignment2 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table2)); - RepairAssignment assignment3 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table3)); - RepairAssignment assignment4 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table4)); + RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table1)); + RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table2)); + RepairAssignment assignment3 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table3)); + RepairAssignment assignment4 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table4)); List assignments = new ArrayList<>(Arrays.asList(assignment1, assignment2, assignment3, assignment4)); @@ -142,7 +174,7 @@ public void testReorderByPriorityWithOneElement() { // Test with a single element (should remain unchanged) String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '5'}"); - RepairAssignment assignment1 = new RepairAssignment(fullRange, KEYSPACE, Collections.singletonList(table1)); + RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table1)); List assignments = new ArrayList<>(Collections.singletonList(assignment1)); @@ -169,7 +201,7 @@ public void testGetRepairAssignmentsForTable_Single() throws Throwable { @Test public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable { repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "2")); - Collection> ranges = Collections.singleton(fullRange); + Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); @@ -183,7 +215,7 @@ public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable { @Test public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable { repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "2")); - Collection> ranges = Collections.singleton(fullRange); + Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(2); List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); @@ -196,7 +228,7 @@ public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable { @Test public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable { repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "1")); - Collection> ranges = Collections.singleton(fullRange); + Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); @@ -207,7 +239,7 @@ public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable { @Test public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable { repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "100")); - Collection> ranges = Collections.singleton(fullRange); + Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(5); List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); @@ -215,8 +247,6 @@ public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable { assertEquals(1, assignments.size()); } - - @Test(expected = IllegalStateException.class) public void testMergeEmptyAssignments() { // Test when the list of assignments is empty @@ -230,12 +260,12 @@ public void testMergeSingleAssignment() { String keyspaceName = "testKeyspace"; List tableNames = Arrays.asList("table1", "table2"); - RepairAssignment assignment = new RepairAssignment(fullRange, keyspaceName, tableNames); + RepairAssignment assignment = new RepairAssignment(FULL_RANGE, keyspaceName, tableNames); List assignments = Collections.singletonList(assignment); RepairAssignment result = RepairRangeSplitter.merge(assignments); - assertEquals(fullRange, result.getTokenRange()); + assertEquals(FULL_RANGE, result.getTokenRange()); assertEquals(keyspaceName, result.getKeyspaceName()); assertEquals(new HashSet<>(tableNames), new HashSet<>(result.getTableNames())); } @@ -247,13 +277,13 @@ public void testMergeMultipleAssignmentsWithSameTokenRangeAndKeyspace() { List tableNames1 = Arrays.asList("table1", "table2"); List tableNames2 = Arrays.asList("table2", "table3"); - RepairAssignment assignment1 = new RepairAssignment(fullRange, keyspaceName, tableNames1); - RepairAssignment assignment2 = new RepairAssignment(fullRange, keyspaceName, tableNames2); + RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, keyspaceName, tableNames1); + RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, keyspaceName, tableNames2); List assignments = Arrays.asList(assignment1, assignment2); RepairAssignment result = RepairRangeSplitter.merge(assignments); - assertEquals(fullRange, result.getTokenRange()); + assertEquals(FULL_RANGE, result.getTokenRange()); assertEquals(keyspaceName, result.getKeyspaceName()); assertEquals(new HashSet<>(Arrays.asList("table1", "table2", "table3")), new HashSet<>(result.getTableNames())); } @@ -261,7 +291,7 @@ public void testMergeMultipleAssignmentsWithSameTokenRangeAndKeyspace() { @Test(expected = IllegalStateException.class) public void testMergeDifferentTokenRange() { // Test merging assignments with different token ranges - Iterator> range = AutoRepairUtils.split(fullRange, 2).iterator(); // Split the full range into two ranges ie (0-100, 100-200 + Iterator> range = AutoRepairUtils.split(FULL_RANGE, 2).iterator(); // Split the full range into two ranges ie (0-100, 100-200 Range tokenRange1 = range.next(); Range tokenRange2 = range.next(); Assert.assertFalse(range.hasNext()); @@ -281,8 +311,8 @@ public void testMergeDifferentKeyspaceName() { // Test merging assignments with different keyspace names List tableNames = Arrays.asList("table1", "table2"); - RepairAssignment assignment1 = new RepairAssignment(fullRange, "keyspace1", tableNames); - RepairAssignment assignment2 = new RepairAssignment(fullRange, "keyspace2", tableNames); + RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, "keyspace1", tableNames); + RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, "keyspace2", tableNames); List assignments = Arrays.asList(assignment1, assignment2); RepairRangeSplitter.merge(assignments); // Should throw IllegalStateException @@ -295,8 +325,8 @@ public void testMergeWithDuplicateTables() { List tableNames1 = Arrays.asList("table1", "table2"); List tableNames2 = Arrays.asList("table2", "table3"); - RepairAssignment assignment1 = new RepairAssignment(fullRange, keyspaceName, tableNames1); - RepairAssignment assignment2 = new RepairAssignment(fullRange, keyspaceName, tableNames2); + RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, keyspaceName, tableNames1); + RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, keyspaceName, tableNames2); List assignments = Arrays.asList(assignment1, assignment2); RepairAssignment result = RepairRangeSplitter.merge(assignments); @@ -322,9 +352,15 @@ private List createAndInsertTables(int count) throws Throwable { } private void insertAndFlushTable(String tableName) throws Throwable { - executeFormattedQuery("INSERT INTO " + KEYSPACE + '.' + tableName + " (k, v) values (?, ?)", 1, 1); + insertAndFlushTable(tableName, 1); + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, tableName); + } + private void insertAndFlushTable(String tableName, int... vals) throws Throwable { + for (int i : vals) + { + executeFormattedQuery("INSERT INTO " + KEYSPACE + '.' + tableName + " (k, v) values (?, ?)", i, i); + } ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, tableName); cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); - assertEquals(1, cfs.getLiveSSTables().size()); } } From 58efc5011b7118ebab2743b16b69d621c9667635 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 10 Nov 2024 14:29:56 -0600 Subject: [PATCH 059/199] Add license header to RepairRangeSplitter --- .../repair/autorepair/RepairRangeSplitter.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 0c66320cc6b8..1939dd8af7eb 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.cassandra.repair.autorepair; import java.io.IOException; From 4eee47d310cf6e1082a6946f3f3f9c4076fe332a Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 10 Nov 2024 15:19:24 -0600 Subject: [PATCH 060/199] Simplify access of repair_type_overrides, key by String A lot of care existed to ensure each RepairType was populated in repair_type_overrides (using ensureOverrides). This could be simplified quite a bit by protecting access to repair_type_overrides by enclosing it in a getOptions method that ensures the provided RepairType has a populated entry. Additionally, change the key type to String and use ConcurrentMap; otherwise the change to make the enum names all caps does not work. --- .../repair/autorepair/AutoRepairConfig.java | 126 ++++++++++-------- .../autorepair/AutoRepairConfigTest.java | 40 +++--- 2 files changed, 85 insertions(+), 81 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index e1e2d5f2baf2..f66f79c6716a 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -19,17 +19,16 @@ package org.apache.cassandra.repair.autorepair; import java.io.Serializable; -import java.util.ArrayList; import java.util.Collections; -import java.util.EnumMap; import java.util.HashSet; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentMap; import java.util.function.Function; +import javax.annotation.Nonnull; + import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.config.ParameterizedClass; @@ -61,6 +60,27 @@ public enum RepairType implements Serializable FULL, INCREMENTAL; + private final String configName; + + RepairType() + { + this.configName = name().toLowerCase(); + } + + RepairType(String configName) + { + this.configName = configName; + } + + /** + * @return Format of the repair type as it should be represented in configuration. + * Canonically this is the enum name in lowerCase. + */ + public String getConfigName() + { + return configName; + } + public static AutoRepairState getAutoRepairState(RepairType repairType) { switch (repairType) @@ -75,8 +95,9 @@ public static AutoRepairState getAutoRepairState(RepairType repairType) } } - // repair_type_overrides overrides the global_settings for a specific repair type - public volatile Map repair_type_overrides = new EnumMap<>(RepairType.class); + // repair_type_overrides overrides the global_settings for a specific repair type. String used as key instead + // of enum to allow lower case key in yaml. + public volatile ConcurrentMap repair_type_overrides = Maps.newConcurrentMap(); public AutoRepairConfig() { @@ -87,10 +108,6 @@ public AutoRepairConfig(boolean enabled) { this.enabled = enabled; global_settings = Options.getDefaultOptions(); - for (RepairType type : RepairType.values()) - { - repair_type_overrides.put(type, new Options()); - } } public DurationSpec.IntSecondsBound getRepairCheckInterval() @@ -146,14 +163,12 @@ public boolean isAutoRepairEnabled(RepairType repairType) public void setAutoRepairEnabled(RepairType repairType, boolean enabled) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).enabled = enabled; + getOptions(repairType).enabled = enabled; } public void setRepairByKeyspace(RepairType repairType, boolean repairByKeyspace) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).repair_by_keyspace = repairByKeyspace; + getOptions(repairType).repair_by_keyspace = repairByKeyspace; } public boolean getRepairByKeyspace(RepairType repairType) @@ -168,8 +183,7 @@ public int getRepairThreads(RepairType repairType) public void setRepairThreads(RepairType repairType, int repairThreads) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).number_of_repair_threads = repairThreads; + getOptions(repairType).number_of_repair_threads = repairThreads; } public int getRepairSubRangeNum(RepairType repairType) @@ -179,8 +193,7 @@ public int getRepairSubRangeNum(RepairType repairType) public void setRepairSubRangeNum(RepairType repairType, int repairSubRanges) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).number_of_subranges = repairSubRanges; + getOptions(repairType).number_of_subranges = repairSubRanges; } public DurationSpec.IntSecondsBound getRepairMinInterval(RepairType repairType) @@ -190,8 +203,7 @@ public DurationSpec.IntSecondsBound getRepairMinInterval(RepairType repairType) public void setRepairMinInterval(RepairType repairType, String minRepairInterval) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).min_repair_interval = new DurationSpec.IntSecondsBound(minRepairInterval); + getOptions(repairType).min_repair_interval = new DurationSpec.IntSecondsBound(minRepairInterval); } public int getRepairSSTableCountHigherThreshold(RepairType repairType) @@ -201,8 +213,7 @@ public int getRepairSSTableCountHigherThreshold(RepairType repairType) public void setRepairSSTableCountHigherThreshold(RepairType repairType, int sstableHigherThreshold) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).sstable_upper_threshold = sstableHigherThreshold; + getOptions(repairType).sstable_upper_threshold = sstableHigherThreshold; } public DurationSpec.IntSecondsBound getAutoRepairTableMaxRepairTime(RepairType repairType) @@ -212,8 +223,7 @@ public DurationSpec.IntSecondsBound getAutoRepairTableMaxRepairTime(RepairType r public void setAutoRepairTableMaxRepairTime(RepairType repairType, String autoRepairTableMaxRepairTime) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).table_max_repair_time = new DurationSpec.IntSecondsBound(autoRepairTableMaxRepairTime); + getOptions(repairType).table_max_repair_time = new DurationSpec.IntSecondsBound(autoRepairTableMaxRepairTime); } public Set getIgnoreDCs(RepairType repairType) @@ -223,8 +233,7 @@ public Set getIgnoreDCs(RepairType repairType) public void setIgnoreDCs(RepairType repairType, Set ignoreDCs) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).ignore_dcs = ignoreDCs; + getOptions(repairType).ignore_dcs = ignoreDCs; } public boolean getRepairPrimaryTokenRangeOnly(RepairType repairType) @@ -234,8 +243,7 @@ public boolean getRepairPrimaryTokenRangeOnly(RepairType repairType) public void setRepairPrimaryTokenRangeOnly(RepairType repairType, boolean primaryTokenRangeOnly) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).repair_primary_token_range_only = primaryTokenRangeOnly; + getOptions(repairType).repair_primary_token_range_only = primaryTokenRangeOnly; } public int getParallelRepairPercentage(RepairType repairType) @@ -245,8 +253,7 @@ public int getParallelRepairPercentage(RepairType repairType) public void setParallelRepairPercentage(RepairType repairType, int percentage) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).parallel_repair_percentage = percentage; + getOptions(repairType).parallel_repair_percentage = percentage; } public int getParallelRepairCount(RepairType repairType) @@ -256,8 +263,7 @@ public int getParallelRepairCount(RepairType repairType) public void setParallelRepairCount(RepairType repairType, int count) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).parallel_repair_count = count; + getOptions(repairType).parallel_repair_count = count; } public boolean getMVRepairEnabled(RepairType repairType) @@ -267,14 +273,12 @@ public boolean getMVRepairEnabled(RepairType repairType) public void setMVRepairEnabled(RepairType repairType, boolean enabled) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).mv_repair_enabled = enabled; + getOptions(repairType).mv_repair_enabled = enabled; } public void setForceRepairNewNode(RepairType repairType, boolean forceRepairNewNode) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).force_repair_new_node = forceRepairNewNode; + getOptions(repairType).force_repair_new_node = forceRepairNewNode; } public boolean getForceRepairNewNode(RepairType repairType) @@ -289,8 +293,7 @@ public ParameterizedClass getTokenRangeSplitter(RepairType repairType) public void setInitialSchedulerDelay(RepairType repairType, String initialSchedulerDelay) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).initial_scheduler_delay = new DurationSpec.IntSecondsBound(initialSchedulerDelay); + getOptions(repairType).initial_scheduler_delay = new DurationSpec.IntSecondsBound(initialSchedulerDelay); } public DurationSpec.IntSecondsBound getInitialSchedulerDelay(RepairType repairType) @@ -305,8 +308,7 @@ public DurationSpec.IntSecondsBound getRepairSessionTimeout(RepairType repairTyp public void setRepairSessionTimeout(RepairType repairType, String repairSessionTimeout) { - ensureOverrides(repairType); - repair_type_overrides.get(repairType).repair_session_timeout = new DurationSpec.IntSecondsBound(repairSessionTimeout); + getOptions(repairType).repair_session_timeout = new DurationSpec.IntSecondsBound(repairSessionTimeout); } // Options configures auto-repair behavior for a given repair type. @@ -435,31 +437,37 @@ public String toString() } } - @VisibleForTesting - protected T applyOverrides(RepairType repairType, Function optionSupplier) + @Nonnull + protected Options getOptions(RepairType repairType) { - ArrayList optsProviders = new ArrayList<>(); - if (repair_type_overrides != null) - { - optsProviders.add(repair_type_overrides.get(repairType)); - } - optsProviders.add(global_settings); - optsProviders.add(Options.defaultOptions); + return repair_type_overrides.computeIfAbsent(repairType.getConfigName(), k -> new Options()); + } - return optsProviders.stream() - .map(opt -> Optional.ofNullable(opt).map(optionSupplier).orElse(null)) - .filter(Objects::nonNull) - .findFirst() - .orElse(null); + private static T getOverride(Options options, Function optionSupplier) + { + return options != null ? optionSupplier.apply(options) : null; } - protected void ensureOverrides(RepairType repairType) + @VisibleForTesting + protected T applyOverrides(RepairType repairType, Function optionSupplier) { - if (repair_type_overrides == null) + // Check option by repair type first + Options repairTypeOverrides = getOptions(repairType); + T val = optionSupplier.apply(repairTypeOverrides); + + if (val != null) + return val; + + // Check option in global settings + if (global_settings != null) { - repair_type_overrides = new EnumMap<>(RepairType.class); + val = getOverride(global_settings, optionSupplier); + + if (val != null) + return val; } - repair_type_overrides.computeIfAbsent(repairType, k -> new Options()); + // Otherwise check defaults + return getOverride(Options.defaultOptions, optionSupplier); } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 4f1dd029ef5e..08ae9631a192 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.repair.autorepair; -import java.util.EnumMap; import java.util.Objects; import java.util.Collections; import java.util.Set; @@ -63,7 +62,6 @@ public static Object[] repairTypes() public void setUp() { config = new AutoRepairConfig(true); - config.repair_type_overrides = null; AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); } @@ -80,7 +78,7 @@ public void autoRepairConfigRepairTypesAreNotNull() AutoRepairConfig config = new AutoRepairConfig(); for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) { - assertNotNull(config.repair_type_overrides.get(repairType)); + assertNotNull(config.getOptions(repairType)); } } @@ -105,10 +103,8 @@ public void testIsAutoRepairEnabledReturnsTrueWhenRepairIsDisabledGlobally() public void testIsAutoRepairEnabledReturnsTrueWhenRepairIsDisabledForRepairType() { config.global_settings.enabled = true; - config.repair_type_overrides = new EnumMap<>(AutoRepairConfig.RepairType.class); - config.repair_type_overrides.put(repairType, new Options()); - config.repair_type_overrides.get(repairType).enabled = false; - assertFalse(config.isAutoRepairEnabled(repairType)); + config.setAutoRepairEnabled(repairType, false); + assertFalse(config.getOptions(repairType).enabled); } @Test @@ -118,7 +114,7 @@ public void testSetAutoRepairEnabledNoMVOrCDC() DatabaseDescriptor.setMaterializedViewsEnabled(false); config.setAutoRepairEnabled(repairType, true); - assertTrue(config.repair_type_overrides.get(repairType).enabled); + assertTrue(config.getOptions(repairType).enabled); } @Test @@ -126,7 +122,7 @@ public void testSetRepairByKeyspace() { config.setRepairByKeyspace(repairType, true); - assertTrue(config.repair_type_overrides.get(repairType).repair_by_keyspace); + assertTrue(config.getOptions(repairType).repair_by_keyspace); } @Test @@ -144,7 +140,7 @@ public void testSetRepairThreads() { config.setRepairThreads(repairType, 5); - assert config.repair_type_overrides.get(repairType).number_of_repair_threads == 5; + assert config.getOptions(repairType).number_of_repair_threads == 5; } @Test @@ -172,7 +168,7 @@ public void testSetRepairSubRangeNum() { config.setRepairSubRangeNum(repairType, 5); - assert config.repair_type_overrides.get(repairType).number_of_subranges == 5; + assert config.getOptions(repairType).number_of_subranges == 5; } @Test @@ -190,7 +186,7 @@ public void testSetRepairMinFrequencyInHours() { config.setRepairMinInterval(repairType, "5s"); - assert config.repair_type_overrides.get(repairType).min_repair_interval.toSeconds() == 5; + assert config.getOptions(repairType).min_repair_interval.toSeconds() == 5; } @Test @@ -226,7 +222,7 @@ public void testSetRepairSSTableCountHigherThreshold() { config.setRepairSSTableCountHigherThreshold(repairType, 5); - assert config.repair_type_overrides.get(repairType).sstable_upper_threshold == 5; + assert config.getOptions(repairType).sstable_upper_threshold == 5; } @Test @@ -244,7 +240,7 @@ public void testSetAutoRepairTableMaxRepairTimeInSec() { config.setAutoRepairTableMaxRepairTime(repairType, "5s"); - assert config.repair_type_overrides.get(repairType).table_max_repair_time.toSeconds() == 5; + assert config.getOptions(repairType).table_max_repair_time.toSeconds() == 5; } @Test @@ -262,7 +258,7 @@ public void testSetIgnoreDCs() { config.setIgnoreDCs(repairType, testSet); - assertEquals(config.repair_type_overrides.get(repairType).ignore_dcs, testSet); + assertEquals(config.getOptions(repairType).ignore_dcs, testSet); } @Test @@ -280,7 +276,7 @@ public void testSetRepairPrimaryTokenRangeOnly() { config.setRepairPrimaryTokenRangeOnly(repairType, true); - assertTrue(config.repair_type_overrides.get(repairType).repair_primary_token_range_only); + assertTrue(config.getOptions(repairType).repair_primary_token_range_only); } @Test @@ -298,7 +294,7 @@ public void testSetParallelRepairPercentageInGroup() { config.setParallelRepairPercentage(repairType, 5); - assert config.repair_type_overrides.get(repairType).parallel_repair_percentage == 5; + assert config.getOptions(repairType).parallel_repair_percentage == 5; } @Test @@ -316,7 +312,7 @@ public void testSetParallelRepairCountInGroup() { config.setParallelRepairCount(repairType, 5); - assert config.repair_type_overrides.get(repairType).parallel_repair_count == 5; + assert config.getOptions(repairType).parallel_repair_count == 5; } @Test @@ -334,7 +330,7 @@ public void testSetMVRepairEnabled() { config.setMVRepairEnabled(repairType, true); - assertTrue(config.repair_type_overrides.get(repairType).mv_repair_enabled); + assertTrue(config.getOptions(repairType).mv_repair_enabled); } @Test @@ -342,7 +338,7 @@ public void testSetForceRepairNewNode() { config.setForceRepairNewNode(repairType, true); - assertTrue(config.repair_type_overrides.get(repairType).force_repair_new_node); + assertTrue(config.getOptions(repairType).force_repair_new_node); } @Test @@ -403,7 +399,7 @@ public void testSetInitialSchedulerDelay() { config.setInitialSchedulerDelay(repairType, "5s"); - assert config.repair_type_overrides.get(repairType).initial_scheduler_delay.toSeconds() == 5; + assert config.getOptions(repairType).initial_scheduler_delay.toSeconds() == 5; } @Test @@ -421,7 +417,7 @@ public void testSetRepairSessionTimeout() { config.setRepairSessionTimeout(repairType, "1h"); - assert config.repair_type_overrides.get(repairType).repair_session_timeout.toSeconds() == 3600; + assert config.getOptions(repairType).repair_session_timeout.toSeconds() == 3600; } } From 3102ffbc69047c2b9de0fcad3aab2235e29d4146 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 10 Nov 2024 22:39:07 -0600 Subject: [PATCH 061/199] Allow partial repair of range, factor in total bytes for IR * Rename maxBytesPerScheduleBytes to maxBytesPerSchedule * Allow partial repair of range if the smallest subrange we can repair is less than the remaining bytes we can repair. * Consider totalSize for IR (because of anticompaction) sizeinRange for full repair. * Increased logging while debugging. --- .../autorepair/RepairRangeSplitter.java | 104 ++++++++++++++---- .../autorepair/RepairRangeSplitterTest.java | 6 +- 2 files changed, 83 insertions(+), 27 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 1939dd8af7eb..7040892fd5ce 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -50,6 +49,7 @@ import org.apache.cassandra.io.sstable.format.big.BigTableScanner; import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.concurrent.Refs; @@ -89,7 +89,7 @@ public class RepairRangeSplitter implements IAutoRepairTokenRangeSplitter static final String MAX_BYTES_PER_SCHEDULE = "max_bytes_per_schedule"; private final int tablesPerAssignmentLimit; - private final long maxBytesPerScheduleBytes; + private final long maxBytesPerSchedule; private final long bytesPerSubrange; private final long partitionsPerSubrange; @@ -114,11 +114,11 @@ public RepairRangeSplitter(Map parameters) if (parameters.containsKey(MAX_BYTES_PER_SCHEDULE)) { - maxBytesPerScheduleBytes = new DataStorageSpec.LongBytesBound(parameters.get(MAX_BYTES_PER_SCHEDULE)).toBytes(); + maxBytesPerSchedule = new DataStorageSpec.LongBytesBound(parameters.get(MAX_BYTES_PER_SCHEDULE)).toBytes(); } else { - maxBytesPerScheduleBytes = DEFAULT_MAX_BYTES_PER_SCHEDULE; + maxBytesPerSchedule = DEFAULT_MAX_BYTES_PER_SCHEDULE; } if (parameters.containsKey(PARTITION_COUNT)) @@ -140,7 +140,7 @@ public RepairRangeSplitter(Map parameters) } logger.info("Configured {} with {}={}, {}={}, {}={}, {}={}", RepairRangeSplitter.class.getName(), - SUBRANGE_SIZE, subrangeSize, MAX_BYTES_PER_SCHEDULE, maxBytesPerScheduleBytes, + SUBRANGE_SIZE, subrangeSize, MAX_BYTES_PER_SCHEDULE, maxBytesPerSchedule, PARTITION_COUNT, partitionsPerSubrange, TABLE_BATCH_LIMIT, tablesPerAssignmentLimit); } @@ -233,7 +233,10 @@ public List getRepairAssignmentsForTable(AutoRepairConfig.Repa Collections.shuffle(sizeEstimates); for (SizeEstimate estimate : sizeEstimates) { - if (estimate.sizeInRange == 0) + // Calculate the smallest subrange if we were to split, this can be used to determine if we can do any + // work if we have a non-zero estimate. + long smallestSubrange = Math.min(estimate.sizeForRepair, bytesPerSubrange); + if (estimate.sizeForRepair == 0) { ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); long memtableSize = cfs.getTracker().getView().getCurrentMemtable().getLiveDataSize(); @@ -248,31 +251,40 @@ public List getRepairAssignmentsForTable(AutoRepairConfig.Repa logger.debug("Skipping {}.{} for range {} because it had no unrepaired SSTables and no memtable data", keyspaceName, tableName, estimate.tokenRange); } } - else if (targetBytesSoFar + estimate.sizeInRange < maxBytesPerScheduleBytes) + else if (targetBytesSoFar + smallestSubrange < maxBytesPerSchedule) { - targetBytesSoFar += estimate.sizeInRange; // Check if the estimate needs splitting based on the criteria - boolean needsSplitting = estimate.sizeInRange > bytesPerSubrange || estimate.partitions > partitionsPerSubrange; + boolean needsSplitting = estimate.sizeForRepair > bytesPerSubrange || estimate.partitions > partitionsPerSubrange; if (needsSplitting) { - // Calculate the number of splits needed for size and partitions - int splitsForSize = (int) Math.ceil((double) estimate.sizeInRange / bytesPerSubrange); - int splitsForPartitions = (int) Math.ceil((double) estimate.partitions / partitionsPerSubrange); - - // Choose the larger of the two as the number of splits - int numberOfSplits = Math.max(splitsForSize, splitsForPartitions); - - // Split the token range into subranges + int numberOfSplits = calculateNumberOfSplits(estimate); + long approximateBytesPerSplit = estimate.sizeForRepair / numberOfSplits; Collection> subranges = split(estimate.tokenRange, numberOfSplits); for (Range subrange : subranges) { + if (targetBytesSoFar + approximateBytesPerSplit > maxBytesPerSchedule) + { + logger.warn("Refusing to add repair assignment for {}.{} for subrange {} with a approximateBytesPerSplit={} because it would increase total repair bytes to {} which is greater than {}={}", + keyspaceName, tableName, subrange, FileUtils.stringifyFileSize(approximateBytesPerSplit), FileUtils.stringifyFileSize(targetBytesSoFar + approximateBytesPerSplit), + MAX_BYTES_PER_SCHEDULE, FileUtils.stringifyFileSize(maxBytesPerSchedule)); + break; + } + logger.info("Added repair assignment for {}.{} for subrange {} (#{}/{})", + keyspaceName, tableName, subrange, repairAssignments.size() + 1, numberOfSplits); RepairAssignment assignment = new RepairAssignment(subrange, keyspaceName, Collections.singletonList(tableName)); repairAssignments.add(assignment); + targetBytesSoFar += approximateBytesPerSplit; } } else { + logger.info("Using 1 repair assignment for {}.{} for range {} as rangeBytes={} is less than {}={} and partitionEstimate={} is less than {}={}", + keyspaceName, tableName, estimate.tokenRange, + FileUtils.stringifyFileSize(estimate.sizeForRepair), + SUBRANGE_SIZE, FileUtils.stringifyFileSize(bytesPerSubrange), + estimate.partitions, + PARTITION_COUNT, partitionsPerSubrange); // No splitting needed, repair the entire range as-is RepairAssignment assignment = new RepairAssignment(estimate.tokenRange, keyspaceName, Collections.singletonList(tableName)); repairAssignments.add(assignment); @@ -280,13 +292,45 @@ else if (targetBytesSoFar + estimate.sizeInRange < maxBytesPerScheduleBytes) } else { + long calculatedSize = targetBytesSoFar + estimate.sizeForRepair; // Really this is "Ok" but it does mean we are relying on randomness to cover the other ranges - logger.info("Skipping range {} for {}.{} as it would exceed maxBytesPerScheduleBytes, consider increasing maxBytesPerScheduleBytes, reducing node density or monitoring to ensure all ranges do get repaired within gc_grace_seconds", estimate.tokenRange, keyspaceName, tableName); + logger.info("Skipping range {} for {}.{} as {} would exceed {}={}, " + + "consider increasing {}, reducing node denity or monitoring to ensure all ranges do get repaired within gc_grace_seconds", + estimate.tokenRange, + keyspaceName, tableName, + FileUtils.stringifyFileSize(calculatedSize), + MAX_BYTES_PER_SCHEDULE, FileUtils.stringifyFileSize(calculatedSize), MAX_BYTES_PER_SCHEDULE); } } return repairAssignments; } + private int calculateNumberOfSplits(SizeEstimate estimate) + { + // Calculate the number of splits needed for size and partitions + int splitsForSize = (int) Math.ceil((double) estimate.sizeForRepair / bytesPerSubrange); + int splitsForPartitions = (int) Math.ceil((double) estimate.partitions / partitionsPerSubrange); + + // Split the token range into subranges based on whichever (partitions, bytes) would generate the most splits. + boolean splitBySize = splitsForSize > splitsForPartitions; + int splits = splitBySize ? splitsForSize : splitsForPartitions; + + // calculate approximation for logging purposes + long approximateBytesPerSplit = estimate.sizeForRepair / splits; + long approximatePartitionsPerSplit = estimate.partitions / splits; + + logger.info("Splitting {}.{} for range {} into {} sub ranges by {} (splitsForSize={}, splitsForPartitions={}, " + + "approximateBytesInRange={}, approximatePartitionsInRange={}, " + + "approximateBytesPerSplit={}, approximatePartitionsPerSplit={})", + estimate.keyspace, estimate.table, estimate.tokenRange, + splits, splitBySize ? "size" : "partitions", + splitsForSize, splitsForPartitions, + FileUtils.stringifyFileSize(estimate.sizeForRepair), estimate.partitions, + FileUtils.stringifyFileSize(approximateBytesPerSplit), approximatePartitionsPerSplit + ); + return splits; + } + public Collection> getTokenRanges(boolean primaryRangeOnly, String keyspaceName) { // Collect all applicable token ranges @@ -317,7 +361,7 @@ private List getRangeSizeEstimate(AutoRepairConfig.RepairType repa logger.debug("Calculating size estimate for {}.{} for range {}", keyspace, table, tokenRange); try (Refs refs = getSSTableReaderRefs(repairType, keyspace, table, tokenRange)) { - SizeEstimate estimate = getSizesForRangeOfSSTables(keyspace, table, tokenRange, refs); + SizeEstimate estimate = getSizesForRangeOfSSTables(repairType, keyspace, table, tokenRange, refs); logger.debug("Size estimate for {}.{} for range {} is {}", keyspace, table, tokenRange, estimate); sizeEstimates.add(estimate); } @@ -326,7 +370,7 @@ private List getRangeSizeEstimate(AutoRepairConfig.RepairType repa } @VisibleForTesting - static SizeEstimate getSizesForRangeOfSSTables(String keyspace, String table, Range tokenRange, Refs refs) + static SizeEstimate getSizesForRangeOfSSTables(AutoRepairConfig.RepairType repairType, String keyspace, String table, Range tokenRange, Refs refs) { ICardinality cardinality = new HyperLogLogPlus(13, 25); long approxBytesInRange = 0L; @@ -373,7 +417,7 @@ static SizeEstimate getSizesForRangeOfSSTables(String keyspace, String table, Ra double ratio = approxBytesInRange / (double) totalBytes; // use the ratio from size to estimate the partitions in the range as well long partitions = (long) Math.max(1, Math.ceil(cardinality.cardinality() * ratio)); - return new SizeEstimate(keyspace, table, tokenRange, partitions, approxBytesInRange, totalBytes); + return new SizeEstimate(repairType, keyspace, table, tokenRange, partitions, approxBytesInRange, totalBytes); } @VisibleForTesting @@ -396,7 +440,7 @@ static Refs getSSTableReaderRefs(AutoRepairConfig.RepairType repa List canonicalSSTables = View.sstablesInBounds(r.left, r.right, tree); if (repairType == AutoRepairConfig.RepairType.INCREMENTAL) { - canonicalSSTables = canonicalSSTables.stream().filter(SSTableReader::isRepaired).collect(Collectors.toList()); + canonicalSSTables = canonicalSSTables.stream().filter((sstable) -> !sstable.isRepaired()).collect(Collectors.toList()); } refs = Refs.tryRef(canonicalSSTables); } @@ -406,21 +450,33 @@ static Refs getSSTableReaderRefs(AutoRepairConfig.RepairType repa @VisibleForTesting protected static class SizeEstimate { + public final AutoRepairConfig.RepairType repairType; public final String keyspace; public final String table; public final Range tokenRange; public final long partitions; public final long sizeInRange; public final long totalSize; - - public SizeEstimate(String keyspace, String table, Range tokenRange, long partitions, long sizeInRange, long totalSize) + /** + * Size to consider in the repair. For incremental repair, we want to consider the total size + * of the estimate as we have to factor in anticompacting the entire SSTable. + * For full repair, just use the size containing the range. + */ + public final long sizeForRepair; + + public SizeEstimate(AutoRepairConfig.RepairType repairType, + String keyspace, String table, Range tokenRange, + long partitions, long sizeInRange, long totalSize) { + this.repairType = repairType; this.keyspace = keyspace; this.table = table; this.tokenRange = tokenRange; this.partitions = partitions; this.sizeInRange = sizeInRange; this.totalSize = totalSize; + + this.sizeForRepair = repairType == AutoRepairConfig.RepairType.INCREMENTAL ? totalSize : sizeInRange; } } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java index aa81e75300cf..71b60a92232e 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java @@ -70,7 +70,7 @@ public void testSizePartitionCount() throws Throwable insertAndFlushTable(tableName, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10); Refs sstables = RepairRangeSplitter.getSSTableReaderRefs(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, FULL_RANGE); assertEquals(10, sstables.iterator().next().getEstimatedPartitionSize().count()); - RepairRangeSplitter.SizeEstimate sizes = RepairRangeSplitter.getSizesForRangeOfSSTables(KEYSPACE, tableName, FULL_RANGE, sstables); + RepairRangeSplitter.SizeEstimate sizes = RepairRangeSplitter.getSizesForRangeOfSSTables(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, FULL_RANGE, sstables); assertEquals(10, sizes.partitions); } @@ -88,8 +88,8 @@ public void testSizePartitionCountSplit() throws Throwable Refs sstables1 = RepairRangeSplitter.getSSTableReaderRefs(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, tokenRange1); Refs sstables2 = RepairRangeSplitter.getSSTableReaderRefs(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, tokenRange2); - RepairRangeSplitter.SizeEstimate sizes1 = RepairRangeSplitter.getSizesForRangeOfSSTables(KEYSPACE, tableName, tokenRange1, sstables1); - RepairRangeSplitter.SizeEstimate sizes2 = RepairRangeSplitter.getSizesForRangeOfSSTables(KEYSPACE, tableName, tokenRange2, sstables2); + RepairRangeSplitter.SizeEstimate sizes1 = RepairRangeSplitter.getSizesForRangeOfSSTables(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, tokenRange1, sstables1); + RepairRangeSplitter.SizeEstimate sizes2 = RepairRangeSplitter.getSizesForRangeOfSSTables(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, tokenRange2, sstables2); // +-5% because HLL merge and the applying of range size approx ratio causes estimation errors assertTrue(Math.abs(10000 - (sizes1.partitions + sizes2.partitions)) <= 60); } From 19d972bc269df0c8efdc9731d0e310c86e6fedec Mon Sep 17 00:00:00 2001 From: Chris Lohfink Date: Thu, 14 Nov 2024 11:41:51 -0600 Subject: [PATCH 062/199] update warnings based on repair type --- .../autorepair/RepairRangeSplitter.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 7040892fd5ce..5f4af886bf76 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -100,7 +100,6 @@ public class RepairRangeSplitter implements IAutoRepairTokenRangeSplitter public RepairRangeSplitter(Map parameters) { - // Demonstrates parameterizing a range splitter so we can have splitter specific options. DataStorageSpec.LongBytesBound subrangeSize; if (parameters.containsKey(SUBRANGE_SIZE)) { @@ -174,7 +173,7 @@ List getRepairAssignments(AutoRepairConfig.RepairType repairTy if (tableAssignments.isEmpty()) continue; - // If the table assignments are for the same token range and we have room to add more tables to the current assignment + // If the table assignments are for the same token range, and we have room to add more tables to the current assignment if (tableAssignments.size() == 1 && currentAssignments.size() < tablesPerAssignmentLimit && (currentAssignments.isEmpty() || currentAssignments.get(0).getTokenRange().equals(tableAssignments.get(0).getTokenRange()))) { @@ -228,7 +227,7 @@ public List getRepairAssignmentsForTable(AutoRepairConfig.Repa long targetBytesSoFar = 0; List sizeEstimates = getRangeSizeEstimate(repairType, keyspaceName, tableName, tokenRanges); - // since its possible for us to hit maxBytesPerScheduleBytes before seeing all ranges, shuffle so there is chance + // since its possible for us to hit maxBytesPerSchedule before seeing all ranges, shuffle so there is chance // at least of hitting all the ranges _eventually_ for the worst case scenarios Collections.shuffle(sizeEstimates); for (SizeEstimate estimate : sizeEstimates) @@ -265,9 +264,7 @@ else if (targetBytesSoFar + smallestSubrange < maxBytesPerSchedule) { if (targetBytesSoFar + approximateBytesPerSplit > maxBytesPerSchedule) { - logger.warn("Refusing to add repair assignment for {}.{} for subrange {} with a approximateBytesPerSplit={} because it would increase total repair bytes to {} which is greater than {}={}", - keyspaceName, tableName, subrange, FileUtils.stringifyFileSize(approximateBytesPerSplit), FileUtils.stringifyFileSize(targetBytesSoFar + approximateBytesPerSplit), - MAX_BYTES_PER_SCHEDULE, FileUtils.stringifyFileSize(maxBytesPerSchedule)); + warnMaxBytesPerSchedule(repairType, keyspaceName, tableName); break; } logger.info("Added repair assignment for {}.{} for subrange {} (#{}/{})", @@ -292,19 +289,23 @@ else if (targetBytesSoFar + smallestSubrange < maxBytesPerSchedule) } else { - long calculatedSize = targetBytesSoFar + estimate.sizeForRepair; // Really this is "Ok" but it does mean we are relying on randomness to cover the other ranges - logger.info("Skipping range {} for {}.{} as {} would exceed {}={}, " + - "consider increasing {}, reducing node denity or monitoring to ensure all ranges do get repaired within gc_grace_seconds", - estimate.tokenRange, - keyspaceName, tableName, - FileUtils.stringifyFileSize(calculatedSize), - MAX_BYTES_PER_SCHEDULE, FileUtils.stringifyFileSize(calculatedSize), MAX_BYTES_PER_SCHEDULE); + warnMaxBytesPerSchedule(repairType, keyspaceName, tableName); } } return repairAssignments; } + private void warnMaxBytesPerSchedule(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName) + { + String warning = "Refusing to add repair assignment for {}.{} because it would increase total repair bytes greater {}"; + if (repairType == AutoRepairConfig.RepairType.FULL) + { + warning = warning + ", everything will not be repaired this schedule. Consider increasing maxBytesPerSchedule, reducing node density or monitoring to ensure all ranges do get repaired within gc_grace_seconds"; + } + logger.warn(warning, keyspaceName, tableName, FileUtils.stringifyFileSize(maxBytesPerSchedule)); + } + private int calculateNumberOfSplits(SizeEstimate estimate) { // Calculate the number of splits needed for size and partitions From c7de80b71def4cfd87d268acbd466db6aa53d2ad Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Thu, 21 Nov 2024 11:14:34 -0800 Subject: [PATCH 063/199] Improve preview repair metrics --- .../cassandra/metrics/KeyspaceMetrics.java | 7 ------- .../apache/cassandra/metrics/TableMetrics.java | 8 ++++---- .../cassandra/repair/PreviewRepairTask.java | 17 +++++++++++++++++ .../org/apache/cassandra/repair/RepairJob.java | 12 +----------- .../repair/consistent/SyncStatSummary.java | 9 +++++++-- 5 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index d85a8846df53..776027e395a1 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -125,11 +125,6 @@ public class KeyspaceMetrics public final Histogram bytesValidated; /** histogram over the number of partitions we have validated */ public final Histogram partitionsValidated; - /** histogram over the number of desynchronized token ranges detected during preview repair */ - public final Histogram previewedDesynchronizedTokenRanges; - /** histogram over the number of desynchronized bytes detected during preview repair */ - public final Histogram previewedDesynchronizedBytes; - /* * Metrics for inconsistencies detected between repaired data sets across replicas. These @@ -249,8 +244,6 @@ public KeyspaceMetrics(final Keyspace ks) repairSyncTime = createKeyspaceTimer("RepairSyncTime"); partitionsValidated = createKeyspaceHistogram("PartitionsValidated", false); bytesValidated = createKeyspaceHistogram("BytesValidated", false); - previewedDesynchronizedTokenRanges = createKeyspaceHistogram("PreviewedDesynchronizedTokenRanges", false); - previewedDesynchronizedBytes = createKeyspaceHistogram("PreviewedDesynchronizedBytes", false); confirmedRepairedInconsistencies = createKeyspaceMeter("RepairedDataInconsistenciesConfirmed"); unconfirmedRepairedInconsistencies = createKeyspaceMeter("RepairedDataInconsistenciesUnconfirmed"); diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 1356b60212ca..77b45f08948d 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -205,9 +205,9 @@ public class TableMetrics /** number of bytes where the whole sstable was contained in a repairing range so that we only mutated the repair status */ public final Counter bytesMutatedAnticompaction; /** number of desynchronized token ranges that were detected during preview repair */ - public final TableHistogram previewedDesynchronizedTokenRanges; + public final Counter previewedDesynchronizedTokenRanges; /** number of desynchronized bytes that were detected during preview repair */ - public final TableHistogram previewedDesynchronizedBytes; + public final Counter previewedDesynchronizedBytes; /** ratio of how much we anticompact vs how much we could mutate the repair status*/ public final Gauge mutatedAnticompactionGauge; @@ -904,8 +904,8 @@ protected double getDenominator() partitionsValidated = createTableHistogram("PartitionsValidated", cfs.keyspace.metric.partitionsValidated, false); bytesAnticompacted = createTableCounter("BytesAnticompacted"); bytesMutatedAnticompaction = createTableCounter("BytesMutatedAnticompaction"); - previewedDesynchronizedTokenRanges = createTableHistogram("PreviewedDesynchronizedTokenRanges", cfs.keyspace.metric.previewedDesynchronizedTokenRanges, false); - previewedDesynchronizedBytes = createTableHistogram("PreviewedDesynchronizedBytes", cfs.keyspace.metric.previewedDesynchronizedBytes, false); + previewedDesynchronizedTokenRanges = createTableCounter("PreviewedDesynchronizedTokenRanges"); + previewedDesynchronizedBytes = createTableCounter("PreviewedDesynchronizedBytes"); mutatedAnticompactionGauge = createTableGauge("MutatedAnticompactionGauge", () -> { double bytesMutated = bytesMutatedAnticompaction.getCount(); diff --git a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java index c97632ef8ff0..ee48e6ce170b 100644 --- a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java +++ b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java @@ -26,6 +26,7 @@ import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.metrics.RepairMetrics; @@ -88,6 +89,7 @@ public Future performUnsafe(ExecutorPlus executor, Sche if (previewKind == PreviewKind.REPAIRED) maybeSnapshotReplicas(parentSession, keyspace, result.results.get()); // we know its present as summary used it } + emitMetrics(summary); successMessage += "; " + message; notifier.notification(message); @@ -95,6 +97,21 @@ public Future performUnsafe(ExecutorPlus executor, Sche }); } + private void emitMetrics(SyncStatSummary summary) + { + if (!summary.isEmpty()) + RepairMetrics.previewFailures.inc(); + + summary.getTotals().forEach((key, table) -> { + if (table.isCounter()) + return; + + ColumnFamilyStore cfs = Keyspace.open(key.left).getColumnFamilyStore(key.right); + cfs.metric.previewedDesynchronizedTokenRanges.inc(table.getRanges()); + cfs.metric.previewedDesynchronizedBytes.inc(table.getBytes()); + }); + } + private void maybeSnapshotReplicas(TimeUUID parentSession, String keyspace, List results) { if (!DatabaseDescriptor.snapshotOnRepairedDataMismatch()) diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java index 7ece7e049bac..d6f8820e56f9 100644 --- a/src/java/org/apache/cassandra/repair/RepairJob.java +++ b/src/java/org/apache/cassandra/repair/RepairJob.java @@ -29,11 +29,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.*; import org.apache.cassandra.concurrent.ExecutorPlus; -import org.apache.cassandra.repair.consistent.SyncStatSummary; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.repair.state.JobState; -import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.utils.concurrent.AsyncFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -230,15 +228,7 @@ public void onFailure(Throwable t) logger.warn("{} {}.{} sync failed", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily); SystemDistributedKeyspace.failedRepairJob(session.getId(), desc.keyspace, desc.columnFamily, t); } - } - else - { - SyncStatSummary.Table summary = new SyncStatSummary.Table(cfs.keyspace.getName(), cfs.getTableName()); - summary.consumeStats(stats); - cfs.metric.previewedDesynchronizedTokenRanges.update(summary.getRanges()); - cfs.metric.previewedDesynchronizedBytes.update(summary.getRanges()); - } - cfs.metric.repairsCompleted.inc(); + cfs.metric.repairsCompleted.inc(); tryFailure(t instanceof NoSuchRepairSessionExceptionWrapper ? ((NoSuchRepairSessionExceptionWrapper) t).wrapped : t); diff --git a/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java index e6fcbe73b418..5536a502c2f1 100644 --- a/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java +++ b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java @@ -114,7 +114,7 @@ void consumeStat(SyncStat stat) } } - public void consumeStats(List stats) + void consumeStats(List stats) { filter(stats, s -> s.summaries != null).forEach(this::consumeStat); } @@ -133,7 +133,7 @@ void calculateTotals() totalsCalculated = true; } - boolean isCounter() + public boolean isCounter() { TableMetadata tmd = Schema.instance.getTableMetadata(keyspace, table); return tmd != null && tmd.isCounter(); @@ -221,6 +221,11 @@ private void calculateTotals() totalsCalculated = true; } + public Map, Table> getTotals() { + calculateTotals(); + return summaries; + } + public String toString() { List> tables = Lists.newArrayList(summaries.keySet()); From f28ce5f5ab967876b8ca2404b5ab739a75ca3942 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Thu, 21 Nov 2024 11:19:43 -0800 Subject: [PATCH 064/199] Update CHANGES.txt --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index f2d97bb3df6b..50c62dcc9151 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1.6 + * Implement preview_repaired auto-repair type (CASSANDRA-20046) * Refresh stale paxos commit (CASSANDRA-19617) * Reduce info logging from automatic paxos repair (CASSANDRA-19445) * Support legacy plain_text_auth section in credentials file removed unintentionally (CASSANDRA-19498) From b65155f2d4b6b7919fd7f2f06b41808534d89f93 Mon Sep 17 00:00:00 2001 From: kzalys Date: Mon, 25 Nov 2024 20:08:49 -0800 Subject: [PATCH 065/199] Update src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java Co-authored-by: Andrew Tolbert <6889771+tolbertam@users.noreply.github.com> --- .../apache/cassandra/repair/autorepair/AutoRepairConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 38c2d10f4975..fbcdd09d9d7c 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -68,7 +68,7 @@ public static AutoRepairState getAutoRepairState(RepairType repairType) return new FullRepairState(); case incremental: return new IncrementalRepairState(); - case preview_repaired:: + case preview_repaired: return new PreviewRepairedState(); } From a2f335889acf931e65bfb0fd57f81d16780e0c8a Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Tue, 26 Nov 2024 22:27:11 -0600 Subject: [PATCH 066/199] Move size comparison up so priority can be factored in Rather than skipping assignments based on size as they are generated, generate all assignments. This way we can use table priorities as a guide for which assignments to exclude. There is still some more work to do here, as we are still splitting by max bytes, where we should really split by remaining bytes. Some further refactoring can be done here but I think this will put more emphasis on table priority when selecting repair assignments and the size of the repair is greater than max schedule bytes. --- .../IAutoRepairTokenRangeSplitter.java | 2 +- .../autorepair/RepairRangeSplitter.java | 176 ++++++++++++++---- .../autorepair/RepairRangeSplitterTest.java | 39 ++-- 3 files changed, 158 insertions(+), 59 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java index 7e679b8dbc9d..fe7fb2519891 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java @@ -54,7 +54,7 @@ public interface IAutoRepairTokenRangeSplitter * The priority is determined using the {@link RepairAssignment#getPriority(AutoRepairConfig.RepairType)} method. */ @VisibleForTesting - default void reorderByPriority(List repairAssignments, AutoRepairConfig.RepairType repairType) + default void reorderByPriority(List repairAssignments, AutoRepairConfig.RepairType repairType) { repairAssignments.sort(Comparator.comparingInt(a -> ((RepairAssignment) a).getPriority(repairType)).reversed()); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 5f4af886bf76..f8cc9a6a6af5 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -25,6 +25,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -138,6 +139,12 @@ public RepairRangeSplitter(Map parameters) tablesPerAssignmentLimit = DEFAULT_TABLE_BATCH_LIMIT; } + if (bytesPerSubrange > maxBytesPerSchedule) + { + throw new RuntimeException(String.format("bytesPerSubrange '%s' cannot be greater than maxBytesPerSchedule '%s'", + FileUtils.stringifyFileSize(bytesPerSubrange), FileUtils.stringifyFileSize(maxBytesPerSchedule))); + } + logger.info("Configured {} with {}={}, {}={}, {}={}, {}={}", RepairRangeSplitter.class.getName(), SUBRANGE_SIZE, subrangeSize, MAX_BYTES_PER_SCHEDULE, maxBytesPerSchedule, PARTITION_COUNT, partitionsPerSubrange, TABLE_BATCH_LIMIT, tablesPerAssignmentLimit); @@ -154,9 +161,9 @@ public List getRepairAssignments(AutoRepairConfig.RepairType r @VisibleForTesting List getRepairAssignments(AutoRepairConfig.RepairType repairType, String keyspaceName, List tableNames, Collection> tokenRanges) { - List repairAssignments = new ArrayList<>(); + List repairAssignments = new ArrayList<>(); // this is used for batching minimal single assignment tables together - List currentAssignments = new ArrayList<>(); + List currentAssignments = new ArrayList<>(); // sort the tables by size so can batch the smallest ones together tableNames.sort((t1, t2) -> { @@ -166,18 +173,30 @@ List getRepairAssignments(AutoRepairConfig.RepairType repairTy throw new IllegalArgumentException(String.format("Could not resolve ColumnFamilyStore from %s.%s", keyspaceName, t1)); return Long.compare(cfs1.metric.totalDiskSpaceUsed.getCount(), cfs2.metric.totalDiskSpaceUsed.getCount()); }); + for (String tableName : tableNames) { - List tableAssignments = getRepairAssignmentsForTable(repairType, keyspaceName, tableName, tokenRanges); + List tableAssignments = getRepairAssignmentsForTable(repairType, keyspaceName, tableName, tokenRanges); if (tableAssignments.isEmpty()) continue; // If the table assignments are for the same token range, and we have room to add more tables to the current assignment - if (tableAssignments.size() == 1 && currentAssignments.size() < tablesPerAssignmentLimit && + if (tableAssignments.size() == 1 && + currentAssignments.size() < tablesPerAssignmentLimit && (currentAssignments.isEmpty() || currentAssignments.get(0).getTokenRange().equals(tableAssignments.get(0).getTokenRange()))) { - currentAssignments.addAll(tableAssignments); + long currentAssignmentsBytes = getBytesInAssignments(currentAssignments); + long tableAssignmentsBytes = getBytesInAssignments(tableAssignments); + // only add assignments together if they don't exceed max bytes per schedule. + if (currentAssignmentsBytes + tableAssignmentsBytes < maxBytesPerSchedule) { + currentAssignments.addAll(tableAssignments); + } + else + { + // add table assignments by themselves + repairAssignments.addAll(tableAssignments); + } } else { @@ -192,12 +211,55 @@ List getRepairAssignments(AutoRepairConfig.RepairType repairTy if (!currentAssignments.isEmpty()) repairAssignments.add(merge(currentAssignments)); + return filterRepairAssignments(repairType, repairAssignments); + } + + /** + * Given a repair type and list of sized-based repair assignments order them by priority of the + * assignments' underlying tables and confine them by maxBytesPerSchedule. + * @param repairType used to determine underyling table priorities + * @param repairAssignments the assignments to filter. + * @return A list of repair assignments ordered by priority and confined by maxBytesPerSchedule. + */ + protected List filterRepairAssignments(AutoRepairConfig.RepairType repairType, List repairAssignments) + { + // Reorder the repair assignments reorderByPriority(repairAssignments, repairType); - return repairAssignments; + + // Confine repair assignemnts by maxBytesPer Schedule if greater than maxBytesPerSchedule. + long assignmentBytes = getBytesInAssignments(repairAssignments); + if (assignmentBytes < maxBytesPerSchedule) + return repairAssignments.stream().map((a) -> (RepairAssignment)a).collect(Collectors.toList()); + else + { + long bytesSoFar = 0L; + List assignmentsToReturn = new ArrayList<>(repairAssignments.size()); + for (SizedRepairAssignment repairAssignment : repairAssignments) { + // skip any repair assignments that would accumulate us past the maxBytesPerSchedule + if (bytesSoFar + repairAssignment.getEstimatedSizeInBytes() > maxBytesPerSchedule) + { + // log that repair assignment was skipped. + warnMaxBytesPerSchedule(repairType, repairAssignment); + } + else + { + assignmentsToReturn.add(repairAssignment); + bytesSoFar += repairAssignment.getEstimatedSizeInBytes(); + } + } + return assignmentsToReturn; + } + } + + @VisibleForTesting + protected static long getBytesInAssignments(List repairAssignments) { + return repairAssignments + .stream() + .mapToLong(SizedRepairAssignment::getEstimatedSizeInBytes).sum(); } @VisibleForTesting - static RepairAssignment merge(List assignments) + static SizedRepairAssignment merge(List assignments) { if (assignments.isEmpty()) throw new IllegalStateException("Cannot merge empty assignments"); @@ -206,7 +268,7 @@ static RepairAssignment merge(List assignments) Range referenceTokenRange = assignments.get(0).getTokenRange(); String referenceKeyspaceName = assignments.get(0).getKeyspaceName(); - for (RepairAssignment assignment : assignments) + for (SizedRepairAssignment assignment : assignments) { // These checks _should_ be unnecessary but are here to ensure that the assignments are consistent if (!assignment.getTokenRange().equals(referenceTokenRange)) @@ -217,40 +279,43 @@ static RepairAssignment merge(List assignments) mergedTableNames.addAll(assignment.getTableNames()); } - return new RepairAssignment(referenceTokenRange, referenceKeyspaceName, new ArrayList<>(mergedTableNames)); + long sizeForAssignment = getBytesInAssignments(assignments); + return new SizedRepairAssignment(referenceTokenRange, referenceKeyspaceName, new ArrayList<>(mergedTableNames), sizeForAssignment); } - public List getRepairAssignmentsForTable(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName, Collection> tokenRanges) + @VisibleForTesting + protected List getRepairAssignmentsForTable(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName, Collection> tokenRanges) { - List repairAssignments = new ArrayList<>(); + List sizeEstimates = getRangeSizeEstimate(repairType, keyspaceName, tableName, tokenRanges); + return getRepairAssignments(sizeEstimates); + } - long targetBytesSoFar = 0; + @VisibleForTesting + protected List getRepairAssignments(List sizeEstimates) + { + List repairAssignments = new ArrayList<>(); - List sizeEstimates = getRangeSizeEstimate(repairType, keyspaceName, tableName, tokenRanges); // since its possible for us to hit maxBytesPerSchedule before seeing all ranges, shuffle so there is chance // at least of hitting all the ranges _eventually_ for the worst case scenarios Collections.shuffle(sizeEstimates); for (SizeEstimate estimate : sizeEstimates) { - // Calculate the smallest subrange if we were to split, this can be used to determine if we can do any - // work if we have a non-zero estimate. - long smallestSubrange = Math.min(estimate.sizeForRepair, bytesPerSubrange); if (estimate.sizeForRepair == 0) { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(estimate.keyspace, estimate.table); long memtableSize = cfs.getTracker().getView().getCurrentMemtable().getLiveDataSize(); if (memtableSize > 0L) { - logger.debug("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", keyspaceName, tableName, estimate.tokenRange, memtableSize); - RepairAssignment assignment = new RepairAssignment(estimate.tokenRange, keyspaceName, Collections.singletonList(tableName)); + logger.debug("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", estimate.keyspace, estimate.table, estimate.tokenRange, memtableSize); + SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), estimate.sizeInRange); repairAssignments.add(assignment); } else { - logger.debug("Skipping {}.{} for range {} because it had no unrepaired SSTables and no memtable data", keyspaceName, tableName, estimate.tokenRange); + logger.debug("Skipping {}.{} for range {} because it had no unrepaired SSTables and no memtable data", estimate.keyspace, estimate.table, estimate.tokenRange); } } - else if (targetBytesSoFar + smallestSubrange < maxBytesPerSchedule) + else { // Check if the estimate needs splitting based on the criteria boolean needsSplitting = estimate.sizeForRepair > bytesPerSubrange || estimate.partitions > partitionsPerSubrange; @@ -262,48 +327,37 @@ else if (targetBytesSoFar + smallestSubrange < maxBytesPerSchedule) Collection> subranges = split(estimate.tokenRange, numberOfSplits); for (Range subrange : subranges) { - if (targetBytesSoFar + approximateBytesPerSplit > maxBytesPerSchedule) - { - warnMaxBytesPerSchedule(repairType, keyspaceName, tableName); - break; - } logger.info("Added repair assignment for {}.{} for subrange {} (#{}/{})", - keyspaceName, tableName, subrange, repairAssignments.size() + 1, numberOfSplits); - RepairAssignment assignment = new RepairAssignment(subrange, keyspaceName, Collections.singletonList(tableName)); + estimate.keyspace, estimate.table, subrange, repairAssignments.size() + 1, numberOfSplits); + SizedRepairAssignment assignment = new SizedRepairAssignment(subrange, estimate.keyspace, Collections.singletonList(estimate.table), approximateBytesPerSplit); repairAssignments.add(assignment); - targetBytesSoFar += approximateBytesPerSplit; } } else { logger.info("Using 1 repair assignment for {}.{} for range {} as rangeBytes={} is less than {}={} and partitionEstimate={} is less than {}={}", - keyspaceName, tableName, estimate.tokenRange, + estimate.keyspace, estimate.table, estimate.tokenRange, FileUtils.stringifyFileSize(estimate.sizeForRepair), SUBRANGE_SIZE, FileUtils.stringifyFileSize(bytesPerSubrange), estimate.partitions, PARTITION_COUNT, partitionsPerSubrange); // No splitting needed, repair the entire range as-is - RepairAssignment assignment = new RepairAssignment(estimate.tokenRange, keyspaceName, Collections.singletonList(tableName)); + SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), estimate.sizeForRepair); repairAssignments.add(assignment); } } - else - { - // Really this is "Ok" but it does mean we are relying on randomness to cover the other ranges - warnMaxBytesPerSchedule(repairType, keyspaceName, tableName); - } } return repairAssignments; } - private void warnMaxBytesPerSchedule(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName) + private void warnMaxBytesPerSchedule(AutoRepairConfig.RepairType repairType, SizedRepairAssignment repairAssignment) { - String warning = "Refusing to add repair assignment for {}.{} because it would increase total repair bytes greater {}"; + String warning = "Refusing to add repair assignment of size {} for {}.{} because it would increase total repair bytes to a value greater than {}"; if (repairType == AutoRepairConfig.RepairType.FULL) { warning = warning + ", everything will not be repaired this schedule. Consider increasing maxBytesPerSchedule, reducing node density or monitoring to ensure all ranges do get repaired within gc_grace_seconds"; } - logger.warn(warning, keyspaceName, tableName, FileUtils.stringifyFileSize(maxBytesPerSchedule)); + logger.warn(warning, repairAssignment.getEstimatedSizeInBytes(), repairAssignment.keyspaceName, repairAssignment.tableNames, FileUtils.stringifyFileSize(maxBytesPerSchedule)); } private int calculateNumberOfSplits(SizeEstimate estimate) @@ -480,4 +534,48 @@ public SizeEstimate(AutoRepairConfig.RepairType repairType, this.sizeForRepair = repairType == AutoRepairConfig.RepairType.INCREMENTAL ? totalSize : sizeInRange; } } + + @VisibleForTesting + protected static class SizedRepairAssignment extends RepairAssignment { + + final long estimatedSizedInBytes; + + public SizedRepairAssignment(Range tokenRange, String keyspaceName, List tableNames, long estimatedSizeInBytes) + { + super(tokenRange, keyspaceName, tableNames); + this.estimatedSizedInBytes = estimatedSizeInBytes; + } + + public long getEstimatedSizeInBytes() { + return estimatedSizedInBytes; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + SizedRepairAssignment that = (SizedRepairAssignment) o; + return estimatedSizedInBytes == that.estimatedSizedInBytes; + } + + @Override + public int hashCode() + { + return Objects.hash(super.hashCode(), estimatedSizedInBytes); + } + + @Override + public String toString() + { + return "SizedRepairAssignement{" + + "estimatedSizedInBytes=" + FileUtils.stringifyFileSize(estimatedSizedInBytes) + + ", keyspaceName='" + keyspaceName + '\'' + + ", tokenRange=" + tokenRange + + ", tableNames=" + tableNames + + '}'; + } + + } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java index 71b60a92232e..2873b2566b0f 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java @@ -39,6 +39,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter.RepairAssignment; +import org.apache.cassandra.repair.autorepair.RepairRangeSplitter.SizedRepairAssignment; import org.apache.cassandra.utils.concurrent.Refs; import static org.apache.cassandra.repair.autorepair.RepairRangeSplitter.TABLE_BATCH_LIMIT; @@ -186,7 +187,7 @@ public void testReorderByPriorityWithOneElement() { @Test public void testGetRepairAssignmentsForTable_NoSSTables() { Collection> ranges = Collections.singleton(new Range<>(Murmur3Partitioner.instance.getMinimumToken(), Murmur3Partitioner.instance.getMaximumToken())); - List assignments = repairRangeSplitter.getRepairAssignmentsForTable(AutoRepairConfig.RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(AutoRepairConfig.RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); assertEquals(0, assignments.size()); } @@ -194,7 +195,7 @@ public void testGetRepairAssignmentsForTable_NoSSTables() { public void testGetRepairAssignmentsForTable_Single() throws Throwable { Collection> ranges = Collections.singleton(new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken())); insertAndFlushSingleTable(tableName); - List assignments = repairRangeSplitter.getRepairAssignmentsForTable(AutoRepairConfig.RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(AutoRepairConfig.RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); assertEquals(1, assignments.size()); } @@ -250,7 +251,7 @@ public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable { @Test(expected = IllegalStateException.class) public void testMergeEmptyAssignments() { // Test when the list of assignments is empty - List emptyAssignments = Collections.emptyList(); + List emptyAssignments = Collections.emptyList(); RepairRangeSplitter.merge(emptyAssignments); } @@ -260,10 +261,10 @@ public void testMergeSingleAssignment() { String keyspaceName = "testKeyspace"; List tableNames = Arrays.asList("table1", "table2"); - RepairAssignment assignment = new RepairAssignment(FULL_RANGE, keyspaceName, tableNames); - List assignments = Collections.singletonList(assignment); + SizedRepairAssignment assignment = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames, 0L); + List assignments = Collections.singletonList(assignment); - RepairAssignment result = RepairRangeSplitter.merge(assignments); + SizedRepairAssignment result = RepairRangeSplitter.merge(assignments); assertEquals(FULL_RANGE, result.getTokenRange()); assertEquals(keyspaceName, result.getKeyspaceName()); @@ -277,11 +278,11 @@ public void testMergeMultipleAssignmentsWithSameTokenRangeAndKeyspace() { List tableNames1 = Arrays.asList("table1", "table2"); List tableNames2 = Arrays.asList("table2", "table3"); - RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, keyspaceName, tableNames1); - RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, keyspaceName, tableNames2); - List assignments = Arrays.asList(assignment1, assignment2); + SizedRepairAssignment assignment1 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames1, 0L); + SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames2, 0L); + List assignments = Arrays.asList(assignment1, assignment2); - RepairAssignment result = RepairRangeSplitter.merge(assignments); + SizedRepairAssignment result = RepairRangeSplitter.merge(assignments); assertEquals(FULL_RANGE, result.getTokenRange()); assertEquals(keyspaceName, result.getKeyspaceName()); @@ -299,9 +300,9 @@ public void testMergeDifferentTokenRange() { String keyspaceName = "testKeyspace"; List tableNames = Arrays.asList("table1", "table2"); - RepairAssignment assignment1 = new RepairAssignment(tokenRange1, keyspaceName, tableNames); - RepairAssignment assignment2 = new RepairAssignment(tokenRange2, keyspaceName, tableNames); - List assignments = Arrays.asList(assignment1, assignment2); + SizedRepairAssignment assignment1 = new SizedRepairAssignment(tokenRange1, keyspaceName, tableNames, 0L); + SizedRepairAssignment assignment2 = new SizedRepairAssignment(tokenRange2, keyspaceName, tableNames, 0L); + List assignments = Arrays.asList(assignment1, assignment2); RepairRangeSplitter.merge(assignments); // Should throw IllegalStateException } @@ -311,9 +312,9 @@ public void testMergeDifferentKeyspaceName() { // Test merging assignments with different keyspace names List tableNames = Arrays.asList("table1", "table2"); - RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, "keyspace1", tableNames); - RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, "keyspace2", tableNames); - List assignments = Arrays.asList(assignment1, assignment2); + SizedRepairAssignment assignment1 = new SizedRepairAssignment(FULL_RANGE, "keyspace1", tableNames, 0L); + SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, "keyspace2", tableNames, 0L); + List assignments = Arrays.asList(assignment1, assignment2); RepairRangeSplitter.merge(assignments); // Should throw IllegalStateException } @@ -325,9 +326,9 @@ public void testMergeWithDuplicateTables() { List tableNames1 = Arrays.asList("table1", "table2"); List tableNames2 = Arrays.asList("table2", "table3"); - RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, keyspaceName, tableNames1); - RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, keyspaceName, tableNames2); - List assignments = Arrays.asList(assignment1, assignment2); + SizedRepairAssignment assignment1 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames1, 0L); + SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames2, 0L); + List assignments = Arrays.asList(assignment1, assignment2); RepairAssignment result = RepairRangeSplitter.merge(assignments); From 1d262004996416c9dd1ab2d43b49b63285ce7fca Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Wed, 27 Nov 2024 08:24:06 -0600 Subject: [PATCH 067/199] Various cleanup --- .../autorepair/RepairRangeSplitter.java | 70 +++++++++++++------ 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index f8cc9a6a6af5..cc8e9df77811 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -141,8 +141,11 @@ public RepairRangeSplitter(Map parameters) if (bytesPerSubrange > maxBytesPerSchedule) { - throw new RuntimeException(String.format("bytesPerSubrange '%s' cannot be greater than maxBytesPerSchedule '%s'", - FileUtils.stringifyFileSize(bytesPerSubrange), FileUtils.stringifyFileSize(maxBytesPerSchedule))); + throw new IllegalArgumentException(String.format("%s='%s' cannot be greater than %s='%s'", + SUBRANGE_SIZE, + FileUtils.stringifyFileSize(bytesPerSubrange), + MAX_BYTES_PER_SCHEDULE, + FileUtils.stringifyFileSize(maxBytesPerSchedule))); } logger.info("Configured {} with {}={}, {}={}, {}={}, {}={}", RepairRangeSplitter.class.getName(), @@ -186,8 +189,8 @@ List getRepairAssignments(AutoRepairConfig.RepairType repairTy currentAssignments.size() < tablesPerAssignmentLimit && (currentAssignments.isEmpty() || currentAssignments.get(0).getTokenRange().equals(tableAssignments.get(0).getTokenRange()))) { - long currentAssignmentsBytes = getBytesInAssignments(currentAssignments); - long tableAssignmentsBytes = getBytesInAssignments(tableAssignments); + long currentAssignmentsBytes = getEstimatedBytes(currentAssignments); + long tableAssignmentsBytes = getEstimatedBytes(tableAssignments); // only add assignments together if they don't exceed max bytes per schedule. if (currentAssignmentsBytes + tableAssignmentsBytes < maxBytesPerSchedule) { currentAssignments.addAll(tableAssignments); @@ -227,7 +230,7 @@ protected List filterRepairAssignments(AutoRepairConfig.Repair reorderByPriority(repairAssignments, repairType); // Confine repair assignemnts by maxBytesPer Schedule if greater than maxBytesPerSchedule. - long assignmentBytes = getBytesInAssignments(repairAssignments); + long assignmentBytes = getEstimatedBytes(repairAssignments); if (assignmentBytes < maxBytesPerSchedule) return repairAssignments.stream().map((a) -> (RepairAssignment)a).collect(Collectors.toList()); else @@ -236,7 +239,7 @@ protected List filterRepairAssignments(AutoRepairConfig.Repair List assignmentsToReturn = new ArrayList<>(repairAssignments.size()); for (SizedRepairAssignment repairAssignment : repairAssignments) { // skip any repair assignments that would accumulate us past the maxBytesPerSchedule - if (bytesSoFar + repairAssignment.getEstimatedSizeInBytes() > maxBytesPerSchedule) + if (bytesSoFar + repairAssignment.getEstimatedBytes() > maxBytesPerSchedule) { // log that repair assignment was skipped. warnMaxBytesPerSchedule(repairType, repairAssignment); @@ -244,18 +247,25 @@ protected List filterRepairAssignments(AutoRepairConfig.Repair else { assignmentsToReturn.add(repairAssignment); - bytesSoFar += repairAssignment.getEstimatedSizeInBytes(); + bytesSoFar += repairAssignment.getEstimatedBytes(); } } return assignmentsToReturn; } } + /** + * @return The sum of {@link SizedRepairAssignment#getEstimatedBytes()} of all given + * repairAssignments. + * @param repairAssignments The assignments to sum + */ @VisibleForTesting - protected static long getBytesInAssignments(List repairAssignments) { + protected static long getEstimatedBytes(List repairAssignments) + { return repairAssignments .stream() - .mapToLong(SizedRepairAssignment::getEstimatedSizeInBytes).sum(); + .mapToLong(SizedRepairAssignment::getEstimatedBytes) + .sum(); } @VisibleForTesting @@ -279,7 +289,7 @@ static SizedRepairAssignment merge(List assignments) mergedTableNames.addAll(assignment.getTableNames()); } - long sizeForAssignment = getBytesInAssignments(assignments); + long sizeForAssignment = getEstimatedBytes(assignments); return new SizedRepairAssignment(referenceTokenRange, referenceKeyspaceName, new ArrayList<>(mergedTableNames), sizeForAssignment); } @@ -352,12 +362,20 @@ protected List getRepairAssignments(List si private void warnMaxBytesPerSchedule(AutoRepairConfig.RepairType repairType, SizedRepairAssignment repairAssignment) { - String warning = "Refusing to add repair assignment of size {} for {}.{} because it would increase total repair bytes to a value greater than {}"; + String warning = "Refusing to add repair assignment of size {} for {}.{} because it would increase total repair bytes to a value greater than {} ({})"; if (repairType == AutoRepairConfig.RepairType.FULL) { warning = warning + ", everything will not be repaired this schedule. Consider increasing maxBytesPerSchedule, reducing node density or monitoring to ensure all ranges do get repaired within gc_grace_seconds"; } - logger.warn(warning, repairAssignment.getEstimatedSizeInBytes(), repairAssignment.keyspaceName, repairAssignment.tableNames, FileUtils.stringifyFileSize(maxBytesPerSchedule)); + // TODO: Add two metrics: + // Meter: RepairRangeSplitter.RepairType.SkippedAssignments + // Meter: RepairRangeSplitter.RepairType.SkippedBytes + logger.warn(warning, + repairAssignment.getEstimatedBytes(), + repairAssignment.keyspaceName, + repairAssignment.tableNames, + FileUtils.stringifyFileSize(maxBytesPerSchedule), + FileUtils.stringifyFileSize(maxBytesPerSchedule + repairAssignment.getEstimatedBytes())); } private int calculateNumberOfSplits(SizeEstimate estimate) @@ -386,7 +404,7 @@ private int calculateNumberOfSplits(SizeEstimate estimate) return splits; } - public Collection> getTokenRanges(boolean primaryRangeOnly, String keyspaceName) + private Collection> getTokenRanges(boolean primaryRangeOnly, String keyspaceName) { // Collect all applicable token ranges Collection> wrappedRanges; @@ -535,19 +553,28 @@ public SizeEstimate(AutoRepairConfig.RepairType repairType, } } + /** + * Implementation of RepairAssignment that also assigns an estimation of bytes involved + * in the repair. + */ @VisibleForTesting protected static class SizedRepairAssignment extends RepairAssignment { - final long estimatedSizedInBytes; + final long estimatedBytes; - public SizedRepairAssignment(Range tokenRange, String keyspaceName, List tableNames, long estimatedSizeInBytes) + public SizedRepairAssignment(Range tokenRange, String keyspaceName, List tableNames, long estimatedBytes) { super(tokenRange, keyspaceName, tableNames); - this.estimatedSizedInBytes = estimatedSizeInBytes; + this.estimatedBytes = estimatedBytes; } - public long getEstimatedSizeInBytes() { - return estimatedSizedInBytes; + /** + * Estimated bytes involved in the assignment. Typically Derived from {@link SizeEstimate#sizeForRepair}. + * @return estimated bytes involved in the assignment. + */ + public long getEstimatedBytes() + { + return estimatedBytes; } @Override @@ -557,25 +584,24 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; SizedRepairAssignment that = (SizedRepairAssignment) o; - return estimatedSizedInBytes == that.estimatedSizedInBytes; + return estimatedBytes == that.estimatedBytes; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), estimatedSizedInBytes); + return Objects.hash(super.hashCode(), estimatedBytes); } @Override public String toString() { return "SizedRepairAssignement{" + - "estimatedSizedInBytes=" + FileUtils.stringifyFileSize(estimatedSizedInBytes) + + "estimatedSizedInBytes=" + FileUtils.stringifyFileSize(estimatedBytes) + ", keyspaceName='" + keyspaceName + '\'' + ", tokenRange=" + tokenRange + ", tableNames=" + tableNames + '}'; } - } } From 98833ba031c71656fcf367f3a1f5b371342b204a Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Wed, 27 Nov 2024 09:12:58 -0600 Subject: [PATCH 068/199] Handle EmpySSTableScanner being returned from getScanner If no SSTables cover a requested range an EmptySSTableScanner will be returned, handle this. --- .../autorepair/RepairRangeSplitter.java | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index cc8e9df77811..6173d3d6444a 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -46,6 +46,7 @@ import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.big.BigTableScanner; import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; @@ -316,13 +317,13 @@ protected List getRepairAssignments(List si long memtableSize = cfs.getTracker().getView().getCurrentMemtable().getLiveDataSize(); if (memtableSize > 0L) { - logger.debug("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", estimate.keyspace, estimate.table, estimate.tokenRange, memtableSize); + logger.info("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", estimate.keyspace, estimate.table, estimate.tokenRange, memtableSize); SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), estimate.sizeInRange); repairAssignments.add(assignment); } else { - logger.debug("Skipping {}.{} for range {} because it had no unrepaired SSTables and no memtable data", estimate.keyspace, estimate.table, estimate.tokenRange); + logger.info("Skipping {}.{} for range {} because it had no unrepaired SSTables and no memtable data", estimate.keyspace, estimate.table, estimate.tokenRange); } } else @@ -465,22 +466,30 @@ static SizeEstimate getSizesForRangeOfSSTables(AutoRepairConfig.RepairType repai // still better to use the cardinality estimator then the index since it wont count duplicates. // get the bounds of the sstable for this range using the index file but do not actually read it. List> bounds = BigTableScanner.makeBounds(reader, Collections.singleton(tokenRange)); - try (BigTableScanner scanner = (BigTableScanner) BigTableScanner.getScanner(reader, Collections.singleton(tokenRange))) - { - assert bounds.size() == 1; - AbstractBounds bound = bounds.get(0); - long startPosition = scanner.getDataPosition(bound.left); - long endPosition = scanner.getDataPosition(bound.right); - // If end position is 0 we can assume the sstable ended before that token, bound at size of file - if (endPosition == 0) + ISSTableScanner rangeScanner = BigTableScanner.getScanner(reader, Collections.singleton(tokenRange)); + // Type check scanner returned as it may be an EmptySSTableScanner if the range is not covered in the + // SSTable, in this case we will avoid incrementing approxBytesInRange. + if (rangeScanner instanceof BigTableScanner) + { + try (BigTableScanner scanner = (BigTableScanner) rangeScanner) { - endPosition = sstableSize; + assert bounds.size() == 1; + + AbstractBounds bound = bounds.get(0); + long startPosition = scanner.getDataPosition(bound.left); + long endPosition = scanner.getDataPosition(bound.right); + // If end position is 0 we can assume the sstable ended before that token, bound at size of file + if (endPosition == 0) + { + endPosition = sstableSize; + } + + long approximateRangeBytesInSSTable = Math.max(0, endPosition - startPosition); + approxBytesInRange += Math.min(approximateRangeBytesInSSTable, sstableSize); } - - long approximateRangeBytesInSSTable = Math.max(0, endPosition - startPosition); - approxBytesInRange += Math.min(approximateRangeBytesInSSTable, sstableSize); } + } catch (IOException | CardinalityMergeException e) { From 7ca4d2800954fc3bb28182402a043cd059944a01 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Wed, 27 Nov 2024 11:47:16 -0600 Subject: [PATCH 069/199] Minor refactoring around bytesPerSubrange, logging Avoid intermediate variable by making DEFAULT_SUBRANGE_SIZE a long. Log bytes using stringifyFileSize --- .../repair/autorepair/RepairRangeSplitter.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 6173d3d6444a..4ec73e968068 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -95,23 +95,21 @@ public class RepairRangeSplitter implements IAutoRepairTokenRangeSplitter private final long bytesPerSubrange; private final long partitionsPerSubrange; - private static final DataStorageSpec.LongBytesBound DEFAULT_SUBRANGE_SIZE = new DataStorageSpec.LongBytesBound("100GiB"); + private static final long DEFAULT_SUBRANGE_SIZE = new DataStorageSpec.LongBytesBound("100GiB").toBytes(); private static final long DEFAULT_MAX_BYTES_PER_SCHEDULE = Long.MAX_VALUE; private static final long DEFAULT_PARTITION_LIMIT = (long) Math.pow(2, DatabaseDescriptor.getRepairSessionMaxTreeDepth()); private static final int DEFAULT_TABLE_BATCH_LIMIT = 64; public RepairRangeSplitter(Map parameters) { - DataStorageSpec.LongBytesBound subrangeSize; if (parameters.containsKey(SUBRANGE_SIZE)) { - subrangeSize = new DataStorageSpec.LongBytesBound(parameters.get(SUBRANGE_SIZE)); + bytesPerSubrange = new DataStorageSpec.LongBytesBound(parameters.get(SUBRANGE_SIZE)).toBytes(); } else { - subrangeSize = DEFAULT_SUBRANGE_SIZE; + bytesPerSubrange = DEFAULT_SUBRANGE_SIZE; } - bytesPerSubrange = subrangeSize.toBytes(); if (parameters.containsKey(MAX_BYTES_PER_SCHEDULE)) { @@ -150,7 +148,8 @@ public RepairRangeSplitter(Map parameters) } logger.info("Configured {} with {}={}, {}={}, {}={}, {}={}", RepairRangeSplitter.class.getName(), - SUBRANGE_SIZE, subrangeSize, MAX_BYTES_PER_SCHEDULE, maxBytesPerSchedule, + SUBRANGE_SIZE, FileUtils.stringifyFileSize(bytesPerSubrange), + MAX_BYTES_PER_SCHEDULE, FileUtils.stringifyFileSize(maxBytesPerSchedule), PARTITION_COUNT, partitionsPerSubrange, TABLE_BATCH_LIMIT, tablesPerAssignmentLimit); } @@ -372,7 +371,7 @@ private void warnMaxBytesPerSchedule(AutoRepairConfig.RepairType repairType, Siz // Meter: RepairRangeSplitter.RepairType.SkippedAssignments // Meter: RepairRangeSplitter.RepairType.SkippedBytes logger.warn(warning, - repairAssignment.getEstimatedBytes(), + FileUtils.stringifyFileSize(repairAssignment.getEstimatedBytes()), repairAssignment.keyspaceName, repairAssignment.tableNames, FileUtils.stringifyFileSize(maxBytesPerSchedule), From 4a7da1870365a2a8bae462256f480d76740a6f98 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:47:27 -0600 Subject: [PATCH 070/199] Cleanup logging of repair assignments --- .../autorepair/RepairRangeSplitter.java | 163 ++++++++++++------ .../autorepair/RepairRangeSplitterTest.java | 18 +- 2 files changed, 124 insertions(+), 57 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 4ec73e968068..5f97cbc0ef59 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -211,10 +211,11 @@ List getRepairAssignments(AutoRepairConfig.RepairType repairTy repairAssignments.addAll(tableAssignments); } } + if (!currentAssignments.isEmpty()) repairAssignments.add(merge(currentAssignments)); - return filterRepairAssignments(repairType, repairAssignments); + return filterRepairAssignments(repairType, keyspaceName, repairAssignments); } /** @@ -224,34 +225,74 @@ List getRepairAssignments(AutoRepairConfig.RepairType repairTy * @param repairAssignments the assignments to filter. * @return A list of repair assignments ordered by priority and confined by maxBytesPerSchedule. */ - protected List filterRepairAssignments(AutoRepairConfig.RepairType repairType, List repairAssignments) + protected List filterRepairAssignments(AutoRepairConfig.RepairType repairType, String keyspaceName, List repairAssignments) { + if (repairAssignments.isEmpty()) + return Collections.emptyList(); + // Reorder the repair assignments reorderByPriority(repairAssignments, repairType); - // Confine repair assignemnts by maxBytesPer Schedule if greater than maxBytesPerSchedule. - long assignmentBytes = getEstimatedBytes(repairAssignments); - if (assignmentBytes < maxBytesPerSchedule) - return repairAssignments.stream().map((a) -> (RepairAssignment)a).collect(Collectors.toList()); - else + // Confine repair assignments by maxBytesPerSchedule. + long bytesSoFar = 0L; + boolean isIncremental = false; + long bytesNotRepaired = 0L; + int assignmentsNotRepaired = 0; + List assignmentsToReturn = new ArrayList<>(repairAssignments.size()); + for (SizedRepairAssignment repairAssignment : repairAssignments) { - long bytesSoFar = 0L; - List assignmentsToReturn = new ArrayList<>(repairAssignments.size()); - for (SizedRepairAssignment repairAssignment : repairAssignments) { - // skip any repair assignments that would accumulate us past the maxBytesPerSchedule - if (bytesSoFar + repairAssignment.getEstimatedBytes() > maxBytesPerSchedule) - { - // log that repair assignment was skipped. - warnMaxBytesPerSchedule(repairType, repairAssignment); - } - else - { - assignmentsToReturn.add(repairAssignment); - bytesSoFar += repairAssignment.getEstimatedBytes(); - } + // skip any repair assignments that would accumulate us past the maxBytesPerSchedule + if (bytesSoFar + repairAssignment.getEstimatedBytes() > maxBytesPerSchedule) + { + // log that repair assignment was skipped. + bytesNotRepaired += repairAssignment.getEstimatedBytes(); + assignmentsNotRepaired++; + logger.warn("Skipping {} because it would increase total repair bytes to {}", + repairAssignment, + getBytesOfMaxBytesPerSchedule(bytesSoFar + repairAssignment.getEstimatedBytes())); + } + else + { + bytesSoFar += repairAssignment.getEstimatedBytes(); + logger.info("Adding {}, increasing repair bytes to {}", + repairAssignment, + getBytesOfMaxBytesPerSchedule(bytesSoFar)); + assignmentsToReturn.add(repairAssignment); } - return assignmentsToReturn; } + + String message = "Returning {} assignment(s) for {}, totaling {}"; + if (assignmentsNotRepaired != 0) + { + message += ". Skipping {} of {} assignment(s), totaling {}"; + if (repairType != AutoRepairConfig.RepairType.INCREMENTAL) + { + message += ". The entire primary range will not be repaired this schedule. " + + "Consider increasing maxBytesPerSchedule, reducing node density or monitoring to ensure " + + "all ranges do get repaired within gc_grace_seconds"; + logger.warn(message, assignmentsToReturn.size(), keyspaceName, getBytesOfMaxBytesPerSchedule(bytesSoFar), + assignmentsNotRepaired, repairAssignments.size(), + FileUtils.stringifyFileSize(bytesNotRepaired)); + } + else + { + logger.info(message, assignmentsToReturn.size(), keyspaceName, getBytesOfMaxBytesPerSchedule(bytesSoFar), + assignmentsNotRepaired, repairAssignments.size(), + FileUtils.stringifyFileSize(bytesNotRepaired)); + } + } + else + { + logger.info(message, assignmentsToReturn.size(), keyspaceName, getBytesOfMaxBytesPerSchedule(bytesSoFar)); + } + return assignmentsToReturn; + } + + private String getBytesOfMaxBytesPerSchedule(long bytes) { + if (maxBytesPerSchedule == Long.MAX_VALUE) + return FileUtils.stringifyFileSize(bytes); + else + return String.format("%s of %s", FileUtils.stringifyFileSize(bytes), FileUtils.stringifyFileSize(maxBytesPerSchedule)); } /** @@ -290,7 +331,8 @@ static SizedRepairAssignment merge(List assignments) } long sizeForAssignment = getEstimatedBytes(assignments); - return new SizedRepairAssignment(referenceTokenRange, referenceKeyspaceName, new ArrayList<>(mergedTableNames), sizeForAssignment); + return new SizedRepairAssignment(referenceTokenRange, referenceKeyspaceName, new ArrayList<>(mergedTableNames), + "full primary range for " + mergedTableNames.size() + " tables", sizeForAssignment); } @VisibleForTesting @@ -316,20 +358,19 @@ protected List getRepairAssignments(List si long memtableSize = cfs.getTracker().getView().getCurrentMemtable().getLiveDataSize(); if (memtableSize > 0L) { - logger.info("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", estimate.keyspace, estimate.table, estimate.tokenRange, memtableSize); - SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), estimate.sizeInRange); + logger.debug("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", estimate.keyspace, estimate.table, estimate.tokenRange, memtableSize); + SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "memtable only", estimate.sizeInRange); repairAssignments.add(assignment); } else { - logger.info("Skipping {}.{} for range {} because it had no unrepaired SSTables and no memtable data", estimate.keyspace, estimate.table, estimate.tokenRange); + logger.debug("Skipping {}.{} for range {} because it had no unrepaired SSTables and no memtable data", estimate.keyspace, estimate.table, estimate.tokenRange); } } else { // Check if the estimate needs splitting based on the criteria boolean needsSplitting = estimate.sizeForRepair > bytesPerSubrange || estimate.partitions > partitionsPerSubrange; - if (needsSplitting) { int numberOfSplits = calculateNumberOfSplits(estimate); @@ -337,22 +378,18 @@ protected List getRepairAssignments(List si Collection> subranges = split(estimate.tokenRange, numberOfSplits); for (Range subrange : subranges) { - logger.info("Added repair assignment for {}.{} for subrange {} (#{}/{})", - estimate.keyspace, estimate.table, subrange, repairAssignments.size() + 1, numberOfSplits); - SizedRepairAssignment assignment = new SizedRepairAssignment(subrange, estimate.keyspace, Collections.singletonList(estimate.table), approximateBytesPerSplit); + SizedRepairAssignment assignment = new SizedRepairAssignment(subrange, estimate.keyspace, Collections.singletonList(estimate.table), + String.format("subrange %d of %d", repairAssignments.size()+1, numberOfSplits), + approximateBytesPerSplit); repairAssignments.add(assignment); } } else { - logger.info("Using 1 repair assignment for {}.{} for range {} as rangeBytes={} is less than {}={} and partitionEstimate={} is less than {}={}", - estimate.keyspace, estimate.table, estimate.tokenRange, - FileUtils.stringifyFileSize(estimate.sizeForRepair), - SUBRANGE_SIZE, FileUtils.stringifyFileSize(bytesPerSubrange), - estimate.partitions, - PARTITION_COUNT, partitionsPerSubrange); // No splitting needed, repair the entire range as-is - SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), estimate.sizeForRepair); + SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, + Collections.singletonList(estimate.table), + "full primary range for table", estimate.sizeForRepair); repairAssignments.add(assignment); } } @@ -362,7 +399,7 @@ protected List getRepairAssignments(List si private void warnMaxBytesPerSchedule(AutoRepairConfig.RepairType repairType, SizedRepairAssignment repairAssignment) { - String warning = "Refusing to add repair assignment of size {} for {}.{} because it would increase total repair bytes to a value greater than {} ({})"; + String warning = "Refusing to add repair assignment {} because it would increase total repair bytes to a value greater than {} ({})"; if (repairType == AutoRepairConfig.RepairType.FULL) { warning = warning + ", everything will not be repaired this schedule. Consider increasing maxBytesPerSchedule, reducing node density or monitoring to ensure all ranges do get repaired within gc_grace_seconds"; @@ -371,9 +408,7 @@ private void warnMaxBytesPerSchedule(AutoRepairConfig.RepairType repairType, Siz // Meter: RepairRangeSplitter.RepairType.SkippedAssignments // Meter: RepairRangeSplitter.RepairType.SkippedBytes logger.warn(warning, - FileUtils.stringifyFileSize(repairAssignment.getEstimatedBytes()), - repairAssignment.keyspaceName, - repairAssignment.tableNames, + repairAssignment, FileUtils.stringifyFileSize(maxBytesPerSchedule), FileUtils.stringifyFileSize(maxBytesPerSchedule + repairAssignment.getEstimatedBytes())); } @@ -435,7 +470,7 @@ private List getRangeSizeEstimate(AutoRepairConfig.RepairType repa try (Refs refs = getSSTableReaderRefs(repairType, keyspace, table, tokenRange)) { SizeEstimate estimate = getSizesForRangeOfSSTables(repairType, keyspace, table, tokenRange, refs); - logger.debug("Size estimate for {}.{} for range {} is {}", keyspace, table, tokenRange, estimate); + logger.debug("Generated size estimate {}", estimate); sizeEstimates.add(estimate); } } @@ -559,6 +594,21 @@ public SizeEstimate(AutoRepairConfig.RepairType repairType, this.sizeForRepair = repairType == AutoRepairConfig.RepairType.INCREMENTAL ? totalSize : sizeInRange; } + + @Override + public String toString() + { + return "SizeEstimate{" + + "repairType=" + repairType + + ", keyspace='" + keyspace + '\'' + + ", table='" + table + '\'' + + ", tokenRange=" + tokenRange + + ", partitions=" + partitions + + ", sizeInRange=" + sizeInRange + + ", totalSize=" + totalSize + + ", sizeForRepair=" + sizeForRepair + + '}'; + } } /** @@ -568,14 +618,30 @@ public SizeEstimate(AutoRepairConfig.RepairType repairType, @VisibleForTesting protected static class SizedRepairAssignment extends RepairAssignment { + final String description; final long estimatedBytes; - public SizedRepairAssignment(Range tokenRange, String keyspaceName, List tableNames, long estimatedBytes) + public SizedRepairAssignment(Range tokenRange, String keyspaceName, List tableNames) + { + this(tokenRange, keyspaceName, tableNames, "", 0L); + } + + public SizedRepairAssignment(Range tokenRange, String keyspaceName, List tableNames, + String description, + long estimatedBytes) { super(tokenRange, keyspaceName, tableNames); + this.description = description; this.estimatedBytes = estimatedBytes; } + /** + * @return Additional metadata about the repair assignment. + */ + public String getDescription() { + return description; + } + /** * Estimated bytes involved in the assignment. Typically Derived from {@link SizeEstimate#sizeForRepair}. * @return estimated bytes involved in the assignment. @@ -592,23 +658,24 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; SizedRepairAssignment that = (SizedRepairAssignment) o; - return estimatedBytes == that.estimatedBytes; + return estimatedBytes == that.estimatedBytes && Objects.equals(description, that.description); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), estimatedBytes); + return Objects.hash(super.hashCode(), description, estimatedBytes); } @Override public String toString() { - return "SizedRepairAssignement{" + - "estimatedSizedInBytes=" + FileUtils.stringifyFileSize(estimatedBytes) + - ", keyspaceName='" + keyspaceName + '\'' + + return "SizedRepairAssignment{" + + "description='" + description + '\'' + ", tokenRange=" + tokenRange + + ", keyspaceName='" + keyspaceName + '\'' + ", tableNames=" + tableNames + + ", estimatedBytes=" + FileUtils.stringifyFileSize(estimatedBytes) + '}'; } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java index 2873b2566b0f..6c91fbd539be 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java @@ -261,7 +261,7 @@ public void testMergeSingleAssignment() { String keyspaceName = "testKeyspace"; List tableNames = Arrays.asList("table1", "table2"); - SizedRepairAssignment assignment = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames, 0L); + SizedRepairAssignment assignment = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames); List assignments = Collections.singletonList(assignment); SizedRepairAssignment result = RepairRangeSplitter.merge(assignments); @@ -278,8 +278,8 @@ public void testMergeMultipleAssignmentsWithSameTokenRangeAndKeyspace() { List tableNames1 = Arrays.asList("table1", "table2"); List tableNames2 = Arrays.asList("table2", "table3"); - SizedRepairAssignment assignment1 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames1, 0L); - SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames2, 0L); + SizedRepairAssignment assignment1 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames1); + SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames2); List assignments = Arrays.asList(assignment1, assignment2); SizedRepairAssignment result = RepairRangeSplitter.merge(assignments); @@ -300,8 +300,8 @@ public void testMergeDifferentTokenRange() { String keyspaceName = "testKeyspace"; List tableNames = Arrays.asList("table1", "table2"); - SizedRepairAssignment assignment1 = new SizedRepairAssignment(tokenRange1, keyspaceName, tableNames, 0L); - SizedRepairAssignment assignment2 = new SizedRepairAssignment(tokenRange2, keyspaceName, tableNames, 0L); + SizedRepairAssignment assignment1 = new SizedRepairAssignment(tokenRange1, keyspaceName, tableNames); + SizedRepairAssignment assignment2 = new SizedRepairAssignment(tokenRange2, keyspaceName, tableNames); List assignments = Arrays.asList(assignment1, assignment2); RepairRangeSplitter.merge(assignments); // Should throw IllegalStateException @@ -312,8 +312,8 @@ public void testMergeDifferentKeyspaceName() { // Test merging assignments with different keyspace names List tableNames = Arrays.asList("table1", "table2"); - SizedRepairAssignment assignment1 = new SizedRepairAssignment(FULL_RANGE, "keyspace1", tableNames, 0L); - SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, "keyspace2", tableNames, 0L); + SizedRepairAssignment assignment1 = new SizedRepairAssignment(FULL_RANGE, "keyspace1", tableNames); + SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, "keyspace2", tableNames); List assignments = Arrays.asList(assignment1, assignment2); RepairRangeSplitter.merge(assignments); // Should throw IllegalStateException @@ -326,8 +326,8 @@ public void testMergeWithDuplicateTables() { List tableNames1 = Arrays.asList("table1", "table2"); List tableNames2 = Arrays.asList("table2", "table3"); - SizedRepairAssignment assignment1 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames1, 0L); - SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames2, 0L); + SizedRepairAssignment assignment1 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames1); + SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames2); List assignments = Arrays.asList(assignment1, assignment2); RepairAssignment result = RepairRangeSplitter.merge(assignments); From f2fa73f2f45644ad0936c2dc44fb49a41e0b0a61 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sat, 30 Nov 2024 12:04:09 -0600 Subject: [PATCH 071/199] Remove UnrepairedBytesBasedTokenRangeSplitter --- ...nrepairedBytesBasedTokenRangeSplitter.java | 336 ------------------ 1 file changed, 336 deletions(-) delete mode 100644 src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java diff --git a/src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java deleted file mode 100644 index 2ac0813b034e..000000000000 --- a/src/java/org/apache/cassandra/repair/autorepair/UnrepairedBytesBasedTokenRangeSplitter.java +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.repair.autorepair; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.config.DataStorageSpec; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.PartitionPosition; -import org.apache.cassandra.db.lifecycle.SSTableIntervalTree; -import org.apache.cassandra.db.lifecycle.SSTableSet; -import org.apache.cassandra.db.lifecycle.View; -import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.big.BigTableScanner; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.concurrent.Refs; - -import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; - -public class UnrepairedBytesBasedTokenRangeSplitter implements IAutoRepairTokenRangeSplitter -{ - private static final Logger logger = LoggerFactory.getLogger(UnrepairedBytesBasedTokenRangeSplitter.class); - - static final String SUBRANGE_SIZE = "subrange_size"; - static final String MAX_BYTES_PER_SCHEDULE = "max_bytes_per_schedule"; - - // target bytes per subrange - private final DataStorageSpec.LongBytesBound subrangeSize; - - // maximum target bytes to repair - private final DataStorageSpec.LongBytesBound maxBytesPerSchedule; - - private final long subrangeBytes; - - private final long maxBytesPerScheduleBytes; - - private static final DataStorageSpec.LongBytesBound DEFAULT_SUBRANGE_SIZE = new DataStorageSpec.LongBytesBound("100GiB"); - private static final DataStorageSpec.LongBytesBound DEFAULT_MAX_BYTES_PER_SCHEDULE = new DataStorageSpec.LongBytesBound("500GiB"); - - public UnrepairedBytesBasedTokenRangeSplitter(Map parameters) - { - // Demonstrates parameterizing a range splitter so we can have splitter specific options. - if (parameters.containsKey(SUBRANGE_SIZE)) - { - subrangeSize = new DataStorageSpec.LongBytesBound(parameters.get(SUBRANGE_SIZE)); - } - else - { - subrangeSize = DEFAULT_SUBRANGE_SIZE; - } - subrangeBytes = subrangeSize.toBytes(); - - if (parameters.containsKey(MAX_BYTES_PER_SCHEDULE)) - { - maxBytesPerSchedule = new DataStorageSpec.LongBytesBound(parameters.get(MAX_BYTES_PER_SCHEDULE)); - } - else - { - maxBytesPerSchedule = DEFAULT_MAX_BYTES_PER_SCHEDULE; - } - maxBytesPerScheduleBytes = maxBytesPerSchedule.toBytes(); - - logger.info("Configured {} with {}={}, {}={}", UnrepairedBytesBasedTokenRangeSplitter.class.getName(), - SUBRANGE_SIZE, subrangeSize, MAX_BYTES_PER_SCHEDULE, maxBytesPerSchedule); - } - - @Override - public List getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames) - { - List repairAssignments = new ArrayList<>(); - - logger.info("Calculating token range splits for repairType={} primaryRangeOnly={} keyspaceName={} tableNames={}", repairType, primaryRangeOnly, keyspaceName, tableNames); - if (repairType != AutoRepairConfig.RepairType.INCREMENTAL) - { - throw new IllegalArgumentException(this.getClass().getName() + " only supports " + AutoRepairConfig.RepairType.INCREMENTAL + " repair"); - } - - // TODO: create a custom repair assignment that indicates number of bytes in repair and join tables by byte size. - Collection> tokenRanges = getTokenRanges(primaryRangeOnly, keyspaceName); - for (String tableName : tableNames) - { - repairAssignments.addAll(getRepairAssignmentsForTable(keyspaceName, tableName, tokenRanges)); - } - return repairAssignments; - } - - public List getRepairAssignmentsForTable(String keyspaceName, String tableName, Collection> tokenRanges) - { - List repairAssignments = new ArrayList<>(); - - long targetBytesSoFar = 0; - - for (Range tokenRange : tokenRanges) - { - logger.info("Calculating unrepaired bytes for {}.{} for range {}", keyspaceName, tableName, tokenRange); - // Capture the amount of unrepaired bytes for range - long approximateUnrepairedBytesForRange = 0L; - // Capture the total bytes in read sstables, this will be useful for calculating the ratio - // of data in SSTables including this range and also useful to know how much anticompaction there will be. - long totalBytesInUnrepairedSSTables = 0L; - try (Refs refs = getSSTableReaderRefs(keyspaceName, tableName, tokenRange)) - { - for (SSTableReader reader : refs) - { - // Only evaluate unrepaired SSTables. - if (!reader.isRepaired()) - { - long sstableSize = reader.bytesOnDisk(); - totalBytesInUnrepairedSSTables += sstableSize; - // get the bounds of the sstable for this range using the index file but do not actually read it. - List> bounds = BigTableScanner.makeBounds(reader, Collections.singleton(tokenRange)); - try (BigTableScanner scanner = (BigTableScanner) BigTableScanner.getScanner(reader, Collections.singleton(tokenRange))) - { - assert bounds.size() == 1; - - AbstractBounds bound = bounds.get(0); - long startPosition = scanner.getDataPosition(bound.left); - long endPosition = scanner.getDataPosition(bound.right); - // If end position is 0 we can assume the sstable ended before that token, bound at size of file - if (endPosition == 0) - { - endPosition = sstableSize; - } - - long approximateRangeBytesInSSTable = Math.max(0, endPosition - startPosition); - // get the fraction of the sstable belonging to the range. - approximateUnrepairedBytesForRange += Math.min(approximateRangeBytesInSSTable, sstableSize); - double ratio = approximateRangeBytesInSSTable / (double) sstableSize; - logger.info("Calculations for {}.{} {}: sstableSize={}, rangeBytesInSSTable={}, startPosition={}, endPosition={}, ratio={}", - keyspaceName, tableName, reader.descriptor.baseFilename(), - FileUtils.stringifyFileSize(sstableSize), FileUtils.stringifyFileSize(approximateRangeBytesInSSTable), startPosition, endPosition, ratio); - } - } - else - { - logger.info("Skipping over {}.{} {} ({}) because it is repaired", keyspaceName, tableName, reader.descriptor.baseFilename(), FileUtils.stringifyFileSize(reader.bytesOnDisk())); - } - } - } - - // Only consider token range if it had unrepaired sstables or live data in memtables. - if (totalBytesInUnrepairedSSTables > 0L) - { - // TODO: Possibly some anticompaction configuration we want here, where if we detect a large amount of anticompaction we want to reduce the work we do. - double ratio = approximateUnrepairedBytesForRange / (double) totalBytesInUnrepairedSSTables; - logger.info("Calculated unrepaired bytes for {}.{} for range {}: sstableSize={}, rangeBytesInSSTables={}, ratio={}", keyspaceName, tableName, tokenRange, - FileUtils.stringifyFileSize(totalBytesInUnrepairedSSTables), FileUtils.stringifyFileSize(approximateUnrepairedBytesForRange), ratio); - - // TODO: split on byte size here, this is currently a bit naive in assuming that data is evenly distributed among the range which may not be the - // right assumption. May want to consider when splitting on these ranges to reevaluate how much data is in the range, but for this - // exists as a demonstration. - if (approximateUnrepairedBytesForRange < subrangeBytes) - { - // accept range as is if less than bytes. - logger.info("Using 1 repair assignment for {}.{} for range {} as {} is less than {}", keyspaceName, tableName, tokenRange, - FileUtils.stringifyFileSize(approximateUnrepairedBytesForRange), subrangeSize); - // TODO: this is a bit repetitive see if can reduce more. - RepairAssignment assignment = new BytesBasedRepairAssignment(tokenRange, keyspaceName, Collections.singletonList(tableName), approximateUnrepairedBytesForRange); - if (canAddAssignment(assignment, targetBytesSoFar, approximateUnrepairedBytesForRange)) - { - repairAssignments.add(assignment); - targetBytesSoFar += approximateUnrepairedBytesForRange; - } - else - return repairAssignments; - } - else - { - long targetRanges = approximateUnrepairedBytesForRange / subrangeBytes; - // TODO: approximation per range, this is a bit lossy since targetRanges rounds down. - long approximateBytesPerSplit = approximateUnrepairedBytesForRange / targetRanges; - logger.info("Splitting {}.{} for range {} into {} sub ranges, approximateBytesPerSplit={}", keyspaceName, tableName, tokenRange, targetRanges, FileUtils.stringifyFileSize(approximateBytesPerSplit)); - Collection> splitRanges = split(tokenRange, (int) targetRanges); - int splitRangeCount = 0; - for (Range splitRange : splitRanges) - { - RepairAssignment assignment = new BytesBasedRepairAssignment(splitRange, keyspaceName, Collections.singletonList(tableName), approximateBytesPerSplit); - if (canAddAssignment(assignment, targetBytesSoFar, approximateBytesPerSplit)) - { - logger.info("Added repair assignment for {}.{} for subrange {} (#{}/{}) with approximateBytes={}", - keyspaceName, tableName, splitRange, ++splitRangeCount, splitRanges.size(), FileUtils.stringifyFileSize(approximateBytesPerSplit)); - repairAssignments.add(assignment); - targetBytesSoFar += approximateBytesPerSplit; - } - else - return repairAssignments; - } - } - } - else - { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); - long memtableSize = cfs.getTracker().getView().getCurrentMemtable().getLiveDataSize(); - if (memtableSize > 0L) - { - logger.info("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", keyspaceName, tableName, tokenRange, memtableSize); - RepairAssignment assignment = new BytesBasedRepairAssignment(tokenRange, keyspaceName, Collections.singletonList(tableName), memtableSize); - if (targetBytesSoFar >= maxBytesPerScheduleBytes) - { - return repairAssignments; - } - repairAssignments.add(assignment); - targetBytesSoFar += memtableSize; - } - else - { - logger.info("Skipping {}.{} for range {} because it had no unrepaired SSTables and no memtable data", keyspaceName, tableName, tokenRange); - } - } - } - return repairAssignments; - } - - private boolean canAddAssignment(RepairAssignment repairAssignment, long targetBytesSoFar, long bytesToBeAdded) - { - if (targetBytesSoFar + bytesToBeAdded < maxBytesPerScheduleBytes) - { - return true; - } - logger.warn("Refusing to add {} with a target size of {} because it would increase total repair bytes to {} which is greater than {}={}", - repairAssignment, FileUtils.stringifyFileSize(bytesToBeAdded), FileUtils.stringifyFileSize(targetBytesSoFar + bytesToBeAdded), MAX_BYTES_PER_SCHEDULE, maxBytesPerSchedule); - return false; - } - - public Collection> getTokenRanges(boolean primaryRangeOnly, String keyspaceName) - { - // Collect all applicable token ranges - Collection> wrappedRanges; - if (primaryRangeOnly) - { - wrappedRanges = StorageService.instance.getPrimaryRanges(keyspaceName); - } - else - { - wrappedRanges = StorageService.instance.getLocalRanges(keyspaceName); - } - - // Unwrap each range as we need to account for ranges that overlap the ring - Collection> ranges = new ArrayList<>(); - for (Range wrappedRange : wrappedRanges) - { - ranges.addAll(wrappedRange.unwrap()); - } - - return ranges; - } - - public Refs getSSTableReaderRefs(String keyspaceName, String tableName, Range tokenRange) - { - final ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); - - if (cfs == null) - { - throw new IllegalArgumentException(String.format("Could not resolve ColumnFamilyStore from %s.%s", keyspaceName, tableName)); - } - - Iterable sstables = cfs.getTracker().getView().select(SSTableSet.CANONICAL); - SSTableIntervalTree tree = SSTableIntervalTree.build(sstables); - Range r = Range.makeRowRange(tokenRange); - Iterable canonicalSSTables = View.sstablesInBounds(r.left, r.right, tree); - - // TODO: may need to reason about this not working. - return Refs.ref(canonicalSSTables); - } - - public static class BytesBasedRepairAssignment extends RepairAssignment - { - private final long approximateBytes; - - public BytesBasedRepairAssignment(Range tokenRange, String keyspaceName, List tableNames, long approximateBytes) - { - super(tokenRange, keyspaceName, tableNames); - this.approximateBytes = approximateBytes; - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - BytesBasedRepairAssignment that = (BytesBasedRepairAssignment) o; - return approximateBytes == that.approximateBytes; - } - - @Override - public int hashCode() - { - return Objects.hash(super.hashCode(), approximateBytes); - } - - @Override - public String toString() - { - return "BytesBasedRepairAssignment{" + - "keyspaceName='" + keyspaceName + '\'' + - ", approximateBytes=" + approximateBytes + - ", tokenRange=" + tokenRange + - ", tableNames=" + tableNames + - '}'; - } - - public long getApproximateBytes() - { - return approximateBytes; - } - } -} From 98af2863d2aec21a74623525b3851f70312ff1e1 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sat, 30 Nov 2024 15:10:02 -0600 Subject: [PATCH 072/199] Refactor RepairRangeSplitter interface to accept all keyspaces This enables us to evaluate all RepairAssignments in the schedule at once which enhances the ability to order and restrict the RepairAssignments returned based on the splitter implementation. For example, RepairRangeSplitter's max_bytes_per_schedule would be more useful if it was evaluated for all keyspaces collectively instead of per keyspace. Also refactors AutoRepair to order keyspaces by highest table priority. This may be enhanced later to bucket repair submissions by priority instead of ordering keyspaces by highest table priority. --- .../repair/autorepair/AutoRepair.java | 54 ++++++- .../DefaultAutoRepairTokenSplitter.java | 15 +- .../IAutoRepairTokenRangeSplitter.java | 6 +- .../autorepair/RepairRangeSplitter.java | 150 ++++++++++-------- ...DefaultTokenSplitterParameterizedTest.java | 67 ++++---- .../AutoRepairParameterizedTest.java | 15 +- .../autorepair/RepairRangeSplitterTest.java | 12 +- 7 files changed, 210 insertions(+), 109 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 1f20e8af00a9..429e489e0d5b 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -22,6 +22,7 @@ import java.util.EnumMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -147,6 +148,37 @@ public void repairAsync(AutoRepairConfig.RepairType repairType) repairExecutors.get(repairType).submit(() -> repair(repairType)); } + /** + * @return The priority of the given table if defined, otherwise 0. + */ + private int getPriority(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName) + { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); + return cfs != null ? cfs.metadata().params.automatedRepair.get(repairType).priority() : 0; + } + + /** + * @return A new map of keyspaces to tables sorted by the keyspace's underlying highest table priority. + */ + private Map> sortKeyspaceMapByTablePriority(AutoRepairConfig.RepairType repairType, Map> keyspacesAndTablesToRepair) + { + // Sort assignments by keyspace with tables with the highest priority. + List>> entriesOrderedByTablePriority = new ArrayList<>(keyspacesAndTablesToRepair.entrySet()); + entriesOrderedByTablePriority.sort((a, b) -> { + int aMaxPriority = a.getValue().stream().mapToInt((tableName) -> getPriority(repairType, a.getKey(), tableName)).max().orElse(0); + int bMaxPriority = b.getValue().stream().mapToInt((tableName) -> getPriority(repairType, b.getKey(), tableName)).max().orElse(0); + return bMaxPriority - aMaxPriority; + }); + + Map> sortedKeyspacesToRepair = new LinkedHashMap<>(); + for (Map.Entry> sortedEntries : entriesOrderedByTablePriority) + { + sortedKeyspacesToRepair.put(sortedEntries.getKey(), sortedEntries.getValue()); + } + + return sortedKeyspacesToRepair; + } + // repair runs a repair session of the given type synchronously. public void repair(AutoRepairConfig.RepairType repairType) { @@ -205,18 +237,30 @@ public void repair(AutoRepairConfig.RepairType repairType) // sessions on overlapping datasets at the same time. Shuffling keyspaces reduces the likelihood of this happening. shuffleFunc.accept(keyspaces); + // Filter out keyspaces and tables to repair and group into a map by keyspace. + Map> keyspacesAndTablesToRepair = new LinkedHashMap<>(); for (Keyspace keyspace : keyspaces) { if (!AutoRepairUtils.checkNodeContainsKeyspaceReplica(keyspace)) { continue; } - - repairState.setRepairKeyspaceCount(repairState.getRepairKeyspaceCount() + 1); List tablesToBeRepairedList = retrieveTablesToBeRepaired(keyspace, config, repairType, repairState, collectectedRepairStats); shuffleFunc.accept(tablesToBeRepairedList); - String keyspaceName = keyspace.getName(); - List repairAssignments = tokenRangeSplitters.get(repairType).getRepairAssignments(repairType, primaryRangeOnly, keyspaceName, tablesToBeRepairedList); + keyspacesAndTablesToRepair.put(keyspace.getName(), tablesToBeRepairedList); + } + // sort keyspaces to repair by table priority + Map> sortedKeyspacesToRepair = sortKeyspaceMapByTablePriority(repairType, keyspacesAndTablesToRepair); + + // calculate the repair assignments for each kesypace. + Map> repairAssignmentsByKeyspace = tokenRangeSplitters.get(repairType).getRepairAssignments(repairType, primaryRangeOnly, sortedKeyspacesToRepair); + + // evaluate over each keyspace's repair assignments. + for (Map.Entry> keyspaceAssignments : repairAssignmentsByKeyspace.entrySet()) + { + String keyspaceName = keyspaceAssignments.getKey(); + List repairAssignments = keyspaceAssignments.getValue(); + repairState.setRepairKeyspaceCount(repairState.getRepairKeyspaceCount() + 1); int totalRepairAssignments = repairAssignments.size(); long keyspaceStartTime = timeFunc.get(); @@ -243,7 +287,7 @@ public void repair(AutoRepairConfig.RepairType repairType) repairState.setRepairInProgress(false); return; } - if (AutoRepairUtils.keyspaceMaxRepairTimeExceeded(repairType, keyspaceStartTime, tablesToBeRepairedList.size())) + if (AutoRepairUtils.keyspaceMaxRepairTimeExceeded(repairType, keyspaceStartTime, sortedKeyspacesToRepair.get(keyspaceName).size())) { collectectedRepairStats.skippedTokenRanges += totalRepairAssignments - totalProcessedAssignments; logger.info("Keyspace took too much time to repair hence skipping it {}", diff --git a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java index d34c02c2c710..5b85137c7493 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java @@ -21,7 +21,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import org.apache.cassandra.service.AutoRepairService; @@ -34,7 +36,18 @@ public class DefaultAutoRepairTokenSplitter implements IAutoRepairTokenRangeSplitter { @Override - public List getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames) + public Map> getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, Map> keyspacesAndTablesToRepair) + { + Map> repairAssignmentsByKeyspace = new LinkedHashMap<>(); + for (Map.Entry> keyspace : keyspacesAndTablesToRepair.entrySet()) + { + repairAssignmentsByKeyspace.put(keyspace.getKey(), getRepairAssignmentsByKeyspace(repairType, primaryRangeOnly, keyspace.getKey(), keyspace.getValue())); + } + + return repairAssignmentsByKeyspace; + } + + private List getRepairAssignmentsByKeyspace(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames) { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); List repairAssignments = new ArrayList<>(); diff --git a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java index fe7fb2519891..db0c86d7b73f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java @@ -28,7 +28,6 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.schema.AutoRepairParams; public interface IAutoRepairTokenRangeSplitter { @@ -38,11 +37,10 @@ public interface IAutoRepairTokenRangeSplitter * The autorepair framework will repair the list of returned subrange in a sequence. * @param repairType The type of repair being executed * @param primaryRangeOnly Whether to repair only this node's primary ranges or all of its ranges. - * @param keyspaceName The keyspace being repaired - * @param tableNames The tables to repair + * @param keyspacesAndTablesToRepair A map keyed by keyspace name and valued by tables to generate assignments for. * @return repair assignments broken up by range, keyspace and tables. */ - List getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames); + Map> getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, Map> keyspacesAndTablesToRepair); /** * Reorders the list of {@link RepairAssignment} objects based on their priority for a given repair type. diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 5f97cbc0ef59..b9afd4c7ddc3 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -52,6 +53,7 @@ import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.concurrent.Refs; @@ -154,28 +156,40 @@ public RepairRangeSplitter(Map parameters) } @Override - public List getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames) + public Map> getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, Map> keyspacesAndTablesToRepair) { - logger.debug("Calculating token range splits for repairType={} primaryRangeOnly={} keyspaceName={} tableNames={}", repairType, primaryRangeOnly, keyspaceName, tableNames); - Collection> tokenRanges = getTokenRanges(primaryRangeOnly, keyspaceName); - return getRepairAssignments(repairType, keyspaceName, tableNames, tokenRanges); + Map> repairAssignmentsByKeyspace = new LinkedHashMap<>(); + for (Map.Entry> keyspace : keyspacesAndTablesToRepair.entrySet()) + { + logger.debug("Calculating token range splits for repairType={} primaryRangeOnly={} keyspaceName={} tableNames={}", repairType, primaryRangeOnly, keyspace.getKey(), keyspace.getValue()); + Collection> tokenRanges = getTokenRanges(primaryRangeOnly, keyspace.getKey()); + repairAssignmentsByKeyspace.put(keyspace.getKey(), getRepairAssignments(repairType, keyspace.getKey(), keyspace.getValue(), tokenRanges)); + } + + return filterAndOrderRepairAssignments(repairType, repairAssignmentsByKeyspace); } @VisibleForTesting - List getRepairAssignments(AutoRepairConfig.RepairType repairType, String keyspaceName, List tableNames, Collection> tokenRanges) + List getRepairAssignments(AutoRepairConfig.RepairType repairType, String keyspaceName, List tableNames, Collection> tokenRanges) { List repairAssignments = new ArrayList<>(); // this is used for batching minimal single assignment tables together List currentAssignments = new ArrayList<>(); - // sort the tables by size so can batch the smallest ones together - tableNames.sort((t1, t2) -> { - ColumnFamilyStore cfs1 = ColumnFamilyStore.getIfExists(keyspaceName, t1); - ColumnFamilyStore cfs2 = ColumnFamilyStore.getIfExists(keyspaceName, t2); - if (cfs1 == null || cfs2 == null) - throw new IllegalArgumentException(String.format("Could not resolve ColumnFamilyStore from %s.%s", keyspaceName, t1)); - return Long.compare(cfs1.metric.totalDiskSpaceUsed.getCount(), cfs2.metric.totalDiskSpaceUsed.getCount()); - }); + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + + // If we can repair by keyspace, sort the tables by size so can batch the smallest ones together + boolean repairByKeyspace = config.getRepairByKeyspace(repairType); + if (repairByKeyspace) + { + tableNames.sort((t1, t2) -> { + ColumnFamilyStore cfs1 = ColumnFamilyStore.getIfExists(keyspaceName, t1); + ColumnFamilyStore cfs2 = ColumnFamilyStore.getIfExists(keyspaceName, t2); + if (cfs1 == null || cfs2 == null) + throw new IllegalArgumentException(String.format("Could not resolve ColumnFamilyStore from %s.%s", keyspaceName, t1)); + return Long.compare(cfs1.metric.totalDiskSpaceUsed.getCount(), cfs2.metric.totalDiskSpaceUsed.getCount()); + }); + } for (String tableName : tableNames) { @@ -184,8 +198,13 @@ List getRepairAssignments(AutoRepairConfig.RepairType repairTy if (tableAssignments.isEmpty()) continue; + // if not repairing by keyspace don't attempt to batch them with others. + if (!repairByKeyspace) + { + repairAssignments.addAll(tableAssignments); + } // If the table assignments are for the same token range, and we have room to add more tables to the current assignment - if (tableAssignments.size() == 1 && + else if (tableAssignments.size() == 1 && currentAssignments.size() < tablesPerAssignmentLimit && (currentAssignments.isEmpty() || currentAssignments.get(0).getTokenRange().equals(tableAssignments.get(0).getTokenRange()))) { @@ -215,77 +234,92 @@ List getRepairAssignments(AutoRepairConfig.RepairType repairTy if (!currentAssignments.isEmpty()) repairAssignments.add(merge(currentAssignments)); - return filterRepairAssignments(repairType, keyspaceName, repairAssignments); + return repairAssignments; } /** - * Given a repair type and list of sized-based repair assignments order them by priority of the + * Given a repair type and map of sized-based repair assignments by keyspace order them by priority of the * assignments' underlying tables and confine them by maxBytesPerSchedule. * @param repairType used to determine underyling table priorities - * @param repairAssignments the assignments to filter. + * @param repairAssignmentsByKeyspace the assignments to filter. * @return A list of repair assignments ordered by priority and confined by maxBytesPerSchedule. */ - protected List filterRepairAssignments(AutoRepairConfig.RepairType repairType, String keyspaceName, List repairAssignments) + @VisibleForTesting + Map> filterAndOrderRepairAssignments(AutoRepairConfig.RepairType repairType, Map> repairAssignmentsByKeyspace) { - if (repairAssignments.isEmpty()) - return Collections.emptyList(); - - // Reorder the repair assignments - reorderByPriority(repairAssignments, repairType); - // Confine repair assignments by maxBytesPerSchedule. long bytesSoFar = 0L; - boolean isIncremental = false; long bytesNotRepaired = 0L; int assignmentsNotRepaired = 0; - List assignmentsToReturn = new ArrayList<>(repairAssignments.size()); - for (SizedRepairAssignment repairAssignment : repairAssignments) + int assignmentsToRepair = 0; + int totalAssignments = 0; + + Map> filteredRepairAssignmentsByKeyspace = new LinkedHashMap<>(); + for (Map.Entry> keyspaceAssignments : repairAssignmentsByKeyspace.entrySet()) { - // skip any repair assignments that would accumulate us past the maxBytesPerSchedule - if (bytesSoFar + repairAssignment.getEstimatedBytes() > maxBytesPerSchedule) + String keyspace = keyspaceAssignments.getKey(); + List repairAssignments = keyspaceAssignments.getValue(); + + // Reorder the repair assignments + reorderByPriority(repairAssignments, repairType); + + List assignmentsToReturn = new ArrayList<>(repairAssignments.size()); + for (SizedRepairAssignment repairAssignment : repairAssignments) { - // log that repair assignment was skipped. - bytesNotRepaired += repairAssignment.getEstimatedBytes(); - assignmentsNotRepaired++; - logger.warn("Skipping {} because it would increase total repair bytes to {}", - repairAssignment, - getBytesOfMaxBytesPerSchedule(bytesSoFar + repairAssignment.getEstimatedBytes())); + totalAssignments++; + // skip any repair assignments that would accumulate us past the maxBytesPerSchedule + if (bytesSoFar + repairAssignment.getEstimatedBytes() > maxBytesPerSchedule) + { + // log that repair assignment was skipped. + bytesNotRepaired += repairAssignment.getEstimatedBytes(); + assignmentsNotRepaired++; + logger.warn("Skipping {} because it would increase total repair bytes to {}", + repairAssignment, + getBytesOfMaxBytesPerSchedule(bytesSoFar + repairAssignment.getEstimatedBytes())); + } + else + { + bytesSoFar += repairAssignment.getEstimatedBytes(); + assignmentsToRepair++; + logger.info("Adding {}, increasing repair bytes to {}", + repairAssignment, + getBytesOfMaxBytesPerSchedule(bytesSoFar)); + assignmentsToReturn.add(repairAssignment); + } } - else + + if (!assignmentsToReturn.isEmpty()) { - bytesSoFar += repairAssignment.getEstimatedBytes(); - logger.info("Adding {}, increasing repair bytes to {}", - repairAssignment, - getBytesOfMaxBytesPerSchedule(bytesSoFar)); - assignmentsToReturn.add(repairAssignment); + filteredRepairAssignmentsByKeyspace.put(keyspace, assignmentsToReturn); } } - String message = "Returning {} assignment(s) for {}, totaling {}"; + String message = "Returning {} assignment(s), totaling {}"; if (assignmentsNotRepaired != 0) { message += ". Skipping {} of {} assignment(s), totaling {}"; if (repairType != AutoRepairConfig.RepairType.INCREMENTAL) { message += ". The entire primary range will not be repaired this schedule. " + - "Consider increasing maxBytesPerSchedule, reducing node density or monitoring to ensure " + - "all ranges do get repaired within gc_grace_seconds"; - logger.warn(message, assignmentsToReturn.size(), keyspaceName, getBytesOfMaxBytesPerSchedule(bytesSoFar), - assignmentsNotRepaired, repairAssignments.size(), + "Consider increasing maxBytesPerSchedule, reducing node density or monitoring to ensure " + + "all ranges do get repaired within gc_grace_seconds"; + logger.warn(message, assignmentsToRepair, getBytesOfMaxBytesPerSchedule(bytesSoFar), + assignmentsNotRepaired, totalAssignments, FileUtils.stringifyFileSize(bytesNotRepaired)); } else { - logger.info(message, assignmentsToReturn.size(), keyspaceName, getBytesOfMaxBytesPerSchedule(bytesSoFar), - assignmentsNotRepaired, repairAssignments.size(), + logger.info(message, assignmentsToRepair, getBytesOfMaxBytesPerSchedule(bytesSoFar), + assignmentsNotRepaired, totalAssignments, FileUtils.stringifyFileSize(bytesNotRepaired)); } } else { - logger.info(message, assignmentsToReturn.size(), keyspaceName, getBytesOfMaxBytesPerSchedule(bytesSoFar)); + logger.info(message, assignmentsToRepair, getBytesOfMaxBytesPerSchedule(bytesSoFar)); } - return assignmentsToReturn; + + return filteredRepairAssignmentsByKeyspace; } private String getBytesOfMaxBytesPerSchedule(long bytes) { @@ -397,22 +431,6 @@ protected List getRepairAssignments(List si return repairAssignments; } - private void warnMaxBytesPerSchedule(AutoRepairConfig.RepairType repairType, SizedRepairAssignment repairAssignment) - { - String warning = "Refusing to add repair assignment {} because it would increase total repair bytes to a value greater than {} ({})"; - if (repairType == AutoRepairConfig.RepairType.FULL) - { - warning = warning + ", everything will not be repaired this schedule. Consider increasing maxBytesPerSchedule, reducing node density or monitoring to ensure all ranges do get repaired within gc_grace_seconds"; - } - // TODO: Add two metrics: - // Meter: RepairRangeSplitter.RepairType.SkippedAssignments - // Meter: RepairRangeSplitter.RepairType.SkippedBytes - logger.warn(warning, - repairAssignment, - FileUtils.stringifyFileSize(maxBytesPerSchedule), - FileUtils.stringifyFileSize(maxBytesPerSchedule + repairAssignment.getEstimatedBytes())); - } - private int calculateNumberOfSplits(SizeEstimate estimate) { // Calculate the number of splits needed for size and partitions diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java index b460a750770f..c7a2ff3efa1f 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java @@ -22,7 +22,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import org.junit.BeforeClass; import org.junit.Test; @@ -43,6 +45,7 @@ import static org.apache.cassandra.Util.setAutoRepairEnabled; import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; @RunWith(Parameterized.class) public class AutoRepairDefaultTokenSplitterParameterizedTest extends CQLTester @@ -67,32 +70,13 @@ public static void setupClass() throws Exception AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); setAutoRepairEnabled(true); - DatabaseDescriptor.setInitialTokens("0,256,1024"); + DatabaseDescriptor.setInitialTokens("-9223372036854775808,-3074457345618258603,3074457345618258602"); requireNetwork(); AutoRepairUtils.setup(); SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1)); } - private static void appendExpectedTokens(long left, long right, int numberOfSplits, List> expectedToken) - { - long repairTokenWidth = (right - left) / numberOfSplits; - for (int i = 0; i < numberOfSplits; i++) - { - long curLeft = left + (i * repairTokenWidth); - long curRight = curLeft + repairTokenWidth; - if ((i + 1) == numberOfSplits) - { - curRight = right; - } - Token childStartToken = StorageService.instance.getTokenMetadata() - .partitioner.getTokenFactory().fromString("" + curLeft); - Token childEndToken = StorageService.instance.getTokenMetadata() - .partitioner.getTokenFactory().fromString("" + curRight); - expectedToken.add(new Range<>(childStartToken, childEndToken)); - } - } - @Test public void testTokenRangesSplitByTable() { @@ -105,19 +89,37 @@ public void testTokenRangesSplitByTable() List> expectedToken = new ArrayList<>(); for (int i = 0; i range : tokens) + { + expectedToken.addAll(AutoRepairUtils.split(range, numberOfSplits)); + } } AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setRepairSubRangeNum(repairType, numberOfSplits); - List assignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, KEYSPACE, tables); + Map> keyspaceToTables = new LinkedHashMap<>(); + keyspaceToTables.put(KEYSPACE, tables); + Map> assignmentsByKeyspace = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, keyspaceToTables); + + // should be 1 entry for the keyspace. + assertEquals(1, assignmentsByKeyspace.size()); + List assignments = assignmentsByKeyspace.get(KEYSPACE); + assertNotNull(assignments); + assertEquals(totalTokenRanges*numberOfSplits*tables.size(), assignments.size()); assertEquals(expectedToken.size(), assignments.size()); int expectedTableIndex = -1; for (int i = 0; i tables = Arrays.asList(TABLE1, TABLE2, TABLE3); List> expectedToken = new ArrayList<>(); - appendExpectedTokens(1024, 0, numberOfSplits, expectedToken); - appendExpectedTokens(0, 256, numberOfSplits, expectedToken); - appendExpectedTokens(256, 1024, numberOfSplits, expectedToken); + for (Range range : tokens) + { + expectedToken.addAll(AutoRepairUtils.split(range, numberOfSplits)); + } AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setRepairSubRangeNum(repairType, numberOfSplits); - List assignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, KEYSPACE, tables); + Map> keyspaceToTables = new LinkedHashMap<>(); + keyspaceToTables.put(KEYSPACE, tables); + Map> assignmentsByKeyspace = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, keyspaceToTables); + + // should be 1 entry for the keyspace. + assertEquals(1, assignmentsByKeyspace.size()); + List assignments = assignmentsByKeyspace.get(KEYSPACE); + assertNotNull(assignments); + assertEquals(totalTokenRanges*numberOfSplits, assignments.size()); assertEquals(expectedToken.size(), assignments.size()); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 3dd25beb0b44..b63cf6fb134b 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -23,7 +23,9 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -81,6 +83,7 @@ import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.NOT_MY_TURN; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; @@ -480,6 +483,7 @@ public void testMetrics() AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setMVRepairEnabled(repairType, true); config.setRepairMinInterval(repairType, "0s"); + config.setRepairRetryBackoff("0s"); config.setAutoRepairTableMaxRepairTime(repairType, "0s"); AutoRepair.timeFunc = () -> { timeFuncCalls++; @@ -568,7 +572,16 @@ public void testTokenRangesNoSplit() assertEquals(1, tokens.size()); List> expectedToken = new ArrayList<>(tokens); - List assignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, KEYSPACE, Collections.singletonList(TABLE)); + Map> keyspaceToTables = new LinkedHashMap<>(); + keyspaceToTables.put(KEYSPACE, Collections.singletonList(TABLE)); + Map> assignmentsByKeyspace = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, keyspaceToTables); + + // should be 1 entry for the keyspace. + assertEquals(1, assignmentsByKeyspace.size()); + List assignments = assignmentsByKeyspace.get(KEYSPACE); + assertNotNull(assignments); + + // should be 1 entry for the table which covers the full range. assertEquals(1, assignments.size()); assertEquals(expectedToken.get(0).left, assignments.get(0).getTokenRange().left); assertEquals(expectedToken.get(0).right, assignments.get(0).getTokenRange().right); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java index 6c91fbd539be..0f18d82eec3f 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java @@ -40,6 +40,7 @@ import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter.RepairAssignment; import org.apache.cassandra.repair.autorepair.RepairRangeSplitter.SizedRepairAssignment; +import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.utils.concurrent.Refs; import static org.apache.cassandra.repair.autorepair.RepairRangeSplitter.TABLE_BATCH_LIMIT; @@ -56,6 +57,9 @@ public class RepairRangeSplitterTest extends CQLTester public static void setUpClass() { CQLTester.setUpClass(); + AutoRepairService.setup(); + // TODO: For now, always repair by keyspace to exercise priority tests + AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(AutoRepairConfig.RepairType.FULL, true); FULL_RANGE = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken()); } @@ -205,7 +209,7 @@ public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable { Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); - List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); // We expect two assignments, one with table1 and table2 batched, and one with table3 assertEquals(2, assignments.size()); @@ -219,7 +223,7 @@ public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable { Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(2); - List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); // We expect one assignment, with two tables batched assertEquals(1, assignments.size()); @@ -232,7 +236,7 @@ public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable { Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); - List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); assertEquals(3, assignments.size()); } @@ -243,7 +247,7 @@ public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable { Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(5); - List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); assertEquals(1, assignments.size()); } From a0165e56747bdf9990fc1622ab1f23f86b5acb2b Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sat, 30 Nov 2024 15:17:46 -0600 Subject: [PATCH 073/199] Naming consistency getRepairAssignmentsByKeyspace -> getRepairAssignmentsForKeyspace getRepairAssignments -> getRepairAssignmentsForKeyspace --- .../repair/autorepair/DefaultAutoRepairTokenSplitter.java | 4 ++-- .../cassandra/repair/autorepair/RepairRangeSplitter.java | 4 ++-- .../repair/autorepair/RepairRangeSplitterTest.java | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java index 5b85137c7493..c441294d32df 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java @@ -41,13 +41,13 @@ public Map> getRepairAssignments(AutoRepairConfig Map> repairAssignmentsByKeyspace = new LinkedHashMap<>(); for (Map.Entry> keyspace : keyspacesAndTablesToRepair.entrySet()) { - repairAssignmentsByKeyspace.put(keyspace.getKey(), getRepairAssignmentsByKeyspace(repairType, primaryRangeOnly, keyspace.getKey(), keyspace.getValue())); + repairAssignmentsByKeyspace.put(keyspace.getKey(), getRepairAssignmentsForKeyspace(repairType, primaryRangeOnly, keyspace.getKey(), keyspace.getValue())); } return repairAssignmentsByKeyspace; } - private List getRepairAssignmentsByKeyspace(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames) + private List getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames) { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); List repairAssignments = new ArrayList<>(); diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index b9afd4c7ddc3..0885ffe4cb2f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -163,14 +163,14 @@ public Map> getRepairAssignments(AutoRepairConfig { logger.debug("Calculating token range splits for repairType={} primaryRangeOnly={} keyspaceName={} tableNames={}", repairType, primaryRangeOnly, keyspace.getKey(), keyspace.getValue()); Collection> tokenRanges = getTokenRanges(primaryRangeOnly, keyspace.getKey()); - repairAssignmentsByKeyspace.put(keyspace.getKey(), getRepairAssignments(repairType, keyspace.getKey(), keyspace.getValue(), tokenRanges)); + repairAssignmentsByKeyspace.put(keyspace.getKey(), getRepairAssignmentsForKeyspace(repairType, keyspace.getKey(), keyspace.getValue(), tokenRanges)); } return filterAndOrderRepairAssignments(repairType, repairAssignmentsByKeyspace); } @VisibleForTesting - List getRepairAssignments(AutoRepairConfig.RepairType repairType, String keyspaceName, List tableNames, Collection> tokenRanges) + List getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType repairType, String keyspaceName, List tableNames, Collection> tokenRanges) { List repairAssignments = new ArrayList<>(); // this is used for batching minimal single assignment tables together diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java index 0f18d82eec3f..606737817ac4 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java @@ -209,7 +209,7 @@ public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable { Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); - List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); // We expect two assignments, one with table1 and table2 batched, and one with table3 assertEquals(2, assignments.size()); @@ -223,7 +223,7 @@ public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable { Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(2); - List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); // We expect one assignment, with two tables batched assertEquals(1, assignments.size()); @@ -236,7 +236,7 @@ public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable { Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); - List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); assertEquals(3, assignments.size()); } @@ -247,7 +247,7 @@ public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable { Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(5); - List assignments = repairRangeSplitter.getRepairAssignments(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); assertEquals(1, assignments.size()); } From f1a97ffa2d0141981d9f346daddac640e79fac77 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sat, 30 Nov 2024 15:47:17 -0600 Subject: [PATCH 074/199] Use memtable size for memtable only --- .../apache/cassandra/repair/autorepair/RepairRangeSplitter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 0885ffe4cb2f..ef3d1cc44fed 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -393,7 +393,7 @@ protected List getRepairAssignments(List si if (memtableSize > 0L) { logger.debug("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", estimate.keyspace, estimate.table, estimate.tokenRange, memtableSize); - SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "memtable only", estimate.sizeInRange); + SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "memtable only", memtableSize); repairAssignments.add(assignment); } else From 4d3ca9c9d66f6d063dc13654ebe5b5d89f102ab1 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 1 Dec 2024 18:48:43 -0600 Subject: [PATCH 075/199] Fix dtest --- .../distributed/test/repair/AutoRepairSchedulerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index 7fdf3a42c75a..6cf55299abc9 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -60,14 +60,14 @@ public static void init() throws IOException .set("auto_repair", ImmutableMap.of( "repair_type_overrides", - ImmutableMap.of(AutoRepairConfig.RepairType.FULL.toString(), + ImmutableMap.of(AutoRepairConfig.RepairType.FULL.getConfigName(), ImmutableMap.of( "initial_scheduler_delay", "5s", "enabled", "true", "parallel_repair_count", "1", "parallel_repair_percentage", "0", "min_repair_interval", "1s"), - AutoRepairConfig.RepairType.INCREMENTAL.toString(), + AutoRepairConfig.RepairType.INCREMENTAL.getConfigName(), ImmutableMap.of( "initial_scheduler_delay", "5s", "enabled", "true", From 98b2a0fa65ed46c4ec5058e58518989347ca8bcb Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 1 Dec 2024 19:37:52 -0600 Subject: [PATCH 076/199] Refactor IAutoRepairTokenRangeSplitter to be Iterator based Refactors IAutoRepairTokenRangeSplitter interface to be iterator based. This makes generation of RepairAssignments for priority buckets and keyspaces lazy instead of generating the RepairAssignments all at once. The main value here is that repairs can take some time, and it is beneficial to calculate repair assignments as close to when repairs would be submitted for a keyspace as possible. Additionally, introduced "PrioritizedRepairPlan" and "KeyspaceRepairPlan" types. Rather than passing around multiple-depth Maps where the contents aren't clear, create specialized types. Introduces PrioritizedRepairPlan.build for organizing a desired repair plan by grouping tables together that share the same priority. Moves RepairAssignment up to its own class to simplify the IAutoRepairTokenRangeSplitter. Since grouping tables by priority is now done in AutoRepair, the splitter implementations no longer have to reason with priority so all of this logic has been removed from the splitter code. Instead, they will now be given a List that is ordered by priority. --- .../repair/autorepair/AutoRepair.java | 302 +++++++++--------- .../DefaultAutoRepairTokenSplitter.java | 69 +++- .../IAutoRepairTokenRangeSplitter.java | 103 +----- .../autorepair/KeyspaceRepairAssignments.java | 53 +++ .../repair/autorepair/KeyspaceRepairPlan.java | 71 ++++ .../autorepair/PrioritizedRepairPlan.java | 160 ++++++++++ .../repair/autorepair/RepairAssignment.java | 85 +++++ .../autorepair/RepairRangeSplitter.java | 160 +++++++--- ...DefaultTokenSplitterParameterizedTest.java | 40 ++- .../AutoRepairParameterizedTest.java | 20 +- .../autorepair/PrioritizedRepairPlanTest.java | 125 ++++++++ .../autorepair/RepairRangeSplitterTest.java | 142 ++------ 12 files changed, 882 insertions(+), 448 deletions(-) create mode 100644 src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairAssignments.java create mode 100644 src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairPlan.java create mode 100644 src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java create mode 100644 src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 429e489e0d5b..3b35d21c5e93 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -40,7 +40,6 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.utils.Clock; -import org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter.RepairAssignment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -157,28 +156,6 @@ private int getPriority(AutoRepairConfig.RepairType repairType, String keyspaceN return cfs != null ? cfs.metadata().params.automatedRepair.get(repairType).priority() : 0; } - /** - * @return A new map of keyspaces to tables sorted by the keyspace's underlying highest table priority. - */ - private Map> sortKeyspaceMapByTablePriority(AutoRepairConfig.RepairType repairType, Map> keyspacesAndTablesToRepair) - { - // Sort assignments by keyspace with tables with the highest priority. - List>> entriesOrderedByTablePriority = new ArrayList<>(keyspacesAndTablesToRepair.entrySet()); - entriesOrderedByTablePriority.sort((a, b) -> { - int aMaxPriority = a.getValue().stream().mapToInt((tableName) -> getPriority(repairType, a.getKey(), tableName)).max().orElse(0); - int bMaxPriority = b.getValue().stream().mapToInt((tableName) -> getPriority(repairType, b.getKey(), tableName)).max().orElse(0); - return bMaxPriority - aMaxPriority; - }); - - Map> sortedKeyspacesToRepair = new LinkedHashMap<>(); - for (Map.Entry> sortedEntries : entriesOrderedByTablePriority) - { - sortedKeyspacesToRepair.put(sortedEntries.getKey(), sortedEntries.getValue()); - } - - return sortedKeyspacesToRepair; - } - // repair runs a repair session of the given type synchronously. public void repair(AutoRepairConfig.RepairType repairType) { @@ -229,7 +206,7 @@ public void repair(AutoRepairConfig.RepairType repairType) repairState.setTotalTablesConsideredForRepair(0); repairState.setTotalMVTablesConsideredForRepair(0); - CollectectedRepairStats collectectedRepairStats = new CollectectedRepairStats(); + CollectedRepairStats collectedRepairStats = new CollectedRepairStats(); List keyspaces = new ArrayList<>(); Keyspace.all().forEach(keyspaces::add); @@ -245,136 +222,32 @@ public void repair(AutoRepairConfig.RepairType repairType) { continue; } - List tablesToBeRepairedList = retrieveTablesToBeRepaired(keyspace, config, repairType, repairState, collectectedRepairStats); + List tablesToBeRepairedList = retrieveTablesToBeRepaired(keyspace, config, repairType, repairState, collectedRepairStats); shuffleFunc.accept(tablesToBeRepairedList); keyspacesAndTablesToRepair.put(keyspace.getName(), tablesToBeRepairedList); } - // sort keyspaces to repair by table priority - Map> sortedKeyspacesToRepair = sortKeyspaceMapByTablePriority(repairType, keyspacesAndTablesToRepair); - // calculate the repair assignments for each kesypace. - Map> repairAssignmentsByKeyspace = tokenRangeSplitters.get(repairType).getRepairAssignments(repairType, primaryRangeOnly, sortedKeyspacesToRepair); + // Separate out the keyspaces and tables to repair based on their priority, with each repair plan representing a uniquely occuring priority. + List repairPlans = PrioritizedRepairPlan.build(keyspacesAndTablesToRepair, repairType, shuffleFunc); - // evaluate over each keyspace's repair assignments. - for (Map.Entry> keyspaceAssignments : repairAssignmentsByKeyspace.entrySet()) + // calculate the repair assignments for each priority:keyspace. + Iterator repairAssignmentsIterator = tokenRangeSplitters.get(repairType).getRepairAssignments(repairType, primaryRangeOnly, repairPlans); + + while (repairAssignmentsIterator.hasNext()) { - String keyspaceName = keyspaceAssignments.getKey(); - List repairAssignments = keyspaceAssignments.getValue(); - repairState.setRepairKeyspaceCount(repairState.getRepairKeyspaceCount() + 1); - - int totalRepairAssignments = repairAssignments.size(); - long keyspaceStartTime = timeFunc.get(); - RepairAssignment previousAssignment = null; - long tableStartTime = timeFunc.get(); - int totalProcessedAssignments = 0; - Set> ranges = new HashSet<>(); - for (RepairAssignment curRepairAssignment : repairAssignments) + KeyspaceRepairAssignments repairAssignments = repairAssignmentsIterator.next(); + List assignments = repairAssignments.getRepairAssignments(); + if (assignments.isEmpty()) { - try - { - totalProcessedAssignments++; - boolean repairOneTableAtATime = !config.getRepairByKeyspace(repairType); - if (previousAssignment != null && repairOneTableAtATime && !previousAssignment.tableNames.equals(curRepairAssignment.tableNames)) - { - // In the repair assignment, all the tables are appended sequnetially. - // Check if we have a different table, and if so, we should reset the table start time. - tableStartTime = timeFunc.get(); - } - previousAssignment = curRepairAssignment; - if (!config.isAutoRepairEnabled(repairType)) - { - logger.error("Auto-repair for type {} is disabled hence not running repair", repairType); - repairState.setRepairInProgress(false); - return; - } - if (AutoRepairUtils.keyspaceMaxRepairTimeExceeded(repairType, keyspaceStartTime, sortedKeyspacesToRepair.get(keyspaceName).size())) - { - collectectedRepairStats.skippedTokenRanges += totalRepairAssignments - totalProcessedAssignments; - logger.info("Keyspace took too much time to repair hence skipping it {}", - keyspaceName); - break; - } - if (repairOneTableAtATime && AutoRepairUtils.tableMaxRepairTimeExceeded(repairType, tableStartTime)) - { - collectectedRepairStats.skippedTokenRanges += 1; - logger.info("Table took too much time to repair hence skipping it table name {}.{}, token range {}", - keyspaceName, curRepairAssignment.tableNames, curRepairAssignment.tokenRange); - continue; - } - - Range tokenRange = curRepairAssignment.getTokenRange(); - logger.debug("Current Token Left side {}, right side {}", - tokenRange.left.toString(), - tokenRange.right.toString()); - - ranges.add(curRepairAssignment.getTokenRange()); - if ((totalProcessedAssignments % config.getRepairThreads(repairType) == 0) || - (totalProcessedAssignments == totalRepairAssignments)) - { - int retryCount = 0; - Future f = null; - while (retryCount <= config.getRepairMaxRetries()) - { - RepairRunnable task = repairState.getRepairRunnable(keyspaceName, - Lists.newArrayList(curRepairAssignment.getTableNames()), - ranges, primaryRangeOnly); - repairState.resetWaitCondition(); - f = repairRunnableExecutors.get(repairType).submit(task); - try - { - repairState.waitForRepairToComplete(config.getRepairSessionTimeout(repairType)); - } - catch (InterruptedException e) - { - logger.error("Exception in cond await:", e); - } - if (repairState.isSuccess()) - { - break; - } - else if (retryCount < config.getRepairMaxRetries()) - { - boolean cancellationStatus = f.cancel(true); - logger.warn("Repair failed for range {}-{} for {} tables {} with cancellationStatus: {} retrying after {} seconds...", - tokenRange.left, tokenRange.right, - keyspaceName, curRepairAssignment.getTableNames(), - cancellationStatus, config.getRepairRetryBackoff().toSeconds()); - sleepFunc.accept(config.getRepairRetryBackoff().toSeconds(), TimeUnit.SECONDS); - } - retryCount++; - } - //check repair status - if (repairState.isSuccess()) - { - logger.info("Repair completed for range {}-{} for {} tables {}, total assignments: {}," + - "processed assignments: {}", tokenRange.left, tokenRange.right, - keyspaceName, curRepairAssignment.getTableNames(), totalRepairAssignments, totalProcessedAssignments); - collectectedRepairStats.succeededTokenRanges += ranges.size(); - } - else - { - boolean cancellationStatus = true; - if (f != null) - { - cancellationStatus = f.cancel(true); - } - //in the future we can add retry, etc. - logger.error("Repair failed for range {}-{} for {} tables {} after {} retries, total assignments: {}," + - "processed assignments: {}, cancellationStatus: {}", tokenRange.left, tokenRange.right, keyspaceName, - curRepairAssignment.getTableNames(), retryCount, totalRepairAssignments, totalProcessedAssignments, cancellationStatus); - collectectedRepairStats.failedTokenRanges += ranges.size(); - } - ranges.clear(); - } - logger.info("Repair completed for {} tables {}, range {}", keyspaceName, curRepairAssignment.getTableNames(), curRepairAssignment.getTokenRange()); - } - catch (Exception e) - { - logger.error("Exception while repairing keyspace {}:", keyspaceName, e); - } + logger.info("Skipping repairs for priorityBucket={} for keyspace={} since it yielded no assignments", repairAssignments.getPriority(), repairAssignments.getKeyspaceName()); + continue; } + + logger.info("Submitting repairs for priorityBucket={} for keyspace={} with assignmentCount={}", repairAssignments.getPriority(), repairAssignments.getKeyspaceName(), repairAssignments.getRepairAssignments().size()); + repairKeyspace(repairType, primaryRangeOnly, repairAssignments.getKeyspaceName(), repairAssignments.getRepairAssignments(), collectedRepairStats); } - cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, collectectedRepairStats); + + cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, collectedRepairStats); } else { @@ -387,6 +260,127 @@ else if (retryCount < config.getRepairMaxRetries()) } } + private void repairKeyspace(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List repairAssignments, CollectedRepairStats collectedRepairStats) + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + AutoRepairState repairState = repairStates.get(repairType); + + // evaluate over each keyspace's repair assignments. + repairState.setRepairKeyspaceCount(repairState.getRepairKeyspaceCount() + 1); + + int totalRepairAssignments = repairAssignments.size(); + long keyspaceStartTime = timeFunc.get(); + RepairAssignment previousAssignment = null; + long tableStartTime = timeFunc.get(); + int totalProcessedAssignments = 0; + Set> ranges = new HashSet<>(); + for (RepairAssignment curRepairAssignment : repairAssignments) + { + try + { + totalProcessedAssignments++; + boolean repairOneTableAtATime = !config.getRepairByKeyspace(repairType); + if (previousAssignment != null && repairOneTableAtATime && !previousAssignment.tableNames.equals(curRepairAssignment.tableNames)) + { + // In the repair assignment, all the tables are appended sequnetially. + // Check if we have a different table, and if so, we should reset the table start time. + tableStartTime = timeFunc.get(); + } + previousAssignment = curRepairAssignment; + if (!config.isAutoRepairEnabled(repairType)) + { + logger.error("Auto-repair for type {} is disabled hence not running repair", repairType); + repairState.setRepairInProgress(false); + return; + } + if (AutoRepairUtils.keyspaceMaxRepairTimeExceeded(repairType, keyspaceStartTime, repairAssignments.size())) + { + collectedRepairStats.skippedTokenRanges += totalRepairAssignments - totalProcessedAssignments; + logger.info("Keyspace took too much time to repair hence skipping it {}", + keyspaceName); + break; + } + if (repairOneTableAtATime && AutoRepairUtils.tableMaxRepairTimeExceeded(repairType, tableStartTime)) + { + collectedRepairStats.skippedTokenRanges += 1; + logger.info("Table took too much time to repair hence skipping it table name {}.{}, token range {}", + keyspaceName, curRepairAssignment.tableNames, curRepairAssignment.tokenRange); + continue; + } + + Range tokenRange = curRepairAssignment.getTokenRange(); + logger.debug("Current Token Left side {}, right side {}", + tokenRange.left.toString(), + tokenRange.right.toString()); + + ranges.add(curRepairAssignment.getTokenRange()); + if ((totalProcessedAssignments % config.getRepairThreads(repairType) == 0) || + (totalProcessedAssignments == totalRepairAssignments)) + { + int retryCount = 0; + Future f = null; + while (retryCount <= config.getRepairMaxRetries()) + { + RepairRunnable task = repairState.getRepairRunnable(keyspaceName, + Lists.newArrayList(curRepairAssignment.getTableNames()), + ranges, primaryRangeOnly); + repairState.resetWaitCondition(); + f = repairRunnableExecutors.get(repairType).submit(task); + try + { + repairState.waitForRepairToComplete(config.getRepairSessionTimeout(repairType)); + } + catch (InterruptedException e) + { + logger.error("Exception in cond await:", e); + } + if (repairState.isSuccess()) + { + break; + } + else if (retryCount < config.getRepairMaxRetries()) + { + boolean cancellationStatus = f.cancel(true); + logger.warn("Repair failed for range {}-{} for {} tables {} with cancellationStatus: {} retrying after {} seconds...", + tokenRange.left, tokenRange.right, + keyspaceName, curRepairAssignment.getTableNames(), + cancellationStatus, config.getRepairRetryBackoff().toSeconds()); + sleepFunc.accept(config.getRepairRetryBackoff().toSeconds(), TimeUnit.SECONDS); + } + retryCount++; + } + //check repair status + if (repairState.isSuccess()) + { + logger.info("Repair completed for range {}-{} for {} tables {}, total assignments: {}," + + "processed assignments: {}", tokenRange.left, tokenRange.right, + keyspaceName, curRepairAssignment.getTableNames(), totalRepairAssignments, totalProcessedAssignments); + collectedRepairStats.succeededTokenRanges += ranges.size(); + } + else + { + boolean cancellationStatus = true; + if (f != null) + { + cancellationStatus = f.cancel(true); + } + //in the future we can add retry, etc. + logger.error("Repair failed for range {}-{} for {} tables {} after {} retries, total assignments: {}," + + "processed assignments: {}, cancellationStatus: {}", tokenRange.left, tokenRange.right, keyspaceName, + curRepairAssignment.getTableNames(), retryCount, totalRepairAssignments, totalProcessedAssignments, cancellationStatus); + collectedRepairStats.failedTokenRanges += ranges.size(); + } + ranges.clear(); + } + logger.info("Repair completed for {} tables {}, range {}", keyspaceName, curRepairAssignment.getTableNames(), curRepairAssignment.getTokenRange()); + } + catch (Exception e) + { + logger.error("Exception while repairing keyspace {}:", keyspaceName, e); + } + } + } + private boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoRepairState repairState, AutoRepairConfig config, UUID myId) { if (repairState.getLastRepairTime() == 0) @@ -409,7 +403,7 @@ private boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoR return false; } - private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairConfig config, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, CollectectedRepairStats collectectedRepairStats) + private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairConfig config, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, CollectedRepairStats collectedRepairStats) { Tables tables = keyspace.getMetadata().tables; List tablesToBeRepaired = new ArrayList<>(); @@ -425,7 +419,7 @@ private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairCon { logger.info("Repair is disabled for keyspace {} for tables: {}", keyspace.getName(), tableName); repairState.setTotalDisabledTablesRepairCount(repairState.getTotalDisabledTablesRepairCount() + 1); - collectectedRepairStats.skippedTables++; + collectedRepairStats.skippedTables++; continue; } @@ -436,7 +430,7 @@ private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairCon { logger.info("Too many SSTables for repair for table {}.{}" + "totalSSTables {}", keyspace.getName(), tableName, totalSSTables); - collectectedRepairStats.skippedTables++; + collectedRepairStats.skippedTables++; continue; } @@ -454,7 +448,7 @@ private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairCon } private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, UUID myId, - long startTime, CollectectedRepairStats collectectedRepairStats) throws InterruptedException + long startTime, CollectedRepairStats collectedRepairStats) throws InterruptedException { //if it was due to priority then remove it now if (turn == MY_TURN_DUE_TO_PRIORITY) @@ -463,10 +457,10 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType AutoRepairUtils.removePriorityStatus(repairType, myId); } - repairState.setFailedTokenRangesCount(collectectedRepairStats.failedTokenRanges); - repairState.setSucceededTokenRangesCount(collectectedRepairStats.succeededTokenRanges); - repairState.setSkippedTokenRangesCount(collectectedRepairStats.skippedTokenRanges); - repairState.setSkippedTablesCount(collectectedRepairStats.skippedTables); + repairState.setFailedTokenRangesCount(collectedRepairStats.failedTokenRanges); + repairState.setSucceededTokenRangesCount(collectedRepairStats.succeededTokenRanges); + repairState.setSkippedTokenRangesCount(collectedRepairStats.skippedTokenRanges); + repairState.setSkippedTablesCount(collectedRepairStats.skippedTables); repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - startTime)); long timeInHours = TimeUnit.SECONDS.toHours(repairState.getNodeRepairTimeInSec()); logger.info("Local {} repair time {} hour(s), stats: repairKeyspaceCount {}, " + @@ -498,7 +492,7 @@ public AutoRepairState getRepairState(AutoRepairConfig.RepairType repairType) return repairStates.get(repairType); } - static class CollectectedRepairStats + static class CollectedRepairStats { int failedTokenRanges = 0; int succeededTokenRanges = 0; diff --git a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java index c441294d32df..35d1b7ca2252 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java @@ -21,9 +21,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.LinkedHashMap; +import java.util.Iterator; import java.util.List; -import java.util.Map; +import java.util.NoSuchElementException; import org.apache.cassandra.service.AutoRepairService; @@ -36,21 +36,70 @@ public class DefaultAutoRepairTokenSplitter implements IAutoRepairTokenRangeSplitter { @Override - public Map> getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, Map> keyspacesAndTablesToRepair) + public Iterator getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, List repairPlans) { - Map> repairAssignmentsByKeyspace = new LinkedHashMap<>(); - for (Map.Entry> keyspace : keyspacesAndTablesToRepair.entrySet()) + return new RepairAssignmentIterator(repairType, primaryRangeOnly, repairPlans); + } + + private class RepairAssignmentIterator implements Iterator + { + + private final AutoRepairConfig.RepairType repairType; + private final boolean primaryRangeOnly; + + private final Iterator repairPlanIterator; + + private Iterator currentIterator = null; + private PrioritizedRepairPlan currentPlan = null; + + RepairAssignmentIterator(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, List repairPlans) + { + this.repairType = repairType; + this.primaryRangeOnly = primaryRangeOnly; + this.repairPlanIterator = repairPlans.iterator(); + } + + private synchronized Iterator currentIterator() + { + if (currentIterator == null || !currentIterator.hasNext()) + { + // Advance the repair plan iterator if the current repair plan is exhausted, but only + // if there are more repair plans. + if (repairPlanIterator.hasNext()) + { + currentPlan = repairPlanIterator.next(); + currentIterator = currentPlan.getKeyspaceRepairPlans().iterator(); + } + } + + return currentIterator; + } + + @Override + public boolean hasNext() { - repairAssignmentsByKeyspace.put(keyspace.getKey(), getRepairAssignmentsForKeyspace(repairType, primaryRangeOnly, keyspace.getKey(), keyspace.getValue())); + return currentIterator().hasNext(); } - return repairAssignmentsByKeyspace; + @Override + public KeyspaceRepairAssignments next() + { + if (!currentIterator.hasNext()) + { + throw new NoSuchElementException("No remaining repair plans"); + } + + final KeyspaceRepairPlan repairPlan = currentIterator().next(); + return getRepairAssignmentsForKeyspace(repairType, primaryRangeOnly, currentPlan.getPriority(), repairPlan); + } } - private List getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List tableNames) + private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, int priority, KeyspaceRepairPlan repairPlan) { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); List repairAssignments = new ArrayList<>(); + String keyspaceName = repairPlan.getKeyspaceName(); + List tableNames = repairPlan.getTableNames(); Collection> tokens = StorageService.instance.getPrimaryRanges(keyspaceName); if (!primaryRangeOnly) @@ -87,7 +136,7 @@ private List getRepairAssignmentsForKeyspace(AutoRepairConfig. } } } - reorderByPriority(repairAssignments, repairType); - return repairAssignments; + + return new KeyspaceRepairAssignments(priority, keyspaceName, repairAssignments); } } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java index db0c86d7b73f..854bc78a036d 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java @@ -17,17 +17,8 @@ */ package org.apache.cassandra.repair.autorepair; - -import java.util.Comparator; +import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.Objects; - -import com.google.common.annotations.VisibleForTesting; - -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; public interface IAutoRepairTokenRangeSplitter { @@ -37,94 +28,8 @@ public interface IAutoRepairTokenRangeSplitter * The autorepair framework will repair the list of returned subrange in a sequence. * @param repairType The type of repair being executed * @param primaryRangeOnly Whether to repair only this node's primary ranges or all of its ranges. - * @param keyspacesAndTablesToRepair A map keyed by keyspace name and valued by tables to generate assignments for. - * @return repair assignments broken up by range, keyspace and tables. - */ - Map> getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, Map> keyspacesAndTablesToRepair); - - /** - * Reorders the list of {@link RepairAssignment} objects based on their priority for a given repair type. - * The list is sorted in descending order, so higher priority assignments appear first. - * If two assignments have the same priority for the specified repair type, their original order is preserved. - * - * @param repairAssignments A list of {@link RepairAssignment} objects to be reordered. - * @param repairType The {@link AutoRepairConfig.RepairType} used to determine the priority of each assignment. - * The priority is determined using the {@link RepairAssignment#getPriority(AutoRepairConfig.RepairType)} method. + * @param repairPlans A list of ordered prioritized repair plans to generate assignments for in order. + * @return Iterator of repair assignments, with each element representing a grouping of repair assignments for a given keyspace. */ - @VisibleForTesting - default void reorderByPriority(List repairAssignments, AutoRepairConfig.RepairType repairType) - { - repairAssignments.sort(Comparator.comparingInt(a -> ((RepairAssignment) a).getPriority(repairType)).reversed()); - } - - /** - * Defines a repair assignment to be issued by the autorepair framework. - */ - class RepairAssignment - { - final Range tokenRange; - - final String keyspaceName; - - final List tableNames; - - public RepairAssignment(Range tokenRange, String keyspaceName, List tableNames) - { - this.tokenRange = tokenRange; - this.keyspaceName = keyspaceName; - this.tableNames = tableNames; - } - - public Range getTokenRange() - { - return tokenRange; - } - - public String getKeyspaceName() - { - return keyspaceName; - } - - public List getTableNames() - { - return tableNames; - } - - public int getPriority(AutoRepairConfig.RepairType type) - { - int max = 0; - for (String table : tableNames) - { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, table); - if (cfs != null) - max = Math.max(max, cfs.metadata().params.automatedRepair.get(type).priority()); - } - return max; - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RepairAssignment that = (RepairAssignment) o; - return Objects.equals(tokenRange, that.tokenRange) && Objects.equals(keyspaceName, that.keyspaceName) && Objects.equals(tableNames, that.tableNames); - } - - @Override - public int hashCode() - { - return Objects.hash(tokenRange, keyspaceName, tableNames); - } - - @Override - public String toString() - { - return "RepairAssignment{" + - "tokenRange=" + tokenRange + - ", keyspaceName='" + keyspaceName + '\'' + - ", tableNames=" + tableNames + - '}'; - } - } + Iterator getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, List repairPlans); } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairAssignments.java b/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairAssignments.java new file mode 100644 index 000000000000..3ea91e9922f9 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairAssignments.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.List; + +/** + * A grouping of repair assignments that were generated for a particular keyspace for a given priority. + */ +public class KeyspaceRepairAssignments +{ + private final int priority; + private final String keyspaceName; + private final List repairAssignments; + + public KeyspaceRepairAssignments(int priority, String keyspaceName, List repairAssignments) + { + this.priority = priority; + this.keyspaceName = keyspaceName; + this.repairAssignments = repairAssignments; + } + + public int getPriority() + { + return priority; + } + + public String getKeyspaceName() + { + return keyspaceName; + } + + public List getRepairAssignments() + { + return repairAssignments; + } +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairPlan.java b/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairPlan.java new file mode 100644 index 000000000000..3c13e3d80d08 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairPlan.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.List; +import java.util.Objects; + +/** + * Encapsulates an intent to repair the given keyspace's tables + */ +public class KeyspaceRepairPlan +{ + private final String keyspaceName; + + private final List tableNames; + + public KeyspaceRepairPlan(String keyspaceName, List tableNames) + { + this.keyspaceName = keyspaceName; + this.tableNames = tableNames; + } + + public String getKeyspaceName() + { + return keyspaceName; + } + + public List getTableNames() + { + return tableNames; + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + KeyspaceRepairPlan that = (KeyspaceRepairPlan) o; + return Objects.equals(keyspaceName, that.keyspaceName) && Objects.equals(tableNames, that.tableNames); + } + + @Override + public int hashCode() + { + return Objects.hash(keyspaceName, tableNames); + } + + @Override + public String toString() + { + return "KeyspaceRepairPlan{" + + "keyspaceName='" + keyspaceName + '\'' + + ", tableNames=" + tableNames + + '}'; + } +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java new file mode 100644 index 000000000000..98cdd95421b1 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeSet; +import java.util.function.Consumer; + +import org.apache.cassandra.db.ColumnFamilyStore; + +/** + * Encapsulates a devised plan to repair tables, grouped by their keyspace and a given priority. This is used + * by {@link AutoRepair} to pass in an organized plan to + * {@link IAutoRepairTokenRangeSplitter#getRepairAssignments(AutoRepairConfig.RepairType, boolean, List)} which + * can iterate over this plan in order to generate {@link RepairAssignment}s. + */ +public class PrioritizedRepairPlan +{ + private final int priority; + + private final List keyspaceRepairPlans; + + public PrioritizedRepairPlan(int priority, List keyspaceRepairPlans) + { + this.priority = priority; + this.keyspaceRepairPlans = keyspaceRepairPlans; + } + + public int getPriority() + { + return priority; + } + + public List getKeyspaceRepairPlans() + { + return keyspaceRepairPlans; + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + PrioritizedRepairPlan that = (PrioritizedRepairPlan) o; + return priority == that.priority && Objects.equals(keyspaceRepairPlans, that.keyspaceRepairPlans); + } + + @Override + public int hashCode() + { + return Objects.hash(priority, keyspaceRepairPlans); + } + + @Override + public String toString() + { + return "PrioritizedRepairPlan{" + + "priority=" + priority + + ", keyspaceRepairPlans=" + keyspaceRepairPlans + + '}'; + } + + /** + * Builds a list of {@link PrioritizedRepairPlan}s for the given keyspace and table map, ordered by priority from + * highest to lowest, where priority is derived from table schema's defined priority for the given repair type. + *

+ * If a keyspace has tables with differing priorities, those tables will be included in the PrioritizedRepairPlan + * for their given priority. + * + * @param keyspacesToTableNames A mapping keyspace to table names + * @param repairType The repair type that is being executed + * @param orderFunc A function to order keyspace and tables in the returned plan. + * @return Ordered list of plan's by table priorities. + */ + public static List build(Map> keyspacesToTableNames, AutoRepairConfig.RepairType repairType, Consumer> orderFunc) + { + // Build a map of priority -> (keyspace -> tables) + Map>> plans = new HashMap<>(); + for (Map.Entry> keyspaceToTableNames : keyspacesToTableNames.entrySet()) + { + String keyspaceName = keyspaceToTableNames.getKey(); + for (String tableName : keyspaceToTableNames.getValue()) + { + int priority = getPriority(repairType, keyspaceName, tableName); + Map> keyspacesForPriority = plans.computeIfAbsent(priority, p -> new HashMap<>()); + List tableNamesAtPriority = keyspacesForPriority.computeIfAbsent(keyspaceName, k -> new ArrayList<>()); + tableNamesAtPriority.add(tableName); + } + } + + // Extract map into a List ordered by priority from highest to lowest. + List planList = new ArrayList<>(plans.size()); + TreeSet priorities = new TreeSet<>(Comparator.reverseOrder()); + priorities.addAll(plans.keySet()); + for (int priority : priorities) + { + Map> keyspacesAndTables = plans.get(priority); + List keyspaceRepairPlans = new ArrayList<>(keyspacesAndTables.size()); + planList.add(new PrioritizedRepairPlan(priority, keyspaceRepairPlans)); + + // Order keyspace and table names based on the input function (typically, this would shuffle the keyspace + // and table names randomly). + List keyspaceNames = new ArrayList<>(keyspacesAndTables.keySet()); + orderFunc.accept(keyspaceNames); + + for(String keyspaceName : keyspaceNames) + { + List tableNames = keyspacesAndTables.get(keyspaceName); + orderFunc.accept(tableNames); + KeyspaceRepairPlan keyspaceRepairPlan = new KeyspaceRepairPlan(keyspaceName, new ArrayList<>(tableNames)); + keyspaceRepairPlans.add(keyspaceRepairPlan); + } + } + + return planList; + } + + /** + * Convenience method to build a repair plan for a single keyspace with tables. Primarily useful in testing. + * @param keyspaceName Keyspace to repair + * @param tableNames tables to repair for the given keyspace. + * @return Single repair plan. + */ + static List buildSingleKeyspacePlan(AutoRepairConfig.RepairType repairType, String keyspaceName, String ... tableNames) + { + Map> keyspaceMap = new HashMap<>(); + keyspaceMap.put(keyspaceName, Arrays.asList(tableNames)); + return build(keyspaceMap, repairType, (l) -> {}); + } + + /** + * @return The priority of the given table if defined, otherwise 0. + */ + private static int getPriority(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName) + { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); + return cfs != null ? cfs.metadata().params.automatedRepair.get(repairType).priority() : 0; + } +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java new file mode 100644 index 000000000000..c06d58e71fa2 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.List; +import java.util.Objects; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; + +/** + * Defines a repair assignment to be issued by the autorepair framework. + */ +public class RepairAssignment +{ + final Range tokenRange; + + final String keyspaceName; + + final List tableNames; + + public RepairAssignment(Range tokenRange, String keyspaceName, List tableNames) + { + this.tokenRange = tokenRange; + this.keyspaceName = keyspaceName; + this.tableNames = tableNames; + } + + public Range getTokenRange() + { + return tokenRange; + } + + public String getKeyspaceName() + { + return keyspaceName; + } + + public List getTableNames() + { + return tableNames; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RepairAssignment that = (RepairAssignment) o; + return Objects.equals(tokenRange, that.tokenRange) && Objects.equals(keyspaceName, that.keyspaceName) && Objects.equals(tableNames, that.tableNames); + } + + @Override + public int hashCode() + { + return Objects.hash(tokenRange, keyspaceName, tableNames); + } + + @Override + public String toString() + { + return "RepairAssignment{" + + "tokenRange=" + tokenRange + + ", keyspaceName='" + keyspaceName + '\'' + + ", tableNames=" + tableNames + + '}'; + } +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index ef3d1cc44fed..11c10600f7b5 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -23,9 +23,10 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.LinkedHashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -156,17 +157,68 @@ public RepairRangeSplitter(Map parameters) } @Override - public Map> getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, Map> keyspacesAndTablesToRepair) + public Iterator getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, List repairPlans) { - Map> repairAssignmentsByKeyspace = new LinkedHashMap<>(); - for (Map.Entry> keyspace : keyspacesAndTablesToRepair.entrySet()) + return new RepairAssignmentIterator(repairType, primaryRangeOnly, repairPlans); + } + + private class RepairAssignmentIterator implements Iterator + { + + private final AutoRepairConfig.RepairType repairType; + private final boolean primaryRangeOnly; + + private final Iterator repairPlanIterator; + + private Iterator currentIterator = null; + private PrioritizedRepairPlan currentPlan = null; + private long bytesSoFar = 0; + + RepairAssignmentIterator(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, List repairPlans) { - logger.debug("Calculating token range splits for repairType={} primaryRangeOnly={} keyspaceName={} tableNames={}", repairType, primaryRangeOnly, keyspace.getKey(), keyspace.getValue()); - Collection> tokenRanges = getTokenRanges(primaryRangeOnly, keyspace.getKey()); - repairAssignmentsByKeyspace.put(keyspace.getKey(), getRepairAssignmentsForKeyspace(repairType, keyspace.getKey(), keyspace.getValue(), tokenRanges)); + this.repairType = repairType; + this.primaryRangeOnly = primaryRangeOnly; + this.repairPlanIterator = repairPlans.iterator(); } - return filterAndOrderRepairAssignments(repairType, repairAssignmentsByKeyspace); + private synchronized Iterator currentIterator() + { + if (currentIterator == null || !currentIterator.hasNext()) + { + // Advance the repair plan iterator if the current repair plan is exhausted, but only + // if there are more repair plans. + if (repairPlanIterator.hasNext()) + { + currentPlan = repairPlanIterator.next(); + currentIterator = currentPlan.getKeyspaceRepairPlans().iterator(); + } + } + + return currentIterator; + } + + @Override + public boolean hasNext() + { + return currentIterator().hasNext(); + } + + @Override + public KeyspaceRepairAssignments next() + { + // Should not happen unless violating the contract of iterator of checking hasNext first. + if (!currentIterator.hasNext()) + { + throw new NoSuchElementException("No remaining repair plans"); + } + + final KeyspaceRepairPlan repairPlan = currentIterator().next(); + Collection> tokenRanges = getTokenRanges(primaryRangeOnly, repairPlan.getKeyspaceName()); + List repairAssignments = getRepairAssignmentsForKeyspace(repairType, repairPlan.getKeyspaceName(), repairPlan.getTableNames(), tokenRanges); + FilteredRepairAssignments filteredRepairAssignments = filterRepairAssignments(repairType, currentPlan.getPriority(), repairPlan.getKeyspaceName(), repairAssignments, bytesSoFar); + bytesSoFar = filteredRepairAssignments.newBytesSoFar; + return new KeyspaceRepairAssignments(currentPlan.getPriority(), repairPlan.getKeyspaceName(), filteredRepairAssignments.repairAssignments); + } } @VisibleForTesting @@ -238,63 +290,49 @@ else if (tableAssignments.size() == 1 && } /** - * Given a repair type and map of sized-based repair assignments by keyspace order them by priority of the - * assignments' underlying tables and confine them by maxBytesPerSchedule. + * Given a repair type and map of sized-based repair assignments, confine them by maxBytesPerSchedule. * @param repairType used to determine underyling table priorities - * @param repairAssignmentsByKeyspace the assignments to filter. - * @return A list of repair assignments ordered by priority and confined by maxBytesPerSchedule. + * @param repairAssignments the assignments to filter. + * @param bytesSoFar repair assignment bytes accumulated so far. + * @return A list of repair assignments confined by maxBytesPerSchedule. */ @VisibleForTesting - Map> filterAndOrderRepairAssignments(AutoRepairConfig.RepairType repairType, Map> repairAssignmentsByKeyspace) + FilteredRepairAssignments filterRepairAssignments(AutoRepairConfig.RepairType repairType, int priority, String keyspaceName, List repairAssignments, long bytesSoFar) { // Confine repair assignments by maxBytesPerSchedule. - long bytesSoFar = 0L; + long bytesSoFarThisIteration = 0L; long bytesNotRepaired = 0L; int assignmentsNotRepaired = 0; int assignmentsToRepair = 0; int totalAssignments = 0; - Map> filteredRepairAssignmentsByKeyspace = new LinkedHashMap<>(); - for (Map.Entry> keyspaceAssignments : repairAssignmentsByKeyspace.entrySet()) + List assignmentsToReturn = new ArrayList<>(repairAssignments.size()); + for (SizedRepairAssignment repairAssignment : repairAssignments) { - String keyspace = keyspaceAssignments.getKey(); - List repairAssignments = keyspaceAssignments.getValue(); - - // Reorder the repair assignments - reorderByPriority(repairAssignments, repairType); - - List assignmentsToReturn = new ArrayList<>(repairAssignments.size()); - for (SizedRepairAssignment repairAssignment : repairAssignments) + totalAssignments++; + // skip any repair assignments that would accumulate us past the maxBytesPerSchedule + if (bytesSoFar + repairAssignment.getEstimatedBytes() > maxBytesPerSchedule) { - totalAssignments++; - // skip any repair assignments that would accumulate us past the maxBytesPerSchedule - if (bytesSoFar + repairAssignment.getEstimatedBytes() > maxBytesPerSchedule) - { - // log that repair assignment was skipped. - bytesNotRepaired += repairAssignment.getEstimatedBytes(); - assignmentsNotRepaired++; - logger.warn("Skipping {} because it would increase total repair bytes to {}", - repairAssignment, - getBytesOfMaxBytesPerSchedule(bytesSoFar + repairAssignment.getEstimatedBytes())); - } - else - { - bytesSoFar += repairAssignment.getEstimatedBytes(); - assignmentsToRepair++; - logger.info("Adding {}, increasing repair bytes to {}", - repairAssignment, - getBytesOfMaxBytesPerSchedule(bytesSoFar)); - assignmentsToReturn.add(repairAssignment); - } + // log that repair assignment was skipped. + bytesNotRepaired += repairAssignment.getEstimatedBytes(); + assignmentsNotRepaired++; + logger.warn("Skipping {} because it would increase total repair bytes to {}", + repairAssignment, + getBytesOfMaxBytesPerSchedule(bytesSoFar + repairAssignment.getEstimatedBytes())); } - - if (!assignmentsToReturn.isEmpty()) + else { - filteredRepairAssignmentsByKeyspace.put(keyspace, assignmentsToReturn); + bytesSoFar += repairAssignment.getEstimatedBytes(); + bytesSoFarThisIteration += repairAssignment.getEstimatedBytes(); + assignmentsToRepair++; + logger.info("Adding {}, increasing repair bytes to {}", + repairAssignment, + getBytesOfMaxBytesPerSchedule(bytesSoFar)); + assignmentsToReturn.add(repairAssignment); } } - String message = "Returning {} assignment(s), totaling {}"; + String message = "Returning {} assignment(s) for priorityBucket {} and keyspace {}, totaling {} ({} overall)"; if (assignmentsNotRepaired != 0) { message += ". Skipping {} of {} assignment(s), totaling {}"; @@ -303,23 +341,41 @@ Map> filterAndOrderRepairAssignments(AutoRepairCo message += ". The entire primary range will not be repaired this schedule. " + "Consider increasing maxBytesPerSchedule, reducing node density or monitoring to ensure " + "all ranges do get repaired within gc_grace_seconds"; - logger.warn(message, assignmentsToRepair, getBytesOfMaxBytesPerSchedule(bytesSoFar), + logger.warn(message, assignmentsToRepair, priority, keyspaceName, + FileUtils.stringifyFileSize(bytesSoFarThisIteration), + getBytesOfMaxBytesPerSchedule(bytesSoFar), assignmentsNotRepaired, totalAssignments, FileUtils.stringifyFileSize(bytesNotRepaired)); } else { - logger.info(message, assignmentsToRepair, getBytesOfMaxBytesPerSchedule(bytesSoFar), + logger.info(message, assignmentsToRepair, priority, keyspaceName, + FileUtils.stringifyFileSize(bytesSoFarThisIteration), + getBytesOfMaxBytesPerSchedule(bytesSoFar), assignmentsNotRepaired, totalAssignments, FileUtils.stringifyFileSize(bytesNotRepaired)); } } else { - logger.info(message, assignmentsToRepair, getBytesOfMaxBytesPerSchedule(bytesSoFar)); + logger.info(message, assignmentsToRepair, priority, keyspaceName, + FileUtils.stringifyFileSize(bytesSoFarThisIteration), + getBytesOfMaxBytesPerSchedule(bytesSoFar)); } - return filteredRepairAssignmentsByKeyspace; + return new FilteredRepairAssignments(assignmentsToReturn, bytesSoFar); + } + + private static class FilteredRepairAssignments + { + private final List repairAssignments; + private final long newBytesSoFar; + + private FilteredRepairAssignments(List repairAssignments, long newBytesSoFar) + { + this.repairAssignments = repairAssignments; + this.newBytesSoFar = newBytesSoFar; + } } private String getBytesOfMaxBytesPerSchedule(long bytes) { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java index c7a2ff3efa1f..eccf162bbb46 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java @@ -22,9 +22,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.LinkedHashMap; +import java.util.Iterator; import java.util.List; -import java.util.Map; import org.junit.BeforeClass; import org.junit.Test; @@ -37,7 +36,6 @@ import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter.RepairAssignment; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.service.StorageService; @@ -45,7 +43,9 @@ import static org.apache.cassandra.Util.setAutoRepairEnabled; import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; @RunWith(Parameterized.class) public class AutoRepairDefaultTokenSplitterParameterizedTest extends CQLTester @@ -97,15 +97,17 @@ public void testTokenRangesSplitByTable() AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setRepairSubRangeNum(repairType, numberOfSplits); - Map> keyspaceToTables = new LinkedHashMap<>(); - keyspaceToTables.put(KEYSPACE, tables); - Map> assignmentsByKeyspace = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, keyspaceToTables); - // should be 1 entry for the keyspace. - assertEquals(1, assignmentsByKeyspace.size()); - List assignments = assignmentsByKeyspace.get(KEYSPACE); - assertNotNull(assignments); + List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); + + Iterator keyspaceAssignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, plan); + + // should be only 1 entry for the keyspace. + assertTrue(keyspaceAssignments.hasNext()); + KeyspaceRepairAssignments keyspace = keyspaceAssignments.next(); + assertFalse(keyspaceAssignments.hasNext()); + List assignments = keyspace.getRepairAssignments(); assertEquals(totalTokenRanges*numberOfSplits*tables.size(), assignments.size()); assertEquals(expectedToken.size(), assignments.size()); @@ -119,6 +121,7 @@ public void testTokenRangesSplitByTable() } expectedTableIndex = -1; + // should be a set of ranges for each table. for (int i = 0; i> keyspaceToTables = new LinkedHashMap<>(); - keyspaceToTables.put(KEYSPACE, tables); - Map> assignmentsByKeyspace = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, keyspaceToTables); - // should be 1 entry for the keyspace. - assertEquals(1, assignmentsByKeyspace.size()); - List assignments = assignmentsByKeyspace.get(KEYSPACE); + List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); + + Iterator keyspaceAssignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, plan); + + // should be only 1 entry for the keyspace. + assertTrue(keyspaceAssignments.hasNext()); + KeyspaceRepairAssignments keyspace = keyspaceAssignments.next(); + assertFalse(keyspaceAssignments.hasNext()); + + List assignments = keyspace.getRepairAssignments(); assertNotNull(assignments); assertEquals(totalTokenRanges*numberOfSplits, assignments.size()); assertEquals(expectedToken.size(), assignments.size()); + // should only be one set of ranges for the entire keyspace. for (int i = 0; i> expectedToken = new ArrayList<>(tokens); - Map> keyspaceToTables = new LinkedHashMap<>(); - keyspaceToTables.put(KEYSPACE, Collections.singletonList(TABLE)); - Map> assignmentsByKeyspace = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, keyspaceToTables); + List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE); - // should be 1 entry for the keyspace. - assertEquals(1, assignmentsByKeyspace.size()); - List assignments = assignmentsByKeyspace.get(KEYSPACE); + Iterator keyspaceAssignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, plan); + + // should be only 1 entry for the keyspace. + assertTrue(keyspaceAssignments.hasNext()); + KeyspaceRepairAssignments keyspace = keyspaceAssignments.next(); + assertFalse(keyspaceAssignments.hasNext()); + + List assignments = keyspace.getRepairAssignments(); assertNotNull(assignments); // should be 1 entry for the table which covers the full range. diff --git a/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java new file mode 100644 index 000000000000..507d26437e36 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.List; + +import org.junit.Test; + +import org.apache.cassandra.cql3.CQLTester; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class PrioritizedRepairPlanTest extends CQLTester +{ + + @Test + public void testBuildWithDifferentPriorities() + { + // Test reordering assignments with different priorities + String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '3'}"); + String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); + + List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE, table1, table2, table3); + assertEquals(3, prioritizedRepairPlans.size()); + + // Verify the order is by descending priority and matches the expected tables + assertEquals(3, prioritizedRepairPlans.get(0).getPriority()); + assertEquals(table2, prioritizedRepairPlans.get(0).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); + + assertEquals(2, prioritizedRepairPlans.get(1).getPriority()); + assertEquals(table1, prioritizedRepairPlans.get(1).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); + + assertEquals(1, prioritizedRepairPlans.get(2).getPriority()); + assertEquals(table3, prioritizedRepairPlans.get(2).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); + } + + @Test + public void testBuildWithSamePriority() + { + // Test reordering assignments with the same priority + String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + + // Expect only 1 plan since all tables share the same priority + List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE, table1, table2, table3); + assertEquals(1, prioritizedRepairPlans.size()); + + // Verify all tables present in the plan + assertEquals(1, prioritizedRepairPlans.get(0).getKeyspaceRepairPlans().size()); + KeyspaceRepairPlan keyspaceRepairPlan = prioritizedRepairPlans.get(0).getKeyspaceRepairPlans().get(0); + + List tableNames = keyspaceRepairPlan.getTableNames(); + assertEquals(3, tableNames.size()); + assertEquals(table1, tableNames.get(0)); + assertEquals(table2, tableNames.get(1)); + assertEquals(table3, tableNames.get(2)); + } + + @Test + public void testBuildWithMixedPriorities() + { + String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '3'}"); + String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table4 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); + + // Expect only 3 plans + List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE, table1, table2, table3, table4); + assertEquals(3, prioritizedRepairPlans.size()); + + // Verify the order is by descending priority and matches the expected tables + assertEquals(3, prioritizedRepairPlans.get(0).getPriority()); + assertEquals(table2, prioritizedRepairPlans.get(0).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); + + assertEquals(2, prioritizedRepairPlans.get(1).getPriority()); + assertEquals(table1, prioritizedRepairPlans.get(1).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); + assertEquals(table3, prioritizedRepairPlans.get(1).getKeyspaceRepairPlans().get(0).getTableNames().get(1)); + + assertEquals(1, prioritizedRepairPlans.get(2).getPriority()); + assertEquals(table4, prioritizedRepairPlans.get(2).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); + } + + @Test + public void testBuildWithEmptyTableList() + { + // Test with an empty table list (should remain empty) + List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE); + assertTrue(prioritizedRepairPlans.isEmpty()); + } + + @Test + public void testBuildWithOneTable() + { + // Test with a single element (should remain unchanged) + String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '5'}"); + + // Expect only 1 plans + List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE, table1); + assertEquals(1, prioritizedRepairPlans.size()); + + // Verify the order is by descending priority and matches the expected tables + assertEquals(5, prioritizedRepairPlans.get(0).getPriority()); + assertEquals(table1, prioritizedRepairPlans.get(0).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); + } + +} diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java index 606737817ac4..b41da38ed641 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java @@ -38,7 +38,6 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter.RepairAssignment; import org.apache.cassandra.repair.autorepair.RepairRangeSplitter.SizedRepairAssignment; import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.utils.concurrent.Refs; @@ -64,7 +63,8 @@ public static void setUpClass() } @Before - public void setUp() { + public void setUp() + { repairRangeSplitter = new RepairRangeSplitter(Collections.emptyMap()); tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); } @@ -100,103 +100,16 @@ public void testSizePartitionCountSplit() throws Throwable } @Test - public void testReorderByPriorityWithDifferentPriorities() { - String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '3'}"); - String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); - - // Test reordering assignments with different priorities - RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table1)); - RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table2)); - RepairAssignment assignment3 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table3)); - - // Assume these priorities based on the repair type - List assignments = new ArrayList<>(Arrays.asList(assignment1, assignment2, assignment3)); - - repairRangeSplitter.reorderByPriority(assignments, AutoRepairConfig.RepairType.FULL); - - // Verify the order is by descending priority - assertEquals(assignment2, assignments.get(0)); - assertEquals(assignment1, assignments.get(1)); - assertEquals(assignment3, assignments.get(2)); - } - - @Test - public void testReorderByPriorityWithSamePriority() { - String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - - // Test reordering assignments with the same priority - RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table1)); - RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table2)); - RepairAssignment assignment3 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table3)); - - List assignments = new ArrayList<>(Arrays.asList(assignment1, assignment2, assignment3)); - - repairRangeSplitter.reorderByPriority(assignments, AutoRepairConfig.RepairType.FULL); - - // Verify the original order is preserved as all priorities are the same - assertEquals(assignment1, assignments.get(0)); - assertEquals(assignment2, assignments.get(1)); - assertEquals(assignment3, assignments.get(2)); - } - - @Test - public void testReorderByPriorityWithMixedPriorities() { - String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '3'}"); - String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table4 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); - - // Test reordering assignments with mixed priorities - RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table1)); - RepairAssignment assignment2 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table2)); - RepairAssignment assignment3 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table3)); - RepairAssignment assignment4 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table4)); - - List assignments = new ArrayList<>(Arrays.asList(assignment1, assignment2, assignment3, assignment4)); - - repairRangeSplitter.reorderByPriority(assignments, AutoRepairConfig.RepairType.FULL); - - // Verify the order: highest priority first, then preserved order for same priority - assertEquals(assignment2, assignments.get(0)); // Priority 3 - assertEquals(assignment1, assignments.get(1)); // Priority 2 - assertEquals(assignment3, assignments.get(2)); // Priority 2 - assertEquals(assignment4, assignments.get(3)); // Priority 1 - } - - @Test - public void testReorderByPriorityWithEmptyList() { - // Test with an empty list (should remain empty) - List assignments = new ArrayList<>(); - repairRangeSplitter.reorderByPriority(assignments, AutoRepairConfig.RepairType.FULL); - assertTrue(assignments.isEmpty()); - } - - @Test - public void testReorderByPriorityWithOneElement() { - // Test with a single element (should remain unchanged) - String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '5'}"); - - RepairAssignment assignment1 = new RepairAssignment(FULL_RANGE, KEYSPACE, Collections.singletonList(table1)); - - List assignments = new ArrayList<>(Collections.singletonList(assignment1)); - - repairRangeSplitter.reorderByPriority(assignments, AutoRepairConfig.RepairType.FULL); - - assertEquals(assignment1, assignments.get(0)); // Single element should remain in place - } - - @Test - public void testGetRepairAssignmentsForTable_NoSSTables() { + public void testGetRepairAssignmentsForTable_NoSSTables() + { Collection> ranges = Collections.singleton(new Range<>(Murmur3Partitioner.instance.getMinimumToken(), Murmur3Partitioner.instance.getMaximumToken())); List assignments = repairRangeSplitter.getRepairAssignmentsForTable(AutoRepairConfig.RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); assertEquals(0, assignments.size()); } @Test - public void testGetRepairAssignmentsForTable_Single() throws Throwable { + public void testGetRepairAssignmentsForTable_Single() throws Throwable + { Collection> ranges = Collections.singleton(new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken())); insertAndFlushSingleTable(tableName); List assignments = repairRangeSplitter.getRepairAssignmentsForTable(AutoRepairConfig.RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); @@ -204,7 +117,8 @@ public void testGetRepairAssignmentsForTable_Single() throws Throwable { } @Test - public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable { + public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable + { repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "2")); Collection> ranges = Collections.singleton(FULL_RANGE); @@ -218,7 +132,8 @@ public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable { } @Test - public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable { + public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable + { repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "2")); Collection> ranges = Collections.singleton(FULL_RANGE); @@ -231,7 +146,8 @@ public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable { } @Test - public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable { + public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable + { repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "1")); Collection> ranges = Collections.singleton(FULL_RANGE); @@ -242,7 +158,8 @@ public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable { } @Test - public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable { + public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable + { repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "100")); Collection> ranges = Collections.singleton(FULL_RANGE); @@ -253,14 +170,16 @@ public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable { } @Test(expected = IllegalStateException.class) - public void testMergeEmptyAssignments() { + public void testMergeEmptyAssignments() + { // Test when the list of assignments is empty List emptyAssignments = Collections.emptyList(); RepairRangeSplitter.merge(emptyAssignments); } @Test - public void testMergeSingleAssignment() { + public void testMergeSingleAssignment() + { // Test when there is only one assignment in the list String keyspaceName = "testKeyspace"; List tableNames = Arrays.asList("table1", "table2"); @@ -276,7 +195,8 @@ public void testMergeSingleAssignment() { } @Test - public void testMergeMultipleAssignmentsWithSameTokenRangeAndKeyspace() { + public void testMergeMultipleAssignmentsWithSameTokenRangeAndKeyspace() + { // Test merging multiple assignments with the same token range and keyspace String keyspaceName = "testKeyspace"; List tableNames1 = Arrays.asList("table1", "table2"); @@ -294,7 +214,8 @@ public void testMergeMultipleAssignmentsWithSameTokenRangeAndKeyspace() { } @Test(expected = IllegalStateException.class) - public void testMergeDifferentTokenRange() { + public void testMergeDifferentTokenRange() + { // Test merging assignments with different token ranges Iterator> range = AutoRepairUtils.split(FULL_RANGE, 2).iterator(); // Split the full range into two ranges ie (0-100, 100-200 Range tokenRange1 = range.next(); @@ -312,7 +233,8 @@ public void testMergeDifferentTokenRange() { } @Test(expected = IllegalStateException.class) - public void testMergeDifferentKeyspaceName() { + public void testMergeDifferentKeyspaceName() + { // Test merging assignments with different keyspace names List tableNames = Arrays.asList("table1", "table2"); @@ -324,7 +246,8 @@ public void testMergeDifferentKeyspaceName() { } @Test - public void testMergeWithDuplicateTables() { + public void testMergeWithDuplicateTables() + { // Test merging assignments with duplicate table names String keyspaceName = "testKeyspace"; List tableNames1 = Arrays.asList("table1", "table2"); @@ -341,12 +264,14 @@ public void testMergeWithDuplicateTables() { } - private void insertAndFlushSingleTable(String tableName) throws Throwable { + private void insertAndFlushSingleTable(String tableName) throws Throwable + { execute("INSERT INTO %s (k, v) values (?, ?)", 1, 1); flush(); } - private List createAndInsertTables(int count) throws Throwable { + private List createAndInsertTables(int count) throws Throwable + { List tableNames = new ArrayList<>(); for (int i = 0; i < count; i++) { String tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); @@ -356,11 +281,14 @@ private List createAndInsertTables(int count) throws Throwable { return tableNames; } - private void insertAndFlushTable(String tableName) throws Throwable { + private void insertAndFlushTable(String tableName) throws Throwable + { insertAndFlushTable(tableName, 1); ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, tableName); } - private void insertAndFlushTable(String tableName, int... vals) throws Throwable { + + private void insertAndFlushTable(String tableName, int... vals) throws Throwable + { for (int i : vals) { executeFormattedQuery("INSERT INTO " + KEYSPACE + '.' + tableName + " (k, v) values (?, ?)", i, i); From 7de0979acccd73863be426bb4c4bd05993ceb938 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 1 Dec 2024 22:42:10 -0600 Subject: [PATCH 077/199] Add more priority and range splitter tests --- .../repair/autorepair/AutoRepair.java | 9 +-- .../autorepair/PrioritizedRepairPlan.java | 2 +- .../autorepair/RepairRangeSplitter.java | 7 +- .../AutoRepairParameterizedTest.java | 20 +++-- .../autorepair/PrioritizedRepairPlanTest.java | 54 +++++++++++-- .../autorepair/RepairRangeSplitterTest.java | 75 +++++++++++++++---- 6 files changed, 127 insertions(+), 40 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 3b35d21c5e93..a4f61719c018 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -84,7 +84,9 @@ public class AutoRepair protected final Map repairStates; @VisibleForTesting - protected static Consumer> shuffleFunc = java.util.Collections::shuffle; + // Auto-repair is likely to be run on multiple nodes independently, we want to avoid running multiple repair + // sessions on overlapping datasets at the same time. Shuffling keyspaces reduces the likelihood of this happening. + protected static Consumer> shuffleFunc = java.util.Collections::shuffle; @VisibleForTesting protected static BiConsumer sleepFunc = Uninterruptibles::sleepUninterruptibly; @@ -210,10 +212,6 @@ public void repair(AutoRepairConfig.RepairType repairType) List keyspaces = new ArrayList<>(); Keyspace.all().forEach(keyspaces::add); - // Auto-repair is likely to be run on multiple nodes independently, we want to avoid running multiple repair - // sessions on overlapping datasets at the same time. Shuffling keyspaces reduces the likelihood of this happening. - shuffleFunc.accept(keyspaces); - // Filter out keyspaces and tables to repair and group into a map by keyspace. Map> keyspacesAndTablesToRepair = new LinkedHashMap<>(); for (Keyspace keyspace : keyspaces) @@ -223,7 +221,6 @@ public void repair(AutoRepairConfig.RepairType repairType) continue; } List tablesToBeRepairedList = retrieveTablesToBeRepaired(keyspace, config, repairType, repairState, collectedRepairStats); - shuffleFunc.accept(tablesToBeRepairedList); keyspacesAndTablesToRepair.put(keyspace.getName(), tablesToBeRepairedList); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java index 98cdd95421b1..6b10e5fe6b3a 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java +++ b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java @@ -93,7 +93,7 @@ public String toString() * @param orderFunc A function to order keyspace and tables in the returned plan. * @return Ordered list of plan's by table priorities. */ - public static List build(Map> keyspacesToTableNames, AutoRepairConfig.RepairType repairType, Consumer> orderFunc) + public static List build(Map> keyspacesToTableNames, AutoRepairConfig.RepairType repairType, Consumer> orderFunc) { // Build a map of priority -> (keyspace -> tables) Map>> plans = new HashMap<>(); diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 11c10600f7b5..31cc97590137 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -366,10 +366,11 @@ FilteredRepairAssignments filterRepairAssignments(AutoRepairConfig.RepairType re return new FilteredRepairAssignments(assignmentsToReturn, bytesSoFar); } - private static class FilteredRepairAssignments + @VisibleForTesting + static class FilteredRepairAssignments { - private final List repairAssignments; - private final long newBytesSoFar; + final List repairAssignments; + final long newBytesSoFar; private FilteredRepairAssignments(List repairAssignments, long newBytesSoFar) { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 2f68df9487f0..f3457f75e4c8 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -80,6 +80,7 @@ import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.NOT_MY_TURN; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -608,25 +609,28 @@ public void testRepairShufflesKeyspacesAndTables() { AtomicInteger shuffleKeyspacesCall = new AtomicInteger(); AtomicInteger shuffleTablesCall = new AtomicInteger(); - AutoRepair.shuffleFunc = (List list) -> { - assertTrue(list.get(0) instanceof Keyspace || list.get(0) instanceof String); - if (list.get(0) instanceof Keyspace) + AtomicInteger keyspaceCount = new AtomicInteger(); + AutoRepair.shuffleFunc = (List list) -> { + // check whether was invoked for keyspaces or tables + if (list.contains(KEYSPACE)) { shuffleKeyspacesCall.getAndIncrement(); - assertFalse(list.isEmpty()); + keyspaceCount.set(list.size()); } - else if (list.get(0) instanceof String) - { + else + // presume list not containing a keyspace is for tables. shuffleTablesCall.getAndIncrement(); - } }; AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setRepairMinInterval(repairType, "0s"); AutoRepair.instance.repair(repairType); + // Expect a single invocation for keyspaces assertEquals(1, shuffleKeyspacesCall.get()); - assertEquals(4, shuffleTablesCall.get()); + // Expect an invocation for tables for each keyspace + assertNotEquals(0, keyspaceCount.get()); + assertEquals(keyspaceCount.get(), shuffleTablesCall.get()); } @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java index 507d26437e36..011df9d1592e 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java @@ -18,8 +18,11 @@ package org.apache.cassandra.repair.autorepair; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import com.google.common.collect.Lists; import org.junit.Test; import org.apache.cassandra.cql3.CQLTester; @@ -78,25 +81,60 @@ public void testBuildWithSamePriority() @Test public void testBuildWithMixedPriorities() { - String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '3'}"); - String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table4 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); - - // Expect only 3 plans - List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE, table1, table2, table3, table4); - assertEquals(3, prioritizedRepairPlans.size()); + String ks1 = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }"); + String table1 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table2 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '3'}"); + String table3 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table4 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); + // No priority table should be bucketed at priority 0 + String table5 = createTable(ks1,"CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); + + // Create a new keyspace to ensure its tables get grouped with appropriate priority bucket + String ks2 = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }"); + String table6 = createTable(ks2,"CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); + String table7 = createTable(ks2,"CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); + + Map> keyspaceToTableMap = new HashMap<>(); + keyspaceToTableMap.put(ks1, Lists.newArrayList(table1, table2, table3, table4, table5)); + keyspaceToTableMap.put(ks2, Lists.newArrayList(table6, table7)); + + // Expect 4 plans + List prioritizedRepairPlans = PrioritizedRepairPlan.build(keyspaceToTableMap, AutoRepairConfig.RepairType.FULL, java.util.Collections::sort); + assertEquals(4, prioritizedRepairPlans.size()); // Verify the order is by descending priority and matches the expected tables assertEquals(3, prioritizedRepairPlans.get(0).getPriority()); + assertEquals(1, prioritizedRepairPlans.get(0).getKeyspaceRepairPlans().size()); + assertEquals(ks1, prioritizedRepairPlans.get(0).getKeyspaceRepairPlans().get(0).getKeyspaceName()); assertEquals(table2, prioritizedRepairPlans.get(0).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); assertEquals(2, prioritizedRepairPlans.get(1).getPriority()); + assertEquals(1, prioritizedRepairPlans.get(1).getKeyspaceRepairPlans().size()); + + assertEquals(ks1, prioritizedRepairPlans.get(1).getKeyspaceRepairPlans().get(0).getKeyspaceName()); assertEquals(table1, prioritizedRepairPlans.get(1).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); assertEquals(table3, prioritizedRepairPlans.get(1).getKeyspaceRepairPlans().get(0).getTableNames().get(1)); assertEquals(1, prioritizedRepairPlans.get(2).getPriority()); + // 2 keyspaces should be present at priority 1 + assertEquals(2, prioritizedRepairPlans.get(2).getKeyspaceRepairPlans().size()); + // ks1.table4 expected in first plan + assertEquals(ks1, prioritizedRepairPlans.get(2).getKeyspaceRepairPlans().get(0).getKeyspaceName()); assertEquals(table4, prioritizedRepairPlans.get(2).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); + // ks2.table7 expected in second plan + assertEquals(ks2, prioritizedRepairPlans.get(2).getKeyspaceRepairPlans().get(1).getKeyspaceName()); + assertEquals(table7, prioritizedRepairPlans.get(2).getKeyspaceRepairPlans().get(1).getTableNames().get(0)); + + // Tables without priority should get bucketed at priority 0 + assertEquals(0, prioritizedRepairPlans.get(3).getPriority()); + // 2 keyspaces expected + assertEquals(2, prioritizedRepairPlans.get(3).getKeyspaceRepairPlans().size()); + // ks1.table5 expected in first plan + assertEquals(ks1, prioritizedRepairPlans.get(3).getKeyspaceRepairPlans().get(0).getKeyspaceName()); + assertEquals(table5, prioritizedRepairPlans.get(3).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); + // ks2.table6 expected in second plan + assertEquals(ks2, prioritizedRepairPlans.get(3).getKeyspaceRepairPlans().get(1).getKeyspaceName()); + assertEquals(table6, prioritizedRepairPlans.get(3).getKeyspaceRepairPlans().get(1).getTableNames().get(0)); } @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java index b41da38ed641..9baaecca545e 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java @@ -22,15 +22,18 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.config.DataStorageSpec.LongMebibytesBound; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; @@ -38,10 +41,15 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.repair.autorepair.RepairRangeSplitter.FilteredRepairAssignments; +import org.apache.cassandra.repair.autorepair.RepairRangeSplitter.SizeEstimate; import org.apache.cassandra.repair.autorepair.RepairRangeSplitter.SizedRepairAssignment; import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.utils.concurrent.Refs; +import static org.apache.cassandra.repair.autorepair.RepairRangeSplitter.MAX_BYTES_PER_SCHEDULE; +import static org.apache.cassandra.repair.autorepair.RepairRangeSplitter.SUBRANGE_SIZE; import static org.apache.cassandra.repair.autorepair.RepairRangeSplitter.TABLE_BATCH_LIMIT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -57,8 +65,7 @@ public static void setUpClass() { CQLTester.setUpClass(); AutoRepairService.setup(); - // TODO: For now, always repair by keyspace to exercise priority tests - AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(AutoRepairConfig.RepairType.FULL, true); + AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(RepairType.FULL, true); FULL_RANGE = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken()); } @@ -73,9 +80,9 @@ public void setUp() public void testSizePartitionCount() throws Throwable { insertAndFlushTable(tableName, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10); - Refs sstables = RepairRangeSplitter.getSSTableReaderRefs(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, FULL_RANGE); + Refs sstables = RepairRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE); assertEquals(10, sstables.iterator().next().getEstimatedPartitionSize().count()); - RepairRangeSplitter.SizeEstimate sizes = RepairRangeSplitter.getSizesForRangeOfSSTables(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, FULL_RANGE, sstables); + SizeEstimate sizes = RepairRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE, sstables); assertEquals(10, sizes.partitions); } @@ -91,10 +98,10 @@ public void testSizePartitionCountSplit() throws Throwable Range tokenRange2 = range.next(); Assert.assertFalse(range.hasNext()); - Refs sstables1 = RepairRangeSplitter.getSSTableReaderRefs(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, tokenRange1); - Refs sstables2 = RepairRangeSplitter.getSSTableReaderRefs(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, tokenRange2); - RepairRangeSplitter.SizeEstimate sizes1 = RepairRangeSplitter.getSizesForRangeOfSSTables(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, tokenRange1, sstables1); - RepairRangeSplitter.SizeEstimate sizes2 = RepairRangeSplitter.getSizesForRangeOfSSTables(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableName, tokenRange2, sstables2); + Refs sstables1 = RepairRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange1); + Refs sstables2 = RepairRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange2); + SizeEstimate sizes1 = RepairRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange1, sstables1); + SizeEstimate sizes2 = RepairRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange2, sstables2); // +-5% because HLL merge and the applying of range size approx ratio causes estimation errors assertTrue(Math.abs(10000 - (sizes1.partitions + sizes2.partitions)) <= 60); } @@ -103,7 +110,7 @@ public void testSizePartitionCountSplit() throws Throwable public void testGetRepairAssignmentsForTable_NoSSTables() { Collection> ranges = Collections.singleton(new Range<>(Murmur3Partitioner.instance.getMinimumToken(), Murmur3Partitioner.instance.getMaximumToken())); - List assignments = repairRangeSplitter.getRepairAssignmentsForTable(AutoRepairConfig.RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); assertEquals(0, assignments.size()); } @@ -112,7 +119,7 @@ public void testGetRepairAssignmentsForTable_Single() throws Throwable { Collection> ranges = Collections.singleton(new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken())); insertAndFlushSingleTable(tableName); - List assignments = repairRangeSplitter.getRepairAssignmentsForTable(AutoRepairConfig.RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); assertEquals(1, assignments.size()); } @@ -123,7 +130,7 @@ public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, ranges); // We expect two assignments, one with table1 and table2 batched, and one with table3 assertEquals(2, assignments.size()); @@ -138,7 +145,7 @@ public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(2); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, ranges); // We expect one assignment, with two tables batched assertEquals(1, assignments.size()); @@ -152,7 +159,7 @@ public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, ranges); assertEquals(3, assignments.size()); } @@ -164,7 +171,7 @@ public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(5); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, ranges); assertEquals(1, assignments.size()); } @@ -263,6 +270,46 @@ public void testMergeWithDuplicateTables() assertEquals(new HashSet<>(Arrays.asList("table1", "table2", "table3")), new HashSet<>(result.getTableNames())); } + @Test + public void testGetRepairAssignmentsSplitsBySubrangeSizeAndFilterLimitsByMaxBytesPerSchedule() + { + // Ensures that getRepairAssignments splits by SUBRANGE_SIZE and filterRepairAssignments limits by MAX_BYTES_PER_SCHEDULE. + Map parameters = new HashMap<>(); + parameters.put(SUBRANGE_SIZE, "50GiB"); + parameters.put(MAX_BYTES_PER_SCHEDULE, "100GiB"); + repairRangeSplitter = new RepairRangeSplitter(parameters); + + // Given a size estimate of 1024GiB, we should expect 21 splits (50GiB*21 = 1050GiB < 1024GiB) + SizeEstimate sizeEstimate = sizeEstimateByBytes(RepairType.INCREMENTAL, new LongMebibytesBound("1024GiB")); + + List assignments = repairRangeSplitter.getRepairAssignments(Collections.singletonList(sizeEstimate)); + + // Should be 21 assignments, each being ~48.76 GiB + assertEquals(21, assignments.size()); + long expectedBytes = 52357696560L; + for (int i = 0; i < assignments.size(); i++) + { + SizedRepairAssignment assignment = assignments.get(i); + assertEquals("Did not get expected value for assignment " + i, 52357696560L, assignment.getEstimatedBytes()); + } + + // When filtering we should only get 2 assignments back (48.76 * 2 < 100GiB) + FilteredRepairAssignments filteredRepairAssignments = repairRangeSplitter.filterRepairAssignments(RepairType.INCREMENTAL, 0, KEYSPACE, assignments, 0); + List finalRepairAssignments = filteredRepairAssignments.repairAssignments; + assertEquals(2, finalRepairAssignments.size()); + assertEquals(expectedBytes*2, filteredRepairAssignments.newBytesSoFar); + } + + private SizeEstimate sizeEstimateByBytes(RepairType repairType, LongMebibytesBound totalSize) + { + return sizeEstimateByBytes(repairType, totalSize, totalSize); + } + + private SizeEstimate sizeEstimateByBytes(RepairType repairType, LongMebibytesBound sizeInRange, LongMebibytesBound totalSize) + { + return new SizeEstimate(repairType, KEYSPACE, "table1", FULL_RANGE, 1, sizeInRange.toBytes(), totalSize.toBytes()); + } + private void insertAndFlushSingleTable(String tableName) throws Throwable { From ae8138437252ca010b5660a8ed3b1ae39a8aa23b Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Tue, 3 Dec 2024 23:13:58 -0800 Subject: [PATCH 078/199] Address comments --- .../statements/schema/TableAttributes.java | 3 +++ .../cassandra/repair/PreviewRepairTask.java | 1 - .../repair/autorepair/AutoRepairState.java | 10 ---------- .../cassandra/schema/AutoRepairParams.java | 3 ++- .../cassandra/schema/SchemaKeyspace.java | 6 +++++- .../apache/cassandra/schema/TableParams.java | 15 ++++++++++++++- .../cql3/statements/DescribeStatementTest.java | 6 ++++-- .../cassandra/db/SchemaCQLHelperTest.java | 3 ++- .../AutoRepairParameterizedTest.java | 18 ++++++++++++------ 9 files changed, 42 insertions(+), 23 deletions(-) diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index baa9849707dd..19d2502c4b1b 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -163,6 +163,9 @@ private TableParams build(TableParams.Builder builder) if (hasOption(Option.REPAIR_INCREMENTAL)) builder.automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.incremental, getMap(Option.REPAIR_INCREMENTAL))); + if (hasOption(Option.REPAIR_PREVIEW_REPAIRED)) + builder.automatedRepairPreviewRepaired(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.preview_repaired, getMap(Option.REPAIR_PREVIEW_REPAIRED))); + return builder.build(); } diff --git a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java index ee48e6ce170b..978d928b04c1 100644 --- a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java +++ b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java @@ -85,7 +85,6 @@ public Future performUnsafe(ExecutorPlus executor, Sche else { message = (previewKind == PreviewKind.REPAIRED ? "Repaired data is inconsistent\n" : "Preview complete\n") + summary; - RepairMetrics.previewFailures.inc(); if (previewKind == PreviewKind.REPAIRED) maybeSnapshotReplicas(parentSession, keyspace, result.results.get()); // we know its present as summary used it } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index f0c9fb0b01b6..d402475e37e0 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -304,16 +304,6 @@ public RepairRunnable getRepairRunnable(String keyspace, List tables, Se return getRepairRunnable(keyspace, option); } - - @Override - public void progress(String tag, ProgressEvent event) { - if (event.getType() == ProgressEventType.COMPLETE) - { - - } - - super.progress(tag, event); - } } class IncrementalRepairState extends AutoRepairState diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index ae97e562635f..22fc613f68f5 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -45,7 +45,8 @@ public String toString() public static final Map> DEFAULT_OPTIONS = ImmutableMap.of(AutoRepairConfig.RepairType.full, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true)), - AutoRepairConfig.RepairType.incremental, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true))); + AutoRepairConfig.RepairType.incremental, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true)), + AutoRepairConfig.RepairType.preview_repaired, ImmutableMap.of(Option.ENABLED.toString(), Boolean.toString(true))); public final AutoRepairConfig.RepairType type; diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index b5c908a18436..59b13476a335 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -124,6 +124,7 @@ private SchemaKeyspace() + "read_repair text," + "repair_full frozen>," + "repair_incremental frozen>," + + "repair_preview_repaired frozen>," + "PRIMARY KEY ((keyspace_name), table_name))"); private static final TableMetadata Columns = @@ -194,6 +195,7 @@ private SchemaKeyspace() + "read_repair text," + "repair_full frozen>," + "repair_incremental frozen>," + + "repair_preview_repaired frozen>," + "PRIMARY KEY ((keyspace_name), view_name))"); private static final TableMetadata Indexes = @@ -559,7 +561,8 @@ private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBui .add("read_repair", params.readRepair.toString()) .add("extensions", params.extensions) .add("repair_full", params.automatedRepair.get(AutoRepairConfig.RepairType.full).asMap()) - .add("repair_incremental", params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()); + .add("repair_incremental", params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()) + .add("repair_preview_repaired", params.automatedRepair.get(AutoRepairConfig.RepairType.preview_repaired).asMap()); // Only add CDC-enabled flag to schema if it's enabled on the node. This is to work around RTE's post-8099 if a 3.8+ @@ -986,6 +989,7 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row) .readRepair(getReadRepairStrategy(row)) .automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.full, row.getFrozenTextMap("repair_full"))) .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.incremental, row.getFrozenTextMap("repair_incremental"))) + .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.preview_repaired, row.getFrozenTextMap("repair_preview_repaired"))) .build(); } diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 1132a9ae461a..db3b8391ed5e 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -62,6 +62,7 @@ public enum Option READ_REPAIR, REPAIR_FULL, REPAIR_INCREMENTAL, + REPAIR_PREVIEW_REPAIRED, ; @Override @@ -117,6 +118,7 @@ private TableParams(Builder builder) { put(AutoRepairConfig.RepairType.full, builder.automatedRepairFull); put(AutoRepairConfig.RepairType.incremental, builder.automatedRepairIncremental); + put(AutoRepairConfig.RepairType.preview_repaired, builder.automatedRepairPreviewRepaired); } }; } @@ -147,6 +149,7 @@ public static Builder builder(TableParams params) .readRepair(params.readRepair) .automatedRepairFull(params.automatedRepair.get(AutoRepairConfig.RepairType.full)) .automatedRepairIncremental(params.automatedRepair.get(AutoRepairConfig.RepairType.incremental)) + .automatedRepairPreviewRepaired(params.automatedRepair.get(AutoRepairConfig.RepairType.preview_repaired)) ; } @@ -288,6 +291,7 @@ public String toString() .add(Option.READ_REPAIR.toString(), readRepair) .add(Option.REPAIR_FULL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.full)) .add(Option.REPAIR_INCREMENTAL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.incremental)) + .add(Option.REPAIR_PREVIEW_REPAIRED.toString(), automatedRepair.get(AutoRepairConfig.RepairType.preview_repaired)) .toString(); } @@ -339,7 +343,9 @@ public void appendCqlTo(CqlBuilder builder, boolean isView) .newLine() .append("AND repair_full = ").append(automatedRepair.get(AutoRepairConfig.RepairType.full).asMap()) .newLine() - .append("AND repair_incremental = ").append(automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()); + .append("AND repair_incremental = ").append(automatedRepair.get(AutoRepairConfig.RepairType.incremental).asMap()) + .newLine() + .append("AND repair_preview_repaired = ").append(automatedRepair.get(AutoRepairConfig.RepairType.preview_repaired).asMap()); } public static final class Builder @@ -364,6 +370,7 @@ public static final class Builder private AutoRepairParams automatedRepairFull = new AutoRepairParams(AutoRepairConfig.RepairType.full); private AutoRepairParams automatedRepairIncremental = new AutoRepairParams(AutoRepairConfig.RepairType.incremental); + private AutoRepairParams automatedRepairPreviewRepaired = new AutoRepairParams(AutoRepairConfig.RepairType.preview_repaired); public Builder() { @@ -487,5 +494,11 @@ public Builder automatedRepairIncremental(AutoRepairParams val) automatedRepairIncremental = val; return this; } + + public Builder automatedRepairPreviewRepaired(AutoRepairParams val) + { + automatedRepairPreviewRepaired = val; + return this; + } } } diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index 2c910b321c07..86208b5643b1 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -1006,7 +1006,8 @@ private static String tableParametersCql() " AND read_repair = 'BLOCKING'\n" + " AND speculative_retry = '99p'\n" + " AND repair_full = {'enabled': 'true'}\n" + - " AND repair_incremental = {'enabled': 'true'};"; + " AND repair_incremental = {'enabled': 'true'}\n" + + " AND repair_preview_repaired = {'enabled': 'true'};"; } private static String mvParametersCql() @@ -1028,7 +1029,8 @@ private static String mvParametersCql() " AND read_repair = 'BLOCKING'\n" + " AND speculative_retry = '99p'\n" + " AND repair_full = {'enabled': 'true'}\n" + - " AND repair_incremental = {'enabled': 'true'};"; + " AND repair_incremental = {'enabled': 'true'}\n" + + " AND repair_preview_repaired = {'enabled': 'true'};"; } private static String keyspaceOutput() diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java index 152b4112a3c0..c23cdbc5fc88 100644 --- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java @@ -314,7 +314,8 @@ public void testCfmOptionsCQL() " AND read_repair = 'BLOCKING'\n" + " AND speculative_retry = 'ALWAYS'\n" + " AND repair_full = {'enabled': 'true'}\n" + - " AND repair_incremental = {'enabled': 'true'};" + " AND repair_incremental = {'enabled': 'true'}\n" + + " AND repair_preview_repaired = {'enabled': 'true'};" )); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 3cd57f68b079..b639d047ee4c 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -133,7 +133,8 @@ public static void setupClass() throws Exception .addClusteringColumn("i", IntegerType.instance) .addRegularColumn("v", UTF8Type.instance) .params(TableParams.builder().automatedRepairFull(AutoRepairParams.create(AutoRepairConfig.RepairType.full, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))). - automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.incremental, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))).build()) + automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.incremental, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))). + automatedRepairPreviewRepaired(AutoRepairParams.create(AutoRepairConfig.RepairType.preview_repaired, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))).build()) .build(); cfmDisabledAutoRepair = TableMetadata.builder(KEYSPACE, TABLE_DISABLED_AUTO_REPAIR) @@ -142,7 +143,8 @@ public static void setupClass() throws Exception .addClusteringColumn("i", IntegerType.instance) .addRegularColumn("v", UTF8Type.instance) .params(TableParams.builder().automatedRepairFull(AutoRepairParams.create(AutoRepairConfig.RepairType.full, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))). - automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.incremental, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))).build()) + automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.incremental, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))). + automatedRepairPreviewRepaired(AutoRepairParams.create(AutoRepairConfig.RepairType.preview_repaired, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))).build()) .build(); SchemaLoader.prepareServer(); @@ -588,15 +590,19 @@ public void testTableAttribute() { assertTrue(TableAttributes.validKeywords().contains("repair_full")); assertTrue(TableAttributes.validKeywords().contains("repair_incremental")); + assertTrue(TableAttributes.validKeywords().contains("repair_preview_repaired")); } @Test public void testDefaultAutomatedRepair() { - Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); - Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).repairEnabled()); - Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); - Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.incremental).repairEnabled()); + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values() ) + { + Assert.assertTrue(String.format("expected repair type %s to be enabled on table %s", repairType, cfm.name), + cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); + Assert.assertFalse(String.format("expected repair type %s to be disabled on table %s", repairType, cfmDisabledAutoRepair.name), + cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); + } } @Test From f442ce05ebc276071a291847e8cc1e07d73e9bdb Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Tue, 3 Dec 2024 23:31:03 -0800 Subject: [PATCH 079/199] Fix preview repair options for system_schema ks --- src/java/org/apache/cassandra/schema/SchemaKeyspace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index 59b13476a335..954983e2b706 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -989,7 +989,7 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row) .readRepair(getReadRepairStrategy(row)) .automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.full, row.getFrozenTextMap("repair_full"))) .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.incremental, row.getFrozenTextMap("repair_incremental"))) - .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.preview_repaired, row.getFrozenTextMap("repair_preview_repaired"))) + .automatedRepairPreviewRepaired(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.preview_repaired, row.getFrozenTextMap("repair_preview_repaired"))) .build(); } From af83712b75126a2c524774a118b9a64e7e4e04e7 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Wed, 4 Dec 2024 15:23:50 -0800 Subject: [PATCH 080/199] Implement BytesPreviewed and TokenRangesPreviewed metrics --- .../db/compaction/CompactionManager.java | 4 ++-- .../cassandra/metrics/TableMetrics.java | 22 ++++++++++++------- .../cassandra/repair/PreviewRepairTask.java | 4 ++-- .../cassandra/repair/ValidationManager.java | 6 +++++ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index bac694d9e8b1..5a313c7757ea 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -854,7 +854,7 @@ private static void mutateFullyContainedSSTables(ColumnFamilyStore cfs, Set fullyContainedSSTables = findSSTablesToAnticompact(sstableIterator, normalizedRanges, sessionID); - cfs.metric.bytesMutatedAnticompaction.inc(SSTableReader.getTotalBytes(fullyContainedSSTables)); + cfs.metric.bytesMutatedAnticompaction.mark(SSTableReader.getTotalBytes(fullyContainedSSTables)); cfs.getCompactionStrategyManager().mutateRepaired(fullyContainedSSTables, UNREPAIRED_SSTABLE, sessionID, isTransient); // since we're just re-writing the sstable metdata for the fully contained sstables, we don't want // them obsoleted when the anti-compaction is complete. So they're removed from the transaction here @@ -1658,7 +1658,7 @@ private void doAntiCompaction(ColumnFamilyStore cfs, // repairedAt values for these, we still avoid anti-compacting already repaired sstables, as we currently don't // make use of any actual repairedAt value and splitting up sstables just for that is not worth it at this point. Set unrepairedSSTables = sstables.stream().filter((s) -> !s.isRepaired()).collect(Collectors.toSet()); - cfs.metric.bytesAnticompacted.inc(SSTableReader.getTotalBytes(unrepairedSSTables)); + cfs.metric.bytesAnticompacted.mark(SSTableReader.getTotalBytes(unrepairedSSTables)); Collection> groupedSSTables = cfs.getCompactionStrategyManager().groupSSTablesForAntiCompaction(unrepairedSSTables); // iterate over sstables to check if the full / transient / unrepaired ranges intersect them. diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 77b45f08948d..ed51869178f0 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -201,13 +201,17 @@ public class TableMetrics /** number of partitions read creating merkle trees */ public final TableHistogram partitionsValidated; /** number of bytes read while doing anticompaction */ - public final Counter bytesAnticompacted; + public final Meter bytesAnticompacted; /** number of bytes where the whole sstable was contained in a repairing range so that we only mutated the repair status */ - public final Counter bytesMutatedAnticompaction; + public final Meter bytesMutatedAnticompaction; + /** number of bytes that were scanned during preview repair */ + public final Meter bytesPreviewed; + /** number of token ranges that were scanned during preview repair */ + public final Meter tokenRangesPreviewed; /** number of desynchronized token ranges that were detected during preview repair */ - public final Counter previewedDesynchronizedTokenRanges; + public final Meter previewedDesynchronizedTokenRanges; /** number of desynchronized bytes that were detected during preview repair */ - public final Counter previewedDesynchronizedBytes; + public final Meter previewedDesynchronizedBytes; /** ratio of how much we anticompact vs how much we could mutate the repair status*/ public final Gauge mutatedAnticompactionGauge; @@ -902,10 +906,12 @@ protected double getDenominator() bytesValidated = createTableHistogram("BytesValidated", cfs.keyspace.metric.bytesValidated, false); partitionsValidated = createTableHistogram("PartitionsValidated", cfs.keyspace.metric.partitionsValidated, false); - bytesAnticompacted = createTableCounter("BytesAnticompacted"); - bytesMutatedAnticompaction = createTableCounter("BytesMutatedAnticompaction"); - previewedDesynchronizedTokenRanges = createTableCounter("PreviewedDesynchronizedTokenRanges"); - previewedDesynchronizedBytes = createTableCounter("PreviewedDesynchronizedBytes"); + bytesAnticompacted = createTableMeter("BytesAnticompacted"); + bytesMutatedAnticompaction = createTableMeter("BytesMutatedAnticompaction"); + bytesPreviewed = createTableMeter("BytesPreviewed"); + tokenRangesPreviewed = createTableMeter("TokenRangesPreviewed"); + previewedDesynchronizedTokenRanges = createTableMeter("PreviewedDesynchronizedTokenRanges"); + previewedDesynchronizedBytes = createTableMeter("PreviewedDesynchronizedBytes"); mutatedAnticompactionGauge = createTableGauge("MutatedAnticompactionGauge", () -> { double bytesMutated = bytesMutatedAnticompaction.getCount(); diff --git a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java index 978d928b04c1..eb7990de1450 100644 --- a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java +++ b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java @@ -106,8 +106,8 @@ private void emitMetrics(SyncStatSummary summary) return; ColumnFamilyStore cfs = Keyspace.open(key.left).getColumnFamilyStore(key.right); - cfs.metric.previewedDesynchronizedTokenRanges.inc(table.getRanges()); - cfs.metric.previewedDesynchronizedBytes.inc(table.getBytes()); + cfs.metric.previewedDesynchronizedTokenRanges.mark(table.getRanges()); + cfs.metric.previewedDesynchronizedBytes.mark(table.getBytes()); }); } diff --git a/src/java/org/apache/cassandra/repair/ValidationManager.java b/src/java/org/apache/cassandra/repair/ValidationManager.java index a6c9fa9ef2e2..2f8e7fb7fb73 100644 --- a/src/java/org/apache/cassandra/repair/ValidationManager.java +++ b/src/java/org/apache/cassandra/repair/ValidationManager.java @@ -37,6 +37,7 @@ import org.apache.cassandra.metrics.TableMetrics; import org.apache.cassandra.metrics.TopPartitionTracker; import org.apache.cassandra.repair.state.ValidationState; +import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTree; import org.apache.cassandra.utils.MerkleTrees; @@ -143,6 +144,11 @@ private void doValidation(ColumnFamilyStore cfs, Validator validator) throws IOE { cfs.metric.bytesValidated.update(state.estimatedTotalBytes); cfs.metric.partitionsValidated.update(state.partitionsProcessed); + if (validator.getPreviewKind() != PreviewKind.NONE) + { + cfs.metric.tokenRangesPreviewed.mark(validator.desc.ranges.size()); + cfs.metric.bytesPreviewed.mark(state.estimatedTotalBytes); + } if (topPartitionCollector != null) cfs.topPartitions.merge(topPartitionCollector); } From 56601672ac64d8f70f40d13c58f337fb68d859cd Mon Sep 17 00:00:00 2001 From: kzalys Date: Fri, 6 Dec 2024 11:17:37 -0800 Subject: [PATCH 081/199] Update test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java Co-authored-by: Andrew Tolbert <6889771+tolbertam@users.noreply.github.com> --- .../repair/autorepair/AutoRepairParameterizedTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index b639d047ee4c..4e47dad53b68 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -596,7 +596,7 @@ public void testTableAttribute() @Test public void testDefaultAutomatedRepair() { - for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values() ) + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) { Assert.assertTrue(String.format("expected repair type %s to be enabled on table %s", repairType, cfm.name), cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.full).repairEnabled()); From 0372e0dffacc396c7dfaf077fd7c8c3900d19070 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Fri, 6 Dec 2024 13:13:12 -0800 Subject: [PATCH 082/199] Address comments --- src/java/org/apache/cassandra/metrics/TableMetrics.java | 8 ++++---- .../org/apache/cassandra/repair/PreviewRepairTask.java | 4 ++-- .../cassandra/repair/consistent/SyncStatSummary.java | 9 ++++++--- .../repair/autorepair/AutoRepairStateFactoryTest.java | 7 ++++--- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index ed51869178f0..51c6f0cce2bb 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -209,9 +209,9 @@ public class TableMetrics /** number of token ranges that were scanned during preview repair */ public final Meter tokenRangesPreviewed; /** number of desynchronized token ranges that were detected during preview repair */ - public final Meter previewedDesynchronizedTokenRanges; + public final Meter tokenRangesPreviewedDesynchronized; /** number of desynchronized bytes that were detected during preview repair */ - public final Meter previewedDesynchronizedBytes; + public final Meter bytesPreviewedDesynchronized; /** ratio of how much we anticompact vs how much we could mutate the repair status*/ public final Gauge mutatedAnticompactionGauge; @@ -910,8 +910,8 @@ protected double getDenominator() bytesMutatedAnticompaction = createTableMeter("BytesMutatedAnticompaction"); bytesPreviewed = createTableMeter("BytesPreviewed"); tokenRangesPreviewed = createTableMeter("TokenRangesPreviewed"); - previewedDesynchronizedTokenRanges = createTableMeter("PreviewedDesynchronizedTokenRanges"); - previewedDesynchronizedBytes = createTableMeter("PreviewedDesynchronizedBytes"); + tokenRangesPreviewedDesynchronized = createTableMeter("TokenRangesPreviewedDesynchronized"); + bytesPreviewedDesynchronized = createTableMeter("BytesPreviewedDesynchronized"); mutatedAnticompactionGauge = createTableGauge("MutatedAnticompactionGauge", () -> { double bytesMutated = bytesMutatedAnticompaction.getCount(); diff --git a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java index eb7990de1450..ee2090c3fa8f 100644 --- a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java +++ b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java @@ -106,8 +106,8 @@ private void emitMetrics(SyncStatSummary summary) return; ColumnFamilyStore cfs = Keyspace.open(key.left).getColumnFamilyStore(key.right); - cfs.metric.previewedDesynchronizedTokenRanges.mark(table.getRanges()); - cfs.metric.previewedDesynchronizedBytes.mark(table.getBytes()); + cfs.metric.tokenRangesPreviewedDesynchronized.mark(table.getRanges()); + cfs.metric.bytesPreviewedDesynchronized.mark(table.getBytes()); }); } diff --git a/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java index 5536a502c2f1..690970993c4d 100644 --- a/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java +++ b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java @@ -155,11 +155,13 @@ public String toString() return output.toString(); } - public long getBytes() { + public long getBytes() + { return this.bytes; } - public long getRanges() { + public long getRanges() + { return this.ranges; } } @@ -221,7 +223,8 @@ private void calculateTotals() totalsCalculated = true; } - public Map, Table> getTotals() { + public Map, Table> getTotals() + { calculateTotals(); return summaries; } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java index c4d961f2a5fc..3acdd313e52d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class AutoRepairStateFactoryTest { @@ -31,15 +32,15 @@ public class AutoRepairStateFactoryTest public void testGetRepairState() { AutoRepairState state = RepairType.getAutoRepairState(RepairType.full); - assert state instanceof FullRepairState; + assertTrue(state instanceof FullRepairState); state = RepairType.getAutoRepairState(RepairType.incremental); - assert state instanceof IncrementalRepairState; + assertTrue(state instanceof IncrementalRepairState); state = RepairType.getAutoRepairState(RepairType.preview_repaired); - assert state instanceof PreviewRepairedState; + assertTrue(state instanceof PreviewRepairedState); } @Test From 20cd6c8fe80a2c621a4cc75ad2ecc08cf89aa43f Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Fri, 6 Dec 2024 13:20:20 -0800 Subject: [PATCH 083/199] Update metrics docs --- doc/modules/cassandra/pages/operating/metrics.adoc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/modules/cassandra/pages/operating/metrics.adoc b/doc/modules/cassandra/pages/operating/metrics.adoc index 69948a00c90a..c9390556b104 100644 --- a/doc/modules/cassandra/pages/operating/metrics.adoc +++ b/doc/modules/cassandra/pages/operating/metrics.adoc @@ -246,12 +246,20 @@ during validation. |PartitionsValidated |Histogram |Histogram over the number of partitions read during validation. -|BytesAnticompacted |Counter |How many bytes we anticompacted. +|BytesAnticompacted |Meter |How many bytes we anticompacted. -|BytesMutatedAnticompaction |Counter |How many bytes we avoided +|BytesMutatedAnticompaction |Meter |How many bytes we avoided anticompacting because the sstable was fully contained in the repaired range. +|BytesPreviewed |Meter |How many bytes we scanned during preview repair. + +|TokenRangesPreviewed |Meter |How many token ranges we scanned during preview repair. + +|BytesPreviewedDesynchronized |Meter |How many desynchronized bytes we found during preview repair. + +|TokenRangesPreviewedDesynchronized |Meter |How many desynchronized token ranges we found during preview repair. + |MutatedAnticompactionGauge |Gauge |Ratio of bytes mutated vs total bytes repaired. |=== From 8198f25fb81fc5b9d166100d27384f1dd69b8e95 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Fri, 6 Dec 2024 15:01:37 -0800 Subject: [PATCH 084/199] Update metrics --- doc/modules/cassandra/pages/operating/metrics.adoc | 9 ++++----- src/java/org/apache/cassandra/metrics/TableMetrics.java | 3 --- .../org/apache/cassandra/repair/ValidationManager.java | 1 - 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/doc/modules/cassandra/pages/operating/metrics.adoc b/doc/modules/cassandra/pages/operating/metrics.adoc index c9390556b104..4dd82b290759 100644 --- a/doc/modules/cassandra/pages/operating/metrics.adoc +++ b/doc/modules/cassandra/pages/operating/metrics.adoc @@ -252,13 +252,12 @@ read during validation. anticompacting because the sstable was fully contained in the repaired range. -|BytesPreviewed |Meter |How many bytes we scanned during preview repair. +|BytesPreviewed |Meter |Estimated number of bytes that were scanned for local replica during preview repair -|TokenRangesPreviewed |Meter |How many token ranges we scanned during preview repair. +|BytesPreviewedDesynchronized |Meter |Number of desynchronized bytes that were detected among all replicas during preview repair -|BytesPreviewedDesynchronized |Meter |How many desynchronized bytes we found during preview repair. - -|TokenRangesPreviewedDesynchronized |Meter |How many desynchronized token ranges we found during preview repair. +|TokenRangesPreviewedDesynchronized |Meter |Number of token ranges among all replicas where desynchronization was found +during preview repair. These ranges would need to be streamed during subsequent repair. |MutatedAnticompactionGauge |Gauge |Ratio of bytes mutated vs total bytes repaired. diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 51c6f0cce2bb..568920875a86 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -206,8 +206,6 @@ public class TableMetrics public final Meter bytesMutatedAnticompaction; /** number of bytes that were scanned during preview repair */ public final Meter bytesPreviewed; - /** number of token ranges that were scanned during preview repair */ - public final Meter tokenRangesPreviewed; /** number of desynchronized token ranges that were detected during preview repair */ public final Meter tokenRangesPreviewedDesynchronized; /** number of desynchronized bytes that were detected during preview repair */ @@ -909,7 +907,6 @@ protected double getDenominator() bytesAnticompacted = createTableMeter("BytesAnticompacted"); bytesMutatedAnticompaction = createTableMeter("BytesMutatedAnticompaction"); bytesPreviewed = createTableMeter("BytesPreviewed"); - tokenRangesPreviewed = createTableMeter("TokenRangesPreviewed"); tokenRangesPreviewedDesynchronized = createTableMeter("TokenRangesPreviewedDesynchronized"); bytesPreviewedDesynchronized = createTableMeter("BytesPreviewedDesynchronized"); mutatedAnticompactionGauge = createTableGauge("MutatedAnticompactionGauge", () -> diff --git a/src/java/org/apache/cassandra/repair/ValidationManager.java b/src/java/org/apache/cassandra/repair/ValidationManager.java index 2f8e7fb7fb73..a042064eb41a 100644 --- a/src/java/org/apache/cassandra/repair/ValidationManager.java +++ b/src/java/org/apache/cassandra/repair/ValidationManager.java @@ -146,7 +146,6 @@ private void doValidation(ColumnFamilyStore cfs, Validator validator) throws IOE cfs.metric.partitionsValidated.update(state.partitionsProcessed); if (validator.getPreviewKind() != PreviewKind.NONE) { - cfs.metric.tokenRangesPreviewed.mark(validator.desc.ranges.size()); cfs.metric.bytesPreviewed.mark(state.estimatedTotalBytes); } if (topPartitionCollector != null) From 9e1037265c7c0698c3f1000c4521ab7523f816d4 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Fri, 6 Dec 2024 20:38:23 -0600 Subject: [PATCH 085/199] Set thread name based on RepairType.getConfigName() Changing to upper-case enums caused the executor name to be: AutoRepair-Repair-FULL instead of AutoRepair-Repair-full --- .../org/apache/cassandra/repair/autorepair/AutoRepair.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index a4f61719c018..c57f16df6d72 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -104,8 +104,8 @@ protected AutoRepair() repairStates = new EnumMap<>(AutoRepairConfig.RepairType.class); for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) { - repairExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-Repair-" + repairType, Thread.NORM_PRIORITY)); - repairRunnableExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-RepairRunnable-" + repairType, Thread.NORM_PRIORITY)); + repairExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-Repair-" + repairType.getConfigName(), Thread.NORM_PRIORITY)); + repairRunnableExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-RepairRunnable-" + repairType.getConfigName(), Thread.NORM_PRIORITY)); repairStates.put(repairType, AutoRepairConfig.RepairType.getAutoRepairState(repairType)); tokenRangeSplitters.put(repairType, FBUtilities.newAutoRepairTokenRangeSplitter(config.getTokenRangeSplitter(repairType))); } From 84413323826a4496d3313667462683293ac3a602 Mon Sep 17 00:00:00 2001 From: Jaydeepkumar Chovatia Date: Sat, 7 Dec 2024 14:49:53 -0800 Subject: [PATCH 086/199] Nested table params POC --- .../org/apache/cassandra/cql3/CqlBuilder.java | 21 +++++++++++++++ .../repair/autorepair/AutoRepairConfig.java | 2 +- .../autorepair/RepairRangeSplitter.java | 5 ++-- .../cassandra/schema/AutoRepairParams.java | 25 ++++++++++++++--- .../cassandra/schema/SchemaKeyspace.java | 1 + .../apache/cassandra/schema/TableParams.java | 27 ++++++++++++++++--- 6 files changed, 70 insertions(+), 11 deletions(-) diff --git a/src/java/org/apache/cassandra/cql3/CqlBuilder.java b/src/java/org/apache/cassandra/cql3/CqlBuilder.java index d3ca1da1db3a..8305b0c7337b 100644 --- a/src/java/org/apache/cassandra/cql3/CqlBuilder.java +++ b/src/java/org/apache/cassandra/cql3/CqlBuilder.java @@ -152,6 +152,27 @@ public CqlBuilder append(Map map) return append(map, true); } + public CqlBuilder append1(Map> map) + { + indentIfNeeded(); + builder.append('{'); + + Iterator>> iter = new TreeMap<>(map).entrySet() + .iterator(); + while(iter.hasNext()) + { + Entry> e = iter.next(); + appendWithSingleQuotes(e.getKey()); + append(": "); + append(e.getValue()); + if (iter.hasNext()) + builder.append(", "); + } + builder.append('}'); + return this; + } + + public CqlBuilder append(Map map, boolean quoteValue) { indentIfNeeded(); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index b84ee5f2f652..7d7d7a405161 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -344,7 +344,7 @@ protected static Options getDefaultOptions() opts.force_repair_new_node = false; opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h"); opts.mv_repair_enabled = false; - opts.token_range_splitter = new ParameterizedClass(DefaultAutoRepairTokenSplitter.class.getName(), Collections.emptyMap()); + opts.token_range_splitter = new ParameterizedClass(RepairRangeSplitter.class.getName(), Collections.emptyMap()); opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("5m"); // 5 minutes opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h"); // 3 hours diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 31cc97590137..97ab72784376 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -98,8 +98,8 @@ public class RepairRangeSplitter implements IAutoRepairTokenRangeSplitter private final long bytesPerSubrange; private final long partitionsPerSubrange; - private static final long DEFAULT_SUBRANGE_SIZE = new DataStorageSpec.LongBytesBound("100GiB").toBytes(); - private static final long DEFAULT_MAX_BYTES_PER_SCHEDULE = Long.MAX_VALUE; + private static final long DEFAULT_SUBRANGE_SIZE = new DataStorageSpec.LongBytesBound("1MiB").toBytes(); + private static final long DEFAULT_MAX_BYTES_PER_SCHEDULE = new DataStorageSpec.LongBytesBound("100GiB").toBytes(); private static final long DEFAULT_PARTITION_LIMIT = (long) Math.pow(2, DatabaseDescriptor.getRepairSessionMaxTreeDepth()); private static final int DEFAULT_TABLE_BATCH_LIMIT = 64; @@ -608,6 +608,7 @@ static SizeEstimate getSizesForRangeOfSSTables(AutoRepairConfig.RepairType repai double ratio = approxBytesInRange / (double) totalBytes; // use the ratio from size to estimate the partitions in the range as well long partitions = (long) Math.max(1, Math.ceil(cardinality.cardinality() * ratio)); + logger.info("approxBytesInRange {}, totalBytes: {}, table: {}", approxBytesInRange, totalBytes, table); return new SizeEstimate(repairType, keyspace, table, tokenRange, partitions, approxBytesInRange, totalBytes); } diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index af965f503082..2c00f4d83478 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -69,10 +69,11 @@ public String toString() public static AutoRepairParams create(AutoRepairConfig.RepairType repairType, Map options) { Map> optionsMap = new EnumMap<>(AutoRepairConfig.RepairType.class); - for (Map.Entry> entry : DEFAULT_OPTIONS.entrySet()) - { - optionsMap.put(entry.getKey(), new HashMap<>(entry.getValue())); - } + //for (Map.Entry> entry : DEFAULT_OPTIONS.get(repairType)) + //{ + // optionsMap.put(entry.getKey(), new HashMap<>(entry.getValue())); + //} + optionsMap.put(repairType, new HashMap<>(DEFAULT_OPTIONS.get(repairType))); if (options != null) { for (Map.Entry entry : options.entrySet()) @@ -131,6 +132,22 @@ public static AutoRepairParams fromMap(AutoRepairConfig.RepairType repairType, M return create(repairType, map); } + public static Map fromMap(Map> map) + { + Map repairParams = new EnumMap<>(AutoRepairConfig.RepairType.class); + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + Map options = null; + if (map != null) + { + options = map.get(repairType.toString()); + } + repairParams.put(repairType, create(repairType, options)); + } + return repairParams; + } + + public Map asMap() { return options.get(type); diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index adc353ea7fae..1acdadec1e67 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -990,6 +990,7 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row) .automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.FULL, row.getFrozenTextMap("repair_full"))) .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.INCREMENTAL, row.getFrozenTextMap("repair_incremental"))) .automatedRepairPreviewRepaired(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, row.getFrozenTextMap("repair_preview_repaired"))) + .automatedRepair(AutoRepairParams.fromMap(row.getFrozenMap("auto_repair", UTF8Type.instance, MapType.getInstance(UTF8Type.instance, UTF8Type.instance, false)))) .build(); } diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 2c29394a0147..350f603c15f1 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -63,6 +63,7 @@ public enum Option REPAIR_FULL, REPAIR_INCREMENTAL, REPAIR_PREVIEW_REPAIRED, + AUTO_REPAIR, ; @Override @@ -91,6 +92,7 @@ public String toString() public final ReadRepairStrategy readRepair; public final Map automatedRepair; + public final Map autoRepair; private TableParams(Builder builder) { @@ -121,6 +123,7 @@ private TableParams(Builder builder) put(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, builder.automatedRepairPreviewRepaired); } }; + autoRepair = builder.autoRepair; } public static Builder builder() @@ -150,7 +153,7 @@ public static Builder builder(TableParams params) .automatedRepairFull(params.automatedRepair.get(AutoRepairConfig.RepairType.FULL)) .automatedRepairIncremental(params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL)) .automatedRepairPreviewRepaired(params.automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED)) - ; + .automatedRepair(params.autoRepair); } public Builder unbuild() @@ -210,6 +213,11 @@ public void validate() { entry.getValue().validate(); } + + for (Map.Entry entry : autoRepair.entrySet()) + { + entry.getValue().validate(); + } } private static void fail(String format, Object... args) @@ -244,7 +252,8 @@ public boolean equals(Object o) && extensions.equals(p.extensions) && cdc == p.cdc && readRepair == p.readRepair - && automatedRepair.equals(p.automatedRepair); + && automatedRepair.equals(p.automatedRepair) + && autoRepair.equals(p.autoRepair); } @Override @@ -266,7 +275,8 @@ public int hashCode() extensions, cdc, readRepair, - automatedRepair); + automatedRepair, + autoRepair); } @Override @@ -292,6 +302,7 @@ public String toString() .add(Option.REPAIR_FULL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.FULL)) .add(Option.REPAIR_INCREMENTAL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL)) .add(Option.REPAIR_PREVIEW_REPAIRED.toString(), automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED)) + .add(Option.AUTO_REPAIR.toString(), autoRepair) .toString(); } @@ -345,7 +356,9 @@ public void appendCqlTo(CqlBuilder builder, boolean isView) .newLine() .append("AND repair_incremental = ").append(automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).asMap()) .newLine() - .append("AND repair_preview_repaired = ").append(automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED).asMap()); + .append("AND repair_preview_repaired = ").append(automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED).asMap()) + .newLine() + .append("AND auto_repair = ").append1(autoRepair.entrySet().stream().collect(toMap(e -> e.getKey().toString().toLowerCase(), e -> e.getValue().asMap()))); } public static final class Builder @@ -371,6 +384,7 @@ public static final class Builder private AutoRepairParams automatedRepairFull = new AutoRepairParams(AutoRepairConfig.RepairType.FULL); private AutoRepairParams automatedRepairIncremental = new AutoRepairParams(AutoRepairConfig.RepairType.INCREMENTAL); private AutoRepairParams automatedRepairPreviewRepaired = new AutoRepairParams(AutoRepairConfig.RepairType.PREVIEW_REPAIRED); + private Map autoRepair = new EnumMap<>(AutoRepairConfig.RepairType.class); public Builder() { @@ -500,5 +514,10 @@ public Builder automatedRepairPreviewRepaired(AutoRepairParams val) automatedRepairPreviewRepaired = val; return this; } + + public Builder automatedRepair(Map val) { + autoRepair = val; + return this; + } } } From 461cb70741f64372325a1cc28a27db382ee8a1b1 Mon Sep 17 00:00:00 2001 From: Jaydeepkumar Chovatia Date: Sun, 8 Dec 2024 09:25:38 -0800 Subject: [PATCH 087/199] Revert "Nested table params POC" This reverts commit 84413323826a4496d3313667462683293ac3a602. --- .../org/apache/cassandra/cql3/CqlBuilder.java | 21 --------------- .../repair/autorepair/AutoRepairConfig.java | 2 +- .../autorepair/RepairRangeSplitter.java | 5 ++-- .../cassandra/schema/AutoRepairParams.java | 25 +++-------------- .../cassandra/schema/SchemaKeyspace.java | 1 - .../apache/cassandra/schema/TableParams.java | 27 +++---------------- 6 files changed, 11 insertions(+), 70 deletions(-) diff --git a/src/java/org/apache/cassandra/cql3/CqlBuilder.java b/src/java/org/apache/cassandra/cql3/CqlBuilder.java index 8305b0c7337b..d3ca1da1db3a 100644 --- a/src/java/org/apache/cassandra/cql3/CqlBuilder.java +++ b/src/java/org/apache/cassandra/cql3/CqlBuilder.java @@ -152,27 +152,6 @@ public CqlBuilder append(Map map) return append(map, true); } - public CqlBuilder append1(Map> map) - { - indentIfNeeded(); - builder.append('{'); - - Iterator>> iter = new TreeMap<>(map).entrySet() - .iterator(); - while(iter.hasNext()) - { - Entry> e = iter.next(); - appendWithSingleQuotes(e.getKey()); - append(": "); - append(e.getValue()); - if (iter.hasNext()) - builder.append(", "); - } - builder.append('}'); - return this; - } - - public CqlBuilder append(Map map, boolean quoteValue) { indentIfNeeded(); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 7d7d7a405161..b84ee5f2f652 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -344,7 +344,7 @@ protected static Options getDefaultOptions() opts.force_repair_new_node = false; opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h"); opts.mv_repair_enabled = false; - opts.token_range_splitter = new ParameterizedClass(RepairRangeSplitter.class.getName(), Collections.emptyMap()); + opts.token_range_splitter = new ParameterizedClass(DefaultAutoRepairTokenSplitter.class.getName(), Collections.emptyMap()); opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("5m"); // 5 minutes opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h"); // 3 hours diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java index 97ab72784376..31cc97590137 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java @@ -98,8 +98,8 @@ public class RepairRangeSplitter implements IAutoRepairTokenRangeSplitter private final long bytesPerSubrange; private final long partitionsPerSubrange; - private static final long DEFAULT_SUBRANGE_SIZE = new DataStorageSpec.LongBytesBound("1MiB").toBytes(); - private static final long DEFAULT_MAX_BYTES_PER_SCHEDULE = new DataStorageSpec.LongBytesBound("100GiB").toBytes(); + private static final long DEFAULT_SUBRANGE_SIZE = new DataStorageSpec.LongBytesBound("100GiB").toBytes(); + private static final long DEFAULT_MAX_BYTES_PER_SCHEDULE = Long.MAX_VALUE; private static final long DEFAULT_PARTITION_LIMIT = (long) Math.pow(2, DatabaseDescriptor.getRepairSessionMaxTreeDepth()); private static final int DEFAULT_TABLE_BATCH_LIMIT = 64; @@ -608,7 +608,6 @@ static SizeEstimate getSizesForRangeOfSSTables(AutoRepairConfig.RepairType repai double ratio = approxBytesInRange / (double) totalBytes; // use the ratio from size to estimate the partitions in the range as well long partitions = (long) Math.max(1, Math.ceil(cardinality.cardinality() * ratio)); - logger.info("approxBytesInRange {}, totalBytes: {}, table: {}", approxBytesInRange, totalBytes, table); return new SizeEstimate(repairType, keyspace, table, tokenRange, partitions, approxBytesInRange, totalBytes); } diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index 2c00f4d83478..af965f503082 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -69,11 +69,10 @@ public String toString() public static AutoRepairParams create(AutoRepairConfig.RepairType repairType, Map options) { Map> optionsMap = new EnumMap<>(AutoRepairConfig.RepairType.class); - //for (Map.Entry> entry : DEFAULT_OPTIONS.get(repairType)) - //{ - // optionsMap.put(entry.getKey(), new HashMap<>(entry.getValue())); - //} - optionsMap.put(repairType, new HashMap<>(DEFAULT_OPTIONS.get(repairType))); + for (Map.Entry> entry : DEFAULT_OPTIONS.entrySet()) + { + optionsMap.put(entry.getKey(), new HashMap<>(entry.getValue())); + } if (options != null) { for (Map.Entry entry : options.entrySet()) @@ -132,22 +131,6 @@ public static AutoRepairParams fromMap(AutoRepairConfig.RepairType repairType, M return create(repairType, map); } - public static Map fromMap(Map> map) - { - Map repairParams = new EnumMap<>(AutoRepairConfig.RepairType.class); - for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) - { - Map options = null; - if (map != null) - { - options = map.get(repairType.toString()); - } - repairParams.put(repairType, create(repairType, options)); - } - return repairParams; - } - - public Map asMap() { return options.get(type); diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index 1acdadec1e67..adc353ea7fae 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -990,7 +990,6 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row) .automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.FULL, row.getFrozenTextMap("repair_full"))) .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.INCREMENTAL, row.getFrozenTextMap("repair_incremental"))) .automatedRepairPreviewRepaired(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, row.getFrozenTextMap("repair_preview_repaired"))) - .automatedRepair(AutoRepairParams.fromMap(row.getFrozenMap("auto_repair", UTF8Type.instance, MapType.getInstance(UTF8Type.instance, UTF8Type.instance, false)))) .build(); } diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 350f603c15f1..2c29394a0147 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -63,7 +63,6 @@ public enum Option REPAIR_FULL, REPAIR_INCREMENTAL, REPAIR_PREVIEW_REPAIRED, - AUTO_REPAIR, ; @Override @@ -92,7 +91,6 @@ public String toString() public final ReadRepairStrategy readRepair; public final Map automatedRepair; - public final Map autoRepair; private TableParams(Builder builder) { @@ -123,7 +121,6 @@ private TableParams(Builder builder) put(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, builder.automatedRepairPreviewRepaired); } }; - autoRepair = builder.autoRepair; } public static Builder builder() @@ -153,7 +150,7 @@ public static Builder builder(TableParams params) .automatedRepairFull(params.automatedRepair.get(AutoRepairConfig.RepairType.FULL)) .automatedRepairIncremental(params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL)) .automatedRepairPreviewRepaired(params.automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED)) - .automatedRepair(params.autoRepair); + ; } public Builder unbuild() @@ -213,11 +210,6 @@ public void validate() { entry.getValue().validate(); } - - for (Map.Entry entry : autoRepair.entrySet()) - { - entry.getValue().validate(); - } } private static void fail(String format, Object... args) @@ -252,8 +244,7 @@ public boolean equals(Object o) && extensions.equals(p.extensions) && cdc == p.cdc && readRepair == p.readRepair - && automatedRepair.equals(p.automatedRepair) - && autoRepair.equals(p.autoRepair); + && automatedRepair.equals(p.automatedRepair); } @Override @@ -275,8 +266,7 @@ public int hashCode() extensions, cdc, readRepair, - automatedRepair, - autoRepair); + automatedRepair); } @Override @@ -302,7 +292,6 @@ public String toString() .add(Option.REPAIR_FULL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.FULL)) .add(Option.REPAIR_INCREMENTAL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL)) .add(Option.REPAIR_PREVIEW_REPAIRED.toString(), automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED)) - .add(Option.AUTO_REPAIR.toString(), autoRepair) .toString(); } @@ -356,9 +345,7 @@ public void appendCqlTo(CqlBuilder builder, boolean isView) .newLine() .append("AND repair_incremental = ").append(automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).asMap()) .newLine() - .append("AND repair_preview_repaired = ").append(automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED).asMap()) - .newLine() - .append("AND auto_repair = ").append1(autoRepair.entrySet().stream().collect(toMap(e -> e.getKey().toString().toLowerCase(), e -> e.getValue().asMap()))); + .append("AND repair_preview_repaired = ").append(automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED).asMap()); } public static final class Builder @@ -384,7 +371,6 @@ public static final class Builder private AutoRepairParams automatedRepairFull = new AutoRepairParams(AutoRepairConfig.RepairType.FULL); private AutoRepairParams automatedRepairIncremental = new AutoRepairParams(AutoRepairConfig.RepairType.INCREMENTAL); private AutoRepairParams automatedRepairPreviewRepaired = new AutoRepairParams(AutoRepairConfig.RepairType.PREVIEW_REPAIRED); - private Map autoRepair = new EnumMap<>(AutoRepairConfig.RepairType.class); public Builder() { @@ -514,10 +500,5 @@ public Builder automatedRepairPreviewRepaired(AutoRepairParams val) automatedRepairPreviewRepaired = val; return this; } - - public Builder automatedRepair(Map val) { - autoRepair = val; - return this; - } } } From 78fa0b5c665bfd522a73c37d525a64d760ef098e Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 8 Dec 2024 14:01:56 -0800 Subject: [PATCH 088/199] Flatten the auto_repair table property --- pylib/cqlshlib/test/test_cqlsh_output.py | 3 +- .../statements/schema/TableAttributes.java | 10 +- .../repair/autorepair/AutoRepair.java | 4 +- .../autorepair/PrioritizedRepairPlan.java | 2 +- .../cassandra/schema/AutoRepairParams.java | 103 +++++++++++------- .../cassandra/schema/SchemaKeyspace.java | 17 +-- .../apache/cassandra/schema/TableParams.java | 62 ++--------- .../statements/DescribeStatementTest.java | 8 +- .../cassandra/db/SchemaCQLHelperTest.java | 4 +- .../AutoRepairParameterizedTest.java | 30 ++--- .../autorepair/PrioritizedRepairPlanTest.java | 24 ++-- 11 files changed, 117 insertions(+), 150 deletions(-) diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index 3cb3482d100a..377aa9f4d007 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -691,8 +691,7 @@ def test_describe_columnfamily_output(self): AND min_index_interval = 128 AND read_repair = 'BLOCKING' AND speculative_retry = '99p' - AND repair_full = {'enabled': 'true'} - AND repair_incremental = {'enabled': 'true'};""" % quote_name(get_keyspace())) + AND auto_repair = {'full_enabled': 'true', 'incremental_enabled': 'true', 'preview_repaired_enabled': 'true', 'priority': '0'};""" % quote_name(get_keyspace())) with testrun_cqlsh(tty=True, env=self.default_env) as c: for cmdword in ('describe table', 'desc columnfamily'): diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index 7a3255a285d5..d0c6fba2f0a9 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -157,14 +157,8 @@ private TableParams build(TableParams.Builder builder) if (hasOption(Option.READ_REPAIR)) builder.readRepair(ReadRepairStrategy.fromString(getString(Option.READ_REPAIR))); - if (hasOption(Option.REPAIR_FULL)) - builder.automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.FULL, getMap(Option.REPAIR_FULL))); - - if (hasOption(Option.REPAIR_INCREMENTAL)) - builder.automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.INCREMENTAL, getMap(Option.REPAIR_INCREMENTAL))); - - if (hasOption(Option.REPAIR_PREVIEW_REPAIRED)) - builder.automatedRepairPreviewRepaired(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, getMap(Option.REPAIR_PREVIEW_REPAIRED))); + if (hasOption(Option.AUTO_REPAIR)) + builder.automatedRepair(AutoRepairParams.fromMap(getMap(Option.AUTO_REPAIR))); return builder.build(); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index c57f16df6d72..9050885abf2b 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -155,7 +155,7 @@ public void repairAsync(AutoRepairConfig.RepairType repairType) private int getPriority(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName) { ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); - return cfs != null ? cfs.metadata().params.automatedRepair.get(repairType).priority() : 0; + return cfs != null ? cfs.metadata().params.autoRepair.priority() : 0; } // repair runs a repair session of the given type synchronously. @@ -412,7 +412,7 @@ private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairCon String tableName = tableMetadata.name; ColumnFamilyStore columnFamilyStore = keyspace.getColumnFamilyStore(tableName); - if (!columnFamilyStore.metadata().params.automatedRepair.get(repairType).repairEnabled()) + if (!columnFamilyStore.metadata().params.autoRepair.repairEnabled(repairType)) { logger.info("Repair is disabled for keyspace {} for tables: {}", keyspace.getName(), tableName); repairState.setTotalDisabledTablesRepairCount(repairState.getTotalDisabledTablesRepairCount() + 1); diff --git a/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java index 6b10e5fe6b3a..857fbe19b406 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java +++ b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java @@ -155,6 +155,6 @@ static List buildSingleKeyspacePlan(AutoRepairConfig.Repa private static int getPriority(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName) { ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); - return cfs != null ? cfs.metadata().params.automatedRepair.get(repairType).priority() : 0; + return cfs != null ? cfs.metadata().params.autoRepair.priority() : 0; } } diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index af965f503082..b012a48188d8 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -18,10 +18,9 @@ package org.apache.cassandra.schema; import java.util.Arrays; -import java.util.EnumMap; -import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableMap; @@ -36,7 +35,9 @@ public final class AutoRepairParams { public enum Option { - ENABLED, + FULL_ENABLED, + INCREMENTAL_ENABLED, + PREVIEW_REPAIRED_ENABLED, PRIORITY; @Override @@ -46,32 +47,29 @@ public String toString() } } - public static final Map DEFAULT_SUB_OPTIONS = ImmutableMap.of( - Option.ENABLED.toString(), Boolean.toString(true), - Option.PRIORITY.toString(), "0" - ); + private ImmutableMap options; - public static final Map> DEFAULT_OPTIONS = ImmutableMap.of( - AutoRepairConfig.RepairType.FULL, DEFAULT_SUB_OPTIONS, - AutoRepairConfig.RepairType.INCREMENTAL, DEFAULT_SUB_OPTIONS, - AutoRepairConfig.RepairType.PREVIEW_REPAIRED, DEFAULT_SUB_OPTIONS + public static final Map DEFAULT_OPTIONS = ImmutableMap.of( + Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(true), + Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(true), + Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(true), + Option.PRIORITY.toString(), "0" ); - public final AutoRepairConfig.RepairType type; - - private Map> options = DEFAULT_OPTIONS; - - AutoRepairParams(AutoRepairConfig.RepairType type) + AutoRepairParams(Map options) { - this.type = type; + this.options = ImmutableMap.copyOf(options); } - public static AutoRepairParams create(AutoRepairConfig.RepairType repairType, Map options) + public static final AutoRepairParams DEFAULT = + new AutoRepairParams(DEFAULT_OPTIONS); + + public static AutoRepairParams create(Map options) { - Map> optionsMap = new EnumMap<>(AutoRepairConfig.RepairType.class); - for (Map.Entry> entry : DEFAULT_OPTIONS.entrySet()) + Map optionsMap = new TreeMap<>(); + for (Map.Entry entry : DEFAULT_OPTIONS.entrySet()) { - optionsMap.put(entry.getKey(), new HashMap<>(entry.getValue())); + optionsMap.put(entry.getKey(), entry.getValue()); } if (options != null) { @@ -81,38 +79,61 @@ public static AutoRepairParams create(AutoRepairConfig.RepairType repairType, Ma { throw new ConfigurationException(format("Unknown property '%s'", entry.getKey())); } - optionsMap.get(repairType).put(entry.getKey(), entry.getValue()); + optionsMap.put(entry.getKey(), entry.getValue()); } } - AutoRepairParams repairParams = new AutoRepairParams(repairType); - repairParams.options = optionsMap; - return repairParams; + return new AutoRepairParams(optionsMap); } - public boolean repairEnabled() + public boolean repairEnabled(AutoRepairConfig.RepairType type) { - String enabled = options.get(type).get(Option.ENABLED.toString()); + String option = type.toString().toLowerCase() + "_enabled"; + String enabled = options.get(option); return enabled == null - ? Boolean.parseBoolean(DEFAULT_OPTIONS.get(type).get(Option.ENABLED.toString())) + ? Boolean.parseBoolean(DEFAULT_OPTIONS.get(option)) : Boolean.parseBoolean(enabled); } public int priority() { - String priority = options.get(type).get(Option.PRIORITY.toString()); + String priority = options.get(Option.PRIORITY.toString()); return priority == null - ? Integer.parseInt(DEFAULT_OPTIONS.get(type).get(Option.PRIORITY.toString())) + ? Integer.parseInt(DEFAULT_OPTIONS.get(Option.PRIORITY.toString())) : Integer.parseInt(priority); } public void validate() { - String enabled = options.get(type).get(Option.ENABLED.toString()); - if (enabled != null && !isValidBoolean(enabled)) + for (Option option : Option.values()) + { + if (!options.containsKey(option.toString().toLowerCase())) + { + throw new ConfigurationException(format("Missing repair sub-option '%s'", option)); + } + } + if (options.get(Option.FULL_ENABLED.toString().toLowerCase()) != null && !isValidBoolean(options.get(Option.FULL_ENABLED.toString().toLowerCase()))) + { + throw new ConfigurationException(format("Invalid value %s for '%s' repair sub-option - must be a boolean", + options.get(Option.FULL_ENABLED.toString().toLowerCase()), + Option.FULL_ENABLED)); + } + if (options.get(Option.INCREMENTAL_ENABLED.toString().toLowerCase()) != null && !isValidBoolean(options.get(Option.INCREMENTAL_ENABLED.toString().toLowerCase()))) { throw new ConfigurationException(format("Invalid value %s for '%s' repair sub-option - must be a boolean", - enabled, - Option.ENABLED)); + options.get(Option.INCREMENTAL_ENABLED.toString().toLowerCase()), + Option.INCREMENTAL_ENABLED)); + } + if (options.get(Option.PREVIEW_REPAIRED_ENABLED.toString().toLowerCase()) != null && !isValidBoolean(options.get(Option.PREVIEW_REPAIRED_ENABLED.toString().toLowerCase()))) + { + throw new ConfigurationException(format("Invalid value %s for '%s' repair sub-option - must be a boolean", + options.get(Option.PREVIEW_REPAIRED_ENABLED.toString().toLowerCase()), + Option.PREVIEW_REPAIRED_ENABLED)); + } + if (options.get(Option.PRIORITY.toString().toLowerCase()) != null && !isValidInt(options.get(Option.PRIORITY.toString().toLowerCase()))) + { + throw new ConfigurationException(format("Invalid value %s for '%s' repair sub-option - must be an integer", + options.get(Option.PRIORITY.toString().toLowerCase()), + Option.PRIORITY)); } } @@ -121,19 +142,25 @@ public static boolean isValidBoolean(String value) return StringUtils.equalsIgnoreCase(value, "true") || StringUtils.equalsIgnoreCase(value, "false"); } + public static boolean isValidInt(String value) + { + return StringUtils.isNumeric(value); + } + + public Map options() { - return options.get(type); + return options; } - public static AutoRepairParams fromMap(AutoRepairConfig.RepairType repairType, Map map) + public static AutoRepairParams fromMap(Map map) { - return create(repairType, map); + return create(map); } public Map asMap() { - return options.get(type); + return options; } @Override diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index adc353ea7fae..8c45e74f1ccb 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -42,7 +42,6 @@ import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.rows.*; import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; import org.apache.cassandra.schema.ColumnMetadata.ClusteringOrder; import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff; @@ -122,9 +121,7 @@ private SchemaKeyspace() + "additional_write_policy text," + "cdc boolean," + "read_repair text," - + "repair_full frozen>," - + "repair_incremental frozen>," - + "repair_preview_repaired frozen>," + + "auto_repair frozen>," + "PRIMARY KEY ((keyspace_name), table_name))"); private static final TableMetadata Columns = @@ -193,9 +190,7 @@ private SchemaKeyspace() + "additional_write_policy text," + "cdc boolean," + "read_repair text," - + "repair_full frozen>," - + "repair_incremental frozen>," - + "repair_preview_repaired frozen>," + + "auto_repair frozen>," + "PRIMARY KEY ((keyspace_name), view_name))"); private static final TableMetadata Indexes = @@ -560,9 +555,7 @@ private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBui .add("compression", params.compression.asMap()) .add("read_repair", params.readRepair.toString()) .add("extensions", params.extensions) - .add("repair_full", params.automatedRepair.get(AutoRepairConfig.RepairType.FULL).asMap()) - .add("repair_incremental", params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).asMap()) - .add("repair_preview_repaired", params.automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED).asMap()); + .add("auto_repair", params.autoRepair.asMap()); // Only add CDC-enabled flag to schema if it's enabled on the node. This is to work around RTE's post-8099 if a 3.8+ @@ -987,9 +980,7 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row) SpeculativeRetryPolicy.fromString("99PERCENTILE")) .cdc(row.has("cdc") && row.getBoolean("cdc")) .readRepair(getReadRepairStrategy(row)) - .automatedRepairFull(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.FULL, row.getFrozenTextMap("repair_full"))) - .automatedRepairIncremental(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.INCREMENTAL, row.getFrozenTextMap("repair_incremental"))) - .automatedRepairPreviewRepaired(AutoRepairParams.fromMap(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, row.getFrozenTextMap("repair_preview_repaired"))) + .automatedRepair(AutoRepairParams.fromMap(row.getFrozenTextMap("auto_repair"))) .build(); } diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 2c29394a0147..11264d221f4d 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -18,7 +18,6 @@ package org.apache.cassandra.schema; import java.nio.ByteBuffer; -import java.util.EnumMap; import java.util.Map; import java.util.Map.Entry; @@ -29,7 +28,6 @@ import org.apache.cassandra.cql3.Attributes; import org.apache.cassandra.cql3.CqlBuilder; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.service.reads.PercentileSpeculativeRetryPolicy; import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; @@ -60,9 +58,7 @@ public enum Option CRC_CHECK_CHANCE, CDC, READ_REPAIR, - REPAIR_FULL, - REPAIR_INCREMENTAL, - REPAIR_PREVIEW_REPAIRED, + AUTO_REPAIR, ; @Override @@ -89,8 +85,7 @@ public String toString() public final ImmutableMap extensions; public final boolean cdc; public final ReadRepairStrategy readRepair; - - public final Map automatedRepair; + public final AutoRepairParams autoRepair; private TableParams(Builder builder) { @@ -113,14 +108,7 @@ private TableParams(Builder builder) extensions = builder.extensions; cdc = builder.cdc; readRepair = builder.readRepair; - automatedRepair = new EnumMap(AutoRepairConfig.RepairType.class) - { - { - put(AutoRepairConfig.RepairType.FULL, builder.automatedRepairFull); - put(AutoRepairConfig.RepairType.INCREMENTAL, builder.automatedRepairIncremental); - put(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, builder.automatedRepairPreviewRepaired); - } - }; + autoRepair = builder.autoRepair; } public static Builder builder() @@ -147,9 +135,7 @@ public static Builder builder(TableParams params) .extensions(params.extensions) .cdc(params.cdc) .readRepair(params.readRepair) - .automatedRepairFull(params.automatedRepair.get(AutoRepairConfig.RepairType.FULL)) - .automatedRepairIncremental(params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL)) - .automatedRepairPreviewRepaired(params.automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED)) + .automatedRepair(params.autoRepair) ; } @@ -206,10 +192,7 @@ public void validate() if (cdc && memtable.factory().writesShouldSkipCommitLog()) fail("CDC cannot work if writes skip the commit log. Check your memtable configuration."); - for (Map.Entry entry : automatedRepair.entrySet()) - { - entry.getValue().validate(); - } + autoRepair.validate(); } private static void fail(String format, Object... args) @@ -244,7 +227,7 @@ public boolean equals(Object o) && extensions.equals(p.extensions) && cdc == p.cdc && readRepair == p.readRepair - && automatedRepair.equals(p.automatedRepair); + && autoRepair.equals(p.autoRepair); } @Override @@ -266,7 +249,7 @@ public int hashCode() extensions, cdc, readRepair, - automatedRepair); + autoRepair); } @Override @@ -289,9 +272,7 @@ public String toString() .add(Option.EXTENSIONS.toString(), extensions) .add(Option.CDC.toString(), cdc) .add(Option.READ_REPAIR.toString(), readRepair) - .add(Option.REPAIR_FULL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.FULL)) - .add(Option.REPAIR_INCREMENTAL.toString(), automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL)) - .add(Option.REPAIR_PREVIEW_REPAIRED.toString(), automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED)) + .add(Option.AUTO_REPAIR.toString(), autoRepair) .toString(); } @@ -341,11 +322,7 @@ public void appendCqlTo(CqlBuilder builder, boolean isView) .newLine() .append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString()) .newLine() - .append("AND repair_full = ").append(automatedRepair.get(AutoRepairConfig.RepairType.FULL).asMap()) - .newLine() - .append("AND repair_incremental = ").append(automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).asMap()) - .newLine() - .append("AND repair_preview_repaired = ").append(automatedRepair.get(AutoRepairConfig.RepairType.PREVIEW_REPAIRED).asMap()); + .append("AND auto_repair = ").append(autoRepair.asMap()); } public static final class Builder @@ -368,10 +345,7 @@ public static final class Builder private boolean cdc; private ReadRepairStrategy readRepair = ReadRepairStrategy.BLOCKING; - private AutoRepairParams automatedRepairFull = new AutoRepairParams(AutoRepairConfig.RepairType.FULL); - private AutoRepairParams automatedRepairIncremental = new AutoRepairParams(AutoRepairConfig.RepairType.INCREMENTAL); - private AutoRepairParams automatedRepairPreviewRepaired = new AutoRepairParams(AutoRepairConfig.RepairType.PREVIEW_REPAIRED); - + private AutoRepairParams autoRepair = AutoRepairParams.DEFAULT; public Builder() { } @@ -483,21 +457,9 @@ public Builder extensions(Map val) return this; } - public Builder automatedRepairFull(AutoRepairParams val) - { - automatedRepairFull = val; - return this; - } - - public Builder automatedRepairIncremental(AutoRepairParams val) - { - automatedRepairIncremental = val; - return this; - } - - public Builder automatedRepairPreviewRepaired(AutoRepairParams val) + public Builder automatedRepair(AutoRepairParams val) { - automatedRepairPreviewRepaired = val; + autoRepair = val; return this; } } diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index 86208b5643b1..e64a43eab522 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -1005,9 +1005,7 @@ private static String tableParametersCql() " AND min_index_interval = 128\n" + " AND read_repair = 'BLOCKING'\n" + " AND speculative_retry = '99p'\n" + - " AND repair_full = {'enabled': 'true'}\n" + - " AND repair_incremental = {'enabled': 'true'}\n" + - " AND repair_preview_repaired = {'enabled': 'true'};"; + " AND auto_repair = {'full_enabled': 'true', 'incremental_enabled': 'true', 'preview_repaired_enabled': 'true', 'priority': '0'};"; } private static String mvParametersCql() @@ -1028,9 +1026,7 @@ private static String mvParametersCql() " AND min_index_interval = 128\n" + " AND read_repair = 'BLOCKING'\n" + " AND speculative_retry = '99p'\n" + - " AND repair_full = {'enabled': 'true'}\n" + - " AND repair_incremental = {'enabled': 'true'}\n" + - " AND repair_preview_repaired = {'enabled': 'true'};"; + " AND auto_repair = {'full_enabled': 'true', 'incremental_enabled': 'true', 'preview_repaired_enabled': 'true', 'priority': '0'};"; } private static String keyspaceOutput() diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java index c23cdbc5fc88..3e2be66a4504 100644 --- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java @@ -313,9 +313,7 @@ public void testCfmOptionsCQL() " AND min_index_interval = 6\n" + " AND read_repair = 'BLOCKING'\n" + " AND speculative_retry = 'ALWAYS'\n" + - " AND repair_full = {'enabled': 'true'}\n" + - " AND repair_incremental = {'enabled': 'true'}\n" + - " AND repair_preview_repaired = {'enabled': 'true'};" + " AND auto_repair = {'full_enabled': 'true', 'incremental_enabled': 'true', 'preview_repaired_enabled': 'true', 'priority': '0'};" )); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index a82154e372ed..8a85b645352d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -135,9 +135,10 @@ public static void setupClass() throws Exception .addClusteringColumn("i", IntegerType.instance) .addRegularColumn("v", UTF8Type.instance) .params(TableParams.builder() - .automatedRepairFull(AutoRepairParams.create(AutoRepairConfig.RepairType.FULL, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))) - .automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.INCREMENTAL, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))) - .automatedRepairPreviewRepaired(AutoRepairParams.create(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(true)))) + .automatedRepair(AutoRepairParams.create(ImmutableMap.of(AutoRepairParams.Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(true), + AutoRepairParams.Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(true), + AutoRepairParams.Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(true), + AutoRepairParams.Option.PRIORITY.name().toString(), Integer.toString(0)))) .build()) .build(); @@ -147,9 +148,10 @@ public static void setupClass() throws Exception .addClusteringColumn("i", IntegerType.instance) .addRegularColumn("v", UTF8Type.instance) .params(TableParams.builder() - .automatedRepairFull(AutoRepairParams.create(AutoRepairConfig.RepairType.FULL, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))) - .automatedRepairIncremental(AutoRepairParams.create(AutoRepairConfig.RepairType.INCREMENTAL, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))) - .automatedRepairPreviewRepaired(AutoRepairParams.create(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, ImmutableMap.of(AutoRepairParams.Option.ENABLED.toString(), Boolean.toString(false)))) + .automatedRepair(AutoRepairParams.create(ImmutableMap.of(AutoRepairParams.Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(false), + AutoRepairParams.Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(false), + AutoRepairParams.Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(false), + AutoRepairParams.Option.PRIORITY.name().toString(), Integer.toString(0)))) .build()) .build(); @@ -552,10 +554,10 @@ public void testRepairWaitsForRepairToFinishBeforeSchedullingNewSession() throws @Test public void testDisabledAutoRepairForATableThroughTableLevelConfiguration() { - Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.FULL).repairEnabled()); - Assert.assertTrue(cfm.params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).repairEnabled()); - Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.FULL).repairEnabled()); - Assert.assertFalse(cfmDisabledAutoRepair.params.automatedRepair.get(AutoRepairConfig.RepairType.INCREMENTAL).repairEnabled()); + Assert.assertTrue(cfm.params.autoRepair.repairEnabled(AutoRepairConfig.RepairType.FULL)); + Assert.assertTrue(cfm.params.autoRepair.repairEnabled(AutoRepairConfig.RepairType.INCREMENTAL)); + Assert.assertFalse(cfmDisabledAutoRepair.params.autoRepair.repairEnabled(AutoRepairConfig.RepairType.FULL)); + Assert.assertFalse(cfmDisabledAutoRepair.params.autoRepair.repairEnabled(AutoRepairConfig.RepairType.INCREMENTAL)); AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setRepairMinInterval(repairType, "0s"); @@ -597,9 +599,7 @@ public void testTokenRangesNoSplit() @Test public void testTableAttribute() { - assertTrue(TableAttributes.validKeywords().contains("repair_full")); - assertTrue(TableAttributes.validKeywords().contains("repair_incremental")); - assertTrue(TableAttributes.validKeywords().contains("repair_preview_repaired")); + assertTrue(TableAttributes.validKeywords().contains("auto_repair")); } @Test @@ -608,9 +608,9 @@ public void testDefaultAutomatedRepair() for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) { Assert.assertTrue(String.format("expected repair type %s to be enabled on table %s", repairType, cfm.name), - cfm.params.automatedRepair.get(repairType).repairEnabled()); + cfm.params.autoRepair.repairEnabled(repairType)); Assert.assertFalse(String.format("expected repair type %s to be disabled on table %s", repairType, cfmDisabledAutoRepair.name), - cfmDisabledAutoRepair.params.automatedRepair.get(repairType).repairEnabled()); + cfmDisabledAutoRepair.params.autoRepair.repairEnabled(repairType)); } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java index 011df9d1592e..7b95acb879de 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java @@ -37,9 +37,9 @@ public class PrioritizedRepairPlanTest extends CQLTester public void testBuildWithDifferentPriorities() { // Test reordering assignments with different priorities - String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '3'}"); - String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); + String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '2'}"); + String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '3'}"); + String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '1'}"); List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE, table1, table2, table3); assertEquals(3, prioritizedRepairPlans.size()); @@ -59,9 +59,9 @@ public void testBuildWithDifferentPriorities() public void testBuildWithSamePriority() { // Test reordering assignments with the same priority - String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); + String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '2'}"); + String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '2'}"); + String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '2'}"); // Expect only 1 plan since all tables share the same priority List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE, table1, table2, table3); @@ -82,17 +82,17 @@ public void testBuildWithSamePriority() public void testBuildWithMixedPriorities() { String ks1 = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }"); - String table1 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table2 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '3'}"); - String table3 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '2'}"); - String table4 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); + String table1 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '2'}"); + String table2 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '3'}"); + String table3 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '2'}"); + String table4 = createTable(ks1, "CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '1'}"); // No priority table should be bucketed at priority 0 String table5 = createTable(ks1,"CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); // Create a new keyspace to ensure its tables get grouped with appropriate priority bucket String ks2 = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }"); String table6 = createTable(ks2,"CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); - String table7 = createTable(ks2,"CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '1'}"); + String table7 = createTable(ks2,"CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '1'}"); Map> keyspaceToTableMap = new HashMap<>(); keyspaceToTableMap.put(ks1, Lists.newArrayList(table1, table2, table3, table4, table5)); @@ -149,7 +149,7 @@ public void testBuildWithEmptyTableList() public void testBuildWithOneTable() { // Test with a single element (should remain unchanged) - String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH repair_full = {'enabled': 'true', 'priority': '5'}"); + String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '5'}"); // Expect only 1 plans List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE, table1); From 9e63f4e7ed4df2efe912c297675a9cbc10e4b44d Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 12 Dec 2024 11:36:47 -0800 Subject: [PATCH 089/199] Fix J8 build failures --- .../cassandra/cql3/statements/schema/TableAttributes.java | 1 - .../org/apache/cassandra/repair/autorepair/AutoRepairUtils.java | 2 +- .../apache/cassandra/repair/autorepair/RepairAssignment.java | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index d0c6fba2f0a9..5363db070d88 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -26,7 +26,6 @@ import org.apache.cassandra.cql3.statements.PropertyDefinitions; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.schema.AutoRepairParams; import org.apache.cassandra.schema.CachingParams; import org.apache.cassandra.schema.CompactionParams; diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index 46eb1bcdc1c3..440638c582d7 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -841,7 +841,7 @@ public static Collection> split(Range tokenRange, int number { Collection> ranges; Optional splitter = DatabaseDescriptor.getPartitioner().splitter(); - if (splitter.isEmpty()) + if (!splitter.isPresent()) { NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 30, TimeUnit.MINUTES, "Partitioner {} does not support splitting, falling back to splitting by token range", DatabaseDescriptor.getPartitioner()); ranges = Collections.singleton(tokenRange); diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java index c06d58e71fa2..63f8fbed4426 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Objects; -import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; From f83f90d420f2c2db2d08d69052c1806d2eb84476 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 15 Dec 2024 10:40:15 -0800 Subject: [PATCH 090/199] utest: calculate expected tables dynamically instead of a fixed value --- .../AutoRepairParameterizedTest.java | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 8a85b645352d..b4fb41025808 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -37,6 +37,7 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.locator.LocalStrategy; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.schema.AutoRepairParams; import org.apache.cassandra.schema.SystemDistributedKeyspace; @@ -109,6 +110,9 @@ public class AutoRepairParameterizedTest extends CQLTester RepairRunnable repairRunnable; private static AutoRepairConfig defaultConfig; + // Expected number of tables that should be repaired. + private static int expectedTablesGoingThroughRepair; + @Parameterized.Parameter() public AutoRepairConfig.RepairType repairType; @@ -166,6 +170,22 @@ public static void setupClass() throws Exception DatabaseDescriptor.setMaterializedViewsEnabled(false); DatabaseDescriptor.setCDCEnabled(false); + + // Calculate the expected number of tables to be repaired, this should be all system keyspaces that are + // distributed, plus 1 for the table we created (ks.tbl), excluding the 'mv' materialized view and + // 'tbl_disabled_auto_repair' we created. + expectedTablesGoingThroughRepair = 0; + for (Keyspace keyspace : Keyspace.all()) + { + // skip LocalStrategy keyspaces as these aren't repaired. + if (keyspace.getReplicationStrategy() instanceof LocalStrategy) + { + continue; + } + + int expectedTables = keyspace.getName().equals("ks") ? 1 : keyspace.getColumnFamilyStores().size(); + expectedTablesGoingThroughRepair += expectedTables; + } } @Before @@ -678,15 +698,10 @@ public void testRepairMaxRetries() AutoRepair.instance.repair(repairType); - //system_auth.role_permissions,system_auth.network_permissions,system_auth.role_members,system_auth.roles, - // system_auth.resource_role_permissons_index,system_traces.sessions,system_traces.events,ks.tbl, - // system_distributed.auto_repair_priority,system_distributed.repair_history,system_distributed.auto_repair_history, - // system_distributed.view_build_status,system_distributed.parent_repair_history,system_distributed.partition_denylist - int exptedTablesGoingThroughRepair = 14; - assertEquals(config.getRepairMaxRetries()*exptedTablesGoingThroughRepair, sleepCalls.get()); + assertEquals(config.getRepairMaxRetries()*expectedTablesGoingThroughRepair, sleepCalls.get()); verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); - verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(exptedTablesGoingThroughRepair); + verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(expectedTablesGoingThroughRepair); } @Test @@ -712,7 +727,7 @@ public void testRepairSuccessAfterRetry() AutoRepair.instance.repair(repairType); assertEquals(1, sleepCalls.get()); - verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(14); + verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); } From 1b1601a46d83f9e6fd99ba5c6e151ddcc8dfd1a5 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 3 Dec 2024 15:40:11 -0800 Subject: [PATCH 091/199] Disable FR and IR by default on system_traces.* tables --- .../statements/schema/TableAttributes.java | 2 +- .../cassandra/schema/AutoRepairParams.java | 7 ++++ .../cassandra/schema/SchemaKeyspace.java | 2 +- .../cassandra/schema/TableMetadata.java | 5 +++ .../apache/cassandra/schema/TableParams.java | 4 +- .../cassandra/tracing/TraceKeyspace.java | 5 ++- .../autorepair/AutoRepairConfigTest.java | 41 +++++++++++++++++++ .../AutoRepairParameterizedTest.java | 38 +++++++++++------ 8 files changed, 87 insertions(+), 17 deletions(-) diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index 5363db070d88..723cbbdd51f3 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -157,7 +157,7 @@ private TableParams build(TableParams.Builder builder) builder.readRepair(ReadRepairStrategy.fromString(getString(Option.READ_REPAIR))); if (hasOption(Option.AUTO_REPAIR)) - builder.automatedRepair(AutoRepairParams.fromMap(getMap(Option.AUTO_REPAIR))); + builder.autoRepair(AutoRepairParams.fromMap(getMap(Option.AUTO_REPAIR))); return builder.build(); } diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index b012a48188d8..66fad8135d75 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -56,6 +56,13 @@ public String toString() Option.PRIORITY.toString(), "0" ); + public static final Map DISABLED_OPTIONS = ImmutableMap.of( + Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(false), + Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(false), + Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(false), + Option.PRIORITY.toString(), "0" + ); + AutoRepairParams(Map options) { this.options = ImmutableMap.copyOf(options); diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index 8c45e74f1ccb..c744dee54201 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -980,7 +980,7 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row) SpeculativeRetryPolicy.fromString("99PERCENTILE")) .cdc(row.has("cdc") && row.getBoolean("cdc")) .readRepair(getReadRepairStrategy(row)) - .automatedRepair(AutoRepairParams.fromMap(row.getFrozenTextMap("auto_repair"))) + .autoRepair(AutoRepairParams.fromMap(row.getFrozenTextMap("auto_repair"))) .build(); } diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index 2e9d507ba9d9..2ffccd5d5e1a 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -873,6 +873,11 @@ public Builder memtable(MemtableParams val) return this; } + public Builder autoRepair(AutoRepairParams val) + { + params.autoRepair(val); + return this; + } public Builder isCounter(boolean val) { diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 11264d221f4d..bec02b5ce318 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -135,7 +135,7 @@ public static Builder builder(TableParams params) .extensions(params.extensions) .cdc(params.cdc) .readRepair(params.readRepair) - .automatedRepair(params.autoRepair) + .autoRepair(params.autoRepair) ; } @@ -457,7 +457,7 @@ public Builder extensions(Map val) return this; } - public Builder automatedRepair(AutoRepairParams val) + public Builder autoRepair(AutoRepairParams val) { autoRepair = val; return this; diff --git a/src/java/org/apache/cassandra/tracing/TraceKeyspace.java b/src/java/org/apache/cassandra/tracing/TraceKeyspace.java index 78ccc3273a1c..3a73afc7d300 100644 --- a/src/java/org/apache/cassandra/tracing/TraceKeyspace.java +++ b/src/java/org/apache/cassandra/tracing/TraceKeyspace.java @@ -28,6 +28,7 @@ import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.schema.AutoRepairParams; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.SchemaConstants; @@ -57,12 +58,13 @@ private TraceKeyspace() * removed default ttl, reduced bloom filter fp chance from 0.1 to 0.01. * gen 1577836800000001: (pre-)adds coordinator_port column to sessions and source_port column to events in 3.0, 3.11, 4.0 * gen 1577836800000002: compression chunk length reduced to 16KiB, memtable_flush_period_in_ms now unset on all tables in 4.0 + * gen 1577836800000003: disable auto-repair CQL table properties * * * Until CASSANDRA-6016 (Oct 13, 2.0.2) and in all of 1.2, we used to create system_traces keyspace and * tables in the same way that we created the purely local 'system' keyspace - using current time on node bounce * (+1). For new definitions to take, we need to bump the generation further than that. */ - public static final long GENERATION = 1577836800000002L; + public static final long GENERATION = 1577836800000003L; public static final String SESSIONS = "sessions"; public static final String EVENTS = "events"; @@ -101,6 +103,7 @@ private static TableMetadata parse(String table, String description, String cql) .id(TableId.forSystemTable(SchemaConstants.TRACE_KEYSPACE_NAME, table)) .gcGraceSeconds(0) .comment(description) + .autoRepair(AutoRepairParams.fromMap(AutoRepairParams.DISABLED_OPTIONS)) .build(); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 08ae9631a192..3274c20d0089 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -18,6 +18,10 @@ package org.apache.cassandra.repair.autorepair; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Collections; import java.util.Set; @@ -33,8 +37,10 @@ import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.Options; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; @@ -63,6 +69,7 @@ public void setUp() { config = new AutoRepairConfig(true); AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); + requireNetwork(); } @Test @@ -420,4 +427,38 @@ public void testSetRepairSessionTimeout() assert config.getOptions(repairType).repair_session_timeout.toSeconds() == 3600; } + @Test + public void testDefaultSystemTablesBehavior() + { + Map> ksTablesRepairDefaultDisabled = new HashMap<>(); + ksTablesRepairDefaultDisabled.put("system_traces", Arrays.asList("sessions", "events")); + + Map> ksTablesRepairDefaultEnabled = new HashMap<>(); + ksTablesRepairDefaultEnabled.put("system_auth", Arrays.asList("role_permissions", "network_permissions", "role_members", "roles", "resource_role_permissons_index")); + ksTablesRepairDefaultEnabled.put("system_distributed", Arrays.asList("auto_repair_priority", "repair_history", "auto_repair_history", "view_build_status", "parent_repair_history", "partition_denylist")); + + for (Map.Entry> disabled : ksTablesRepairDefaultDisabled.entrySet()) + { + for (String table : disabled.getValue()) + { + TableMetadata sessionCfm = Keyspace.open(disabled.getKey()).getColumnFamilyStore(table).metadata(); + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + assertFalse(disabled.getKey()+"."+table, sessionCfm.params.autoRepair.repairEnabled(repairType)); + } + } + } + + for (Map.Entry> enabled : ksTablesRepairDefaultEnabled.entrySet()) + { + for (String table : enabled.getValue()) + { + TableMetadata sessionCfm = Keyspace.open(enabled.getKey()).getColumnFamilyStore(table).metadata(); + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + assertTrue(enabled.getKey()+"."+table, sessionCfm.params.autoRepair.repairEnabled(repairType)); + } + } + } + } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index b4fb41025808..ce50ef89a4be 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -113,6 +113,9 @@ public class AutoRepairParameterizedTest extends CQLTester // Expected number of tables that should be repaired. private static int expectedTablesGoingThroughRepair; + // Expected number of tables that should be skipped. + private static int expectedTablesSkipped; + @Parameterized.Parameter() public AutoRepairConfig.RepairType repairType; @@ -139,10 +142,10 @@ public static void setupClass() throws Exception .addClusteringColumn("i", IntegerType.instance) .addRegularColumn("v", UTF8Type.instance) .params(TableParams.builder() - .automatedRepair(AutoRepairParams.create(ImmutableMap.of(AutoRepairParams.Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(true), - AutoRepairParams.Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(true), - AutoRepairParams.Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(true), - AutoRepairParams.Option.PRIORITY.name().toString(), Integer.toString(0)))) + .autoRepair(AutoRepairParams.create(ImmutableMap.of(AutoRepairParams.Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(true), + AutoRepairParams.Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(true), + AutoRepairParams.Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(true), + AutoRepairParams.Option.PRIORITY.name().toString(), Integer.toString(0)))) .build()) .build(); @@ -152,10 +155,10 @@ public static void setupClass() throws Exception .addClusteringColumn("i", IntegerType.instance) .addRegularColumn("v", UTF8Type.instance) .params(TableParams.builder() - .automatedRepair(AutoRepairParams.create(ImmutableMap.of(AutoRepairParams.Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(false), - AutoRepairParams.Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(false), - AutoRepairParams.Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(false), - AutoRepairParams.Option.PRIORITY.name().toString(), Integer.toString(0)))) + .autoRepair(AutoRepairParams.create(ImmutableMap.of(AutoRepairParams.Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(false), + AutoRepairParams.Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(false), + AutoRepairParams.Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(false), + AutoRepairParams.Option.PRIORITY.name().toString(), Integer.toString(0)))) .build()) .build(); @@ -175,6 +178,7 @@ public static void setupClass() throws Exception // distributed, plus 1 for the table we created (ks.tbl), excluding the 'mv' materialized view and // 'tbl_disabled_auto_repair' we created. expectedTablesGoingThroughRepair = 0; + expectedTablesSkipped = 0; for (Keyspace keyspace : Keyspace.all()) { // skip LocalStrategy keyspaces as these aren't repaired. @@ -182,6 +186,12 @@ public static void setupClass() throws Exception { continue; } + // skip system_traces keyspaces + if (keyspace.getName().equals(SchemaConstants.TRACE_KEYSPACE_NAME)) + { + expectedTablesSkipped += keyspace.getColumnFamilyStores().size(); + continue; + } int expectedTables = keyspace.getName().equals("ks") ? 1 : keyspace.getColumnFamilyStores().size(); expectedTablesGoingThroughRepair += expectedTables; @@ -475,8 +485,11 @@ public void testSkipRepairSSTableCountHigherThreshold() // skipping both the tables - one table is due to its repair has been disabled, and another one due to high sstable count assertEquals(0, state.getSkippedTokenRangesCount()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); - assertEquals(2, state.getSkippedTablesCount()); - assertEquals(2, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + int skippedDutoToDisabled = 1; + int skippedDueToHighSSTables = 1; + int totalExpectedSkippedCount = expectedTablesSkipped + skippedDutoToDisabled + skippedDueToHighSSTables; + assertEquals(totalExpectedSkippedCount, state.getSkippedTablesCount()); + assertEquals(totalExpectedSkippedCount, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); // set it to higher value, and this time, the tables should not be skipped config.setRepairSSTableCountHigherThreshold(repairType, beforeCount); @@ -488,8 +501,9 @@ public void testSkipRepairSSTableCountHigherThreshold() assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); assertEquals(0, state.getSkippedTokenRangesCount()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); - assertEquals(1, state.getSkippedTablesCount()); - assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + totalExpectedSkippedCount = expectedTablesSkipped + skippedDutoToDisabled; + assertEquals(totalExpectedSkippedCount, state.getSkippedTablesCount()); + assertEquals(totalExpectedSkippedCount, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); } @Test From f43577aee0f9d6a09ff8607d3b1d6220d5083497 Mon Sep 17 00:00:00 2001 From: Jaydeepkumar Chovatia Date: Tue, 17 Dec 2024 20:01:36 -0800 Subject: [PATCH 092/199] Revert "Disable FR and IR by default on system_traces.* tables" This reverts commit 1b1601a46d83f9e6fd99ba5c6e151ddcc8dfd1a5. --- .../statements/schema/TableAttributes.java | 2 +- .../cassandra/schema/AutoRepairParams.java | 7 ---- .../cassandra/schema/SchemaKeyspace.java | 2 +- .../cassandra/schema/TableMetadata.java | 5 --- .../apache/cassandra/schema/TableParams.java | 4 +- .../cassandra/tracing/TraceKeyspace.java | 5 +-- .../autorepair/AutoRepairConfigTest.java | 41 ------------------- .../AutoRepairParameterizedTest.java | 38 ++++++----------- 8 files changed, 17 insertions(+), 87 deletions(-) diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index 723cbbdd51f3..5363db070d88 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -157,7 +157,7 @@ private TableParams build(TableParams.Builder builder) builder.readRepair(ReadRepairStrategy.fromString(getString(Option.READ_REPAIR))); if (hasOption(Option.AUTO_REPAIR)) - builder.autoRepair(AutoRepairParams.fromMap(getMap(Option.AUTO_REPAIR))); + builder.automatedRepair(AutoRepairParams.fromMap(getMap(Option.AUTO_REPAIR))); return builder.build(); } diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index 66fad8135d75..b012a48188d8 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -56,13 +56,6 @@ public String toString() Option.PRIORITY.toString(), "0" ); - public static final Map DISABLED_OPTIONS = ImmutableMap.of( - Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(false), - Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(false), - Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(false), - Option.PRIORITY.toString(), "0" - ); - AutoRepairParams(Map options) { this.options = ImmutableMap.copyOf(options); diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index c744dee54201..8c45e74f1ccb 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -980,7 +980,7 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row) SpeculativeRetryPolicy.fromString("99PERCENTILE")) .cdc(row.has("cdc") && row.getBoolean("cdc")) .readRepair(getReadRepairStrategy(row)) - .autoRepair(AutoRepairParams.fromMap(row.getFrozenTextMap("auto_repair"))) + .automatedRepair(AutoRepairParams.fromMap(row.getFrozenTextMap("auto_repair"))) .build(); } diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index 2ffccd5d5e1a..2e9d507ba9d9 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -873,11 +873,6 @@ public Builder memtable(MemtableParams val) return this; } - public Builder autoRepair(AutoRepairParams val) - { - params.autoRepair(val); - return this; - } public Builder isCounter(boolean val) { diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index bec02b5ce318..11264d221f4d 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -135,7 +135,7 @@ public static Builder builder(TableParams params) .extensions(params.extensions) .cdc(params.cdc) .readRepair(params.readRepair) - .autoRepair(params.autoRepair) + .automatedRepair(params.autoRepair) ; } @@ -457,7 +457,7 @@ public Builder extensions(Map val) return this; } - public Builder autoRepair(AutoRepairParams val) + public Builder automatedRepair(AutoRepairParams val) { autoRepair = val; return this; diff --git a/src/java/org/apache/cassandra/tracing/TraceKeyspace.java b/src/java/org/apache/cassandra/tracing/TraceKeyspace.java index 3a73afc7d300..78ccc3273a1c 100644 --- a/src/java/org/apache/cassandra/tracing/TraceKeyspace.java +++ b/src/java/org/apache/cassandra/tracing/TraceKeyspace.java @@ -28,7 +28,6 @@ import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.gms.Gossiper; -import org.apache.cassandra.schema.AutoRepairParams; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.SchemaConstants; @@ -58,13 +57,12 @@ private TraceKeyspace() * removed default ttl, reduced bloom filter fp chance from 0.1 to 0.01. * gen 1577836800000001: (pre-)adds coordinator_port column to sessions and source_port column to events in 3.0, 3.11, 4.0 * gen 1577836800000002: compression chunk length reduced to 16KiB, memtable_flush_period_in_ms now unset on all tables in 4.0 - * gen 1577836800000003: disable auto-repair CQL table properties * * * Until CASSANDRA-6016 (Oct 13, 2.0.2) and in all of 1.2, we used to create system_traces keyspace and * tables in the same way that we created the purely local 'system' keyspace - using current time on node bounce * (+1). For new definitions to take, we need to bump the generation further than that. */ - public static final long GENERATION = 1577836800000003L; + public static final long GENERATION = 1577836800000002L; public static final String SESSIONS = "sessions"; public static final String EVENTS = "events"; @@ -103,7 +101,6 @@ private static TableMetadata parse(String table, String description, String cql) .id(TableId.forSystemTable(SchemaConstants.TRACE_KEYSPACE_NAME, table)) .gcGraceSeconds(0) .comment(description) - .autoRepair(AutoRepairParams.fromMap(AutoRepairParams.DISABLED_OPTIONS)) .build(); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 3274c20d0089..08ae9631a192 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -18,10 +18,6 @@ package org.apache.cassandra.repair.autorepair; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Collections; import java.util.Set; @@ -37,10 +33,8 @@ import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.cql3.CQLTester; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.Options; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; @@ -69,7 +63,6 @@ public void setUp() { config = new AutoRepairConfig(true); AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); - requireNetwork(); } @Test @@ -427,38 +420,4 @@ public void testSetRepairSessionTimeout() assert config.getOptions(repairType).repair_session_timeout.toSeconds() == 3600; } - @Test - public void testDefaultSystemTablesBehavior() - { - Map> ksTablesRepairDefaultDisabled = new HashMap<>(); - ksTablesRepairDefaultDisabled.put("system_traces", Arrays.asList("sessions", "events")); - - Map> ksTablesRepairDefaultEnabled = new HashMap<>(); - ksTablesRepairDefaultEnabled.put("system_auth", Arrays.asList("role_permissions", "network_permissions", "role_members", "roles", "resource_role_permissons_index")); - ksTablesRepairDefaultEnabled.put("system_distributed", Arrays.asList("auto_repair_priority", "repair_history", "auto_repair_history", "view_build_status", "parent_repair_history", "partition_denylist")); - - for (Map.Entry> disabled : ksTablesRepairDefaultDisabled.entrySet()) - { - for (String table : disabled.getValue()) - { - TableMetadata sessionCfm = Keyspace.open(disabled.getKey()).getColumnFamilyStore(table).metadata(); - for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) - { - assertFalse(disabled.getKey()+"."+table, sessionCfm.params.autoRepair.repairEnabled(repairType)); - } - } - } - - for (Map.Entry> enabled : ksTablesRepairDefaultEnabled.entrySet()) - { - for (String table : enabled.getValue()) - { - TableMetadata sessionCfm = Keyspace.open(enabled.getKey()).getColumnFamilyStore(table).metadata(); - for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) - { - assertTrue(enabled.getKey()+"."+table, sessionCfm.params.autoRepair.repairEnabled(repairType)); - } - } - } - } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index ce50ef89a4be..b4fb41025808 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -113,9 +113,6 @@ public class AutoRepairParameterizedTest extends CQLTester // Expected number of tables that should be repaired. private static int expectedTablesGoingThroughRepair; - // Expected number of tables that should be skipped. - private static int expectedTablesSkipped; - @Parameterized.Parameter() public AutoRepairConfig.RepairType repairType; @@ -142,10 +139,10 @@ public static void setupClass() throws Exception .addClusteringColumn("i", IntegerType.instance) .addRegularColumn("v", UTF8Type.instance) .params(TableParams.builder() - .autoRepair(AutoRepairParams.create(ImmutableMap.of(AutoRepairParams.Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(true), - AutoRepairParams.Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(true), - AutoRepairParams.Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(true), - AutoRepairParams.Option.PRIORITY.name().toString(), Integer.toString(0)))) + .automatedRepair(AutoRepairParams.create(ImmutableMap.of(AutoRepairParams.Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(true), + AutoRepairParams.Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(true), + AutoRepairParams.Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(true), + AutoRepairParams.Option.PRIORITY.name().toString(), Integer.toString(0)))) .build()) .build(); @@ -155,10 +152,10 @@ public static void setupClass() throws Exception .addClusteringColumn("i", IntegerType.instance) .addRegularColumn("v", UTF8Type.instance) .params(TableParams.builder() - .autoRepair(AutoRepairParams.create(ImmutableMap.of(AutoRepairParams.Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(false), - AutoRepairParams.Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(false), - AutoRepairParams.Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(false), - AutoRepairParams.Option.PRIORITY.name().toString(), Integer.toString(0)))) + .automatedRepair(AutoRepairParams.create(ImmutableMap.of(AutoRepairParams.Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(false), + AutoRepairParams.Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(false), + AutoRepairParams.Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(false), + AutoRepairParams.Option.PRIORITY.name().toString(), Integer.toString(0)))) .build()) .build(); @@ -178,7 +175,6 @@ public static void setupClass() throws Exception // distributed, plus 1 for the table we created (ks.tbl), excluding the 'mv' materialized view and // 'tbl_disabled_auto_repair' we created. expectedTablesGoingThroughRepair = 0; - expectedTablesSkipped = 0; for (Keyspace keyspace : Keyspace.all()) { // skip LocalStrategy keyspaces as these aren't repaired. @@ -186,12 +182,6 @@ public static void setupClass() throws Exception { continue; } - // skip system_traces keyspaces - if (keyspace.getName().equals(SchemaConstants.TRACE_KEYSPACE_NAME)) - { - expectedTablesSkipped += keyspace.getColumnFamilyStores().size(); - continue; - } int expectedTables = keyspace.getName().equals("ks") ? 1 : keyspace.getColumnFamilyStores().size(); expectedTablesGoingThroughRepair += expectedTables; @@ -485,11 +475,8 @@ public void testSkipRepairSSTableCountHigherThreshold() // skipping both the tables - one table is due to its repair has been disabled, and another one due to high sstable count assertEquals(0, state.getSkippedTokenRangesCount()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); - int skippedDutoToDisabled = 1; - int skippedDueToHighSSTables = 1; - int totalExpectedSkippedCount = expectedTablesSkipped + skippedDutoToDisabled + skippedDueToHighSSTables; - assertEquals(totalExpectedSkippedCount, state.getSkippedTablesCount()); - assertEquals(totalExpectedSkippedCount, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + assertEquals(2, state.getSkippedTablesCount()); + assertEquals(2, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); // set it to higher value, and this time, the tables should not be skipped config.setRepairSSTableCountHigherThreshold(repairType, beforeCount); @@ -501,9 +488,8 @@ public void testSkipRepairSSTableCountHigherThreshold() assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); assertEquals(0, state.getSkippedTokenRangesCount()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); - totalExpectedSkippedCount = expectedTablesSkipped + skippedDutoToDisabled; - assertEquals(totalExpectedSkippedCount, state.getSkippedTablesCount()); - assertEquals(totalExpectedSkippedCount, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); + assertEquals(1, state.getSkippedTablesCount()); + assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).skippedTablesCount.getValue().intValue()); } @Test From f06b8c41bc222c9adb8f9d11577113c7c31b430c Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 17 Dec 2024 20:16:54 -0800 Subject: [PATCH 093/199] Ignore repair for system_traces keyspace --- .../cassandra/repair/autorepair/AutoRepair.java | 2 +- .../repair/autorepair/AutoRepairUtils.java | 17 ++++++++++++----- .../autorepair/AutoRepairParameterizedTest.java | 6 +++++- .../repair/autorepair/AutoRepairTest.java | 4 ++-- .../repair/autorepair/AutoRepairUtilsTest.java | 12 +++++++++--- 5 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 9050885abf2b..04910e0ce09f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -216,7 +216,7 @@ public void repair(AutoRepairConfig.RepairType repairType) Map> keyspacesAndTablesToRepair = new LinkedHashMap<>(); for (Keyspace keyspace : keyspaces) { - if (!AutoRepairUtils.checkNodeContainsKeyspaceReplica(keyspace)) + if (!AutoRepairUtils.shouldConsiderKeyspace(keyspace)) { continue; } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index 440638c582d7..31d48173582f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -67,6 +67,7 @@ import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.tracing.TraceKeyspace; import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.transport.messages.ResultMessage; @@ -785,24 +786,30 @@ public static Set getPriorityHosts(RepairType repairType) return hosts; } - public static boolean checkNodeContainsKeyspaceReplica(Keyspace ks) + public static boolean shouldConsiderKeyspace(Keyspace ks) { AbstractReplicationStrategy replicationStrategy = ks.getReplicationStrategy(); - boolean ksReplicaOnNode = true; + boolean repair = true; if (replicationStrategy instanceof NetworkTopologyStrategy) { Set datacenters = ((NetworkTopologyStrategy) replicationStrategy).getDatacenters(); String localDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort()); if (!datacenters.contains(localDC)) { - ksReplicaOnNode = false; + repair = false; } } if (replicationStrategy instanceof LocalStrategy) { - ksReplicaOnNode = false; + repair = false; } - return ksReplicaOnNode; + if (ks.getName().equalsIgnoreCase(SchemaConstants.TRACE_KEYSPACE_NAME)) + { + // by default, ignore the tables under system_traces as they do not have + // that much important data + repair = false; + } + return repair; } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index b4fb41025808..fcea50e9bd6b 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -182,7 +182,11 @@ public static void setupClass() throws Exception { continue; } - + // skip system_traces keyspaces + if (keyspace.getName().equalsIgnoreCase(SchemaConstants.TRACE_KEYSPACE_NAME)) + { + continue; + } int expectedTables = keyspace.getName().equals("ks") ? 1 : keyspace.getColumnFamilyStores().size(); expectedTablesGoingThroughRepair += expectedTables; } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index 4ae69fa6395a..a7d5eb2f9e48 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -137,14 +137,14 @@ public void testCheckNTSreplicationNodeInsideOutsideDC() // case 1 : // node reside in "datacenter1" // keyspace has replica in "datacenter1" - Assert.assertTrue(AutoRepairUtils.checkNodeContainsKeyspaceReplica(ks)); + Assert.assertTrue(AutoRepairUtils.shouldConsiderKeyspace(ks)); } else if (ks.getName().equals(ksname2)) { // case 2 : // node reside in "datacenter1" // keyspace has replica in "datacenter2" - Assert.assertFalse(AutoRepairUtils.checkNodeContainsKeyspaceReplica(ks)); + Assert.assertFalse(AutoRepairUtils.shouldConsiderKeyspace(ks)); } } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index 38110e903c08..d42f8b015491 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -430,7 +430,7 @@ public void testCheckNodeContainsKeyspaceReplica() { Keyspace ks = Keyspace.open("ks"); - assertTrue(AutoRepairUtils.checkNodeContainsKeyspaceReplica(ks)); + assertTrue(AutoRepairUtils.shouldConsiderKeyspace(ks)); } @Test @@ -479,8 +479,8 @@ public void testMyTurnToRunRepairShouldReturnMyTurnWhenRepairOngoing() @Test public void testLocalStrategyAndNetworkKeyspace() { - assertFalse(AutoRepairUtils.checkNodeContainsKeyspaceReplica(Keyspace.open("system"))); - assertTrue(AutoRepairUtils.checkNodeContainsKeyspaceReplica(Keyspace.open(KEYSPACE))); + assertFalse(AutoRepairUtils.shouldConsiderKeyspace(Keyspace.open("system"))); + assertTrue(AutoRepairUtils.shouldConsiderKeyspace(Keyspace.open(KEYSPACE))); } @Test @@ -502,4 +502,10 @@ public void testGetLastRepairTimeForNodeWhenHistoryIsEmpty() assertEquals(0, AutoRepairUtils.getLastRepairTimeForNode(repairType, myID)); } + + @Test + public void tesSkipSystemTraces() + { + assertFalse(AutoRepairUtils.shouldConsiderKeyspace(Keyspace.open(SchemaConstants.TRACE_KEYSPACE_NAME))); + } } From e3f0b4d2b5ba808ec5826e73280615574f0c7227 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 19 Dec 2024 09:54:13 -0800 Subject: [PATCH 094/199] cr from Andy T --- .../org/apache/cassandra/repair/autorepair/AutoRepairUtils.java | 1 - .../apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index 31d48173582f..f5678d1e88fe 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -67,7 +67,6 @@ import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.tracing.TraceKeyspace; import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.transport.messages.ResultMessage; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index d42f8b015491..9fd588b2ce79 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -504,7 +504,7 @@ public void testGetLastRepairTimeForNodeWhenHistoryIsEmpty() } @Test - public void tesSkipSystemTraces() + public void testSkipSystemTraces() { assertFalse(AutoRepairUtils.shouldConsiderKeyspace(Keyspace.open(SchemaConstants.TRACE_KEYSPACE_NAME))); } From b45c5b5ef1f6087c164589b396260d7d231611f8 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 18 Dec 2024 20:35:17 -0800 Subject: [PATCH 095/199] Fix diff40 --- test/data/jmxdump/cassandra-4.0-jmx.yaml | 768 ++++++++++++++++++++++- 1 file changed, 740 insertions(+), 28 deletions(-) diff --git a/test/data/jmxdump/cassandra-4.0-jmx.yaml b/test/data/jmxdump/cassandra-4.0-jmx.yaml index c69487909d8b..d99a183c67d4 100644 --- a/test/data/jmxdump/cassandra-4.0-jmx.yaml +++ b/test/data/jmxdump/cassandra-4.0-jmx.yaml @@ -10190,6 +10190,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,n org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -10204,6 +10209,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,n org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -11095,6 +11105,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -11109,6 +11124,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12000,6 +12020,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12014,6 +12039,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12905,6 +12935,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12919,6 +12954,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -13810,6 +13850,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -13824,6 +13869,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -14715,6 +14765,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -14729,6 +14784,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -15620,6 +15680,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -15634,6 +15699,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -16525,6 +16595,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -16539,6 +16614,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -17430,6 +17510,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -17444,6 +17529,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -18335,6 +18425,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -18349,6 +18444,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -19240,6 +19340,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -19254,6 +19359,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -20145,6 +20255,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,na org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -20159,6 +20274,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,na org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21050,6 +21170,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_st org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_statements,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21064,6 +21189,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_st org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_statements,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21955,6 +22085,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21969,6 +22104,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -22860,6 +23000,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estima org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -22874,6 +23019,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estima org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23765,6 +23915,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_act org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_activity,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23779,6 +23934,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_act org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_activity,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -24670,6 +24830,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estim org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -24684,6 +24849,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estim org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -25575,6 +25745,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -25589,6 +25764,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -26480,6 +26660,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -26494,6 +26679,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -27385,6 +27575,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=view_builds org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=view_builds_in_progress,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -28290,6 +28485,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=networ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=network_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -30100,6 +30300,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_m org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_members,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -30114,6 +30319,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_m org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_members,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31005,6 +31215,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_p org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31019,6 +31234,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_p org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31910,6 +32130,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles, org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31924,6 +32149,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles, org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -33720,6 +33950,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope=repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -34625,6 +34860,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope=view_build_status,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -35530,6 +35770,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggr org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggregates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -35544,6 +35789,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggr org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggregates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -36435,6 +36685,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=colu org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -36449,6 +36704,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=colu org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -37340,6 +37600,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=drop org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=dropped_columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -38245,6 +38510,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=func org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=functions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -38259,6 +38529,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=func org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=functions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -39150,6 +39425,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=inde org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=indexes,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -39164,6 +39444,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=inde org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=indexes,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40055,6 +40340,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keys org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keyspaces,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40069,6 +40359,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keys org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keyspaces,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40960,6 +41255,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tabl org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tables,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40974,6 +41274,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tabl org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tables,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -41865,6 +42170,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=trig org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=triggers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -41879,6 +42189,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=trig org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=triggers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -42770,6 +43085,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=type org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=types,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -42784,6 +43104,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=type org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=types,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -43675,6 +44000,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=view org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -43689,6 +44019,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=view org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44580,6 +44915,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=even org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44594,6 +44934,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=even org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -45485,6 +45830,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sess org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sessions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -45499,6 +45849,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sess org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sessions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -46387,13 +46742,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BloomFilterOffHeapMemoryUsed - name: objectName parameters: [] returnType: javax.management.ObjectName -org.apache.cassandra.metrics:type=ColumnFamily,name=BytesAnticompacted: - attributes: - - {access: read-only, name: Value, type: java.lang.Object} - operations: - - name: objectName - parameters: [] - returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesFlushed: attributes: - {access: read-only, name: Value, type: java.lang.Object} @@ -46401,13 +46749,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BytesFlushed: - name: objectName parameters: [] returnType: javax.management.ObjectName -org.apache.cassandra.metrics:type=ColumnFamily,name=BytesMutatedAnticompaction: - attributes: - - {access: read-only, name: Value, type: java.lang.Object} - operations: - - name: objectName - parameters: [] - returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesPendingRepair: attributes: - {access: read-only, name: Value, type: java.lang.Object} @@ -58279,6 +58620,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=Blo org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -58293,6 +58639,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=Byt org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -59388,6 +59739,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -59402,6 +59758,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -60497,6 +60858,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -60511,6 +60877,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -61606,6 +61977,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=Bloom org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -61620,6 +61996,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=Bytes org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -62715,6 +63096,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -62729,6 +63115,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -63824,6 +64215,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -63838,6 +64234,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -64933,6 +65334,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -64947,6 +65353,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -66042,6 +66453,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -66056,6 +66472,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -67151,6 +67572,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -67165,6 +67591,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -68260,6 +68691,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -68274,6 +68710,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -69369,6 +69810,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -69383,6 +69829,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -70478,6 +70929,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=Bloo org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -70492,6 +70948,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=Byte org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -71587,6 +72048,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statement org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statements,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -71601,6 +72067,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statement org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statements,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -72696,6 +73167,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=Bloom org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -72710,6 +73186,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=Bytes org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -73805,6 +74286,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -73819,6 +74305,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -74914,6 +75405,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -74928,6 +75424,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -76023,6 +76524,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,na org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -76037,6 +76543,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,na org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -77132,6 +77643,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -77146,6 +77662,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -78241,6 +78762,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -78255,6 +78781,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -79350,6 +79881,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_pro org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_progress,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -79364,6 +79900,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_pro org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_progress,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -80459,6 +81000,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -80473,6 +81019,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -81568,6 +82119,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=resource_role org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=resource_role_permissons_index,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -82677,6 +83233,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members, org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -82691,6 +83252,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members, org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -83786,6 +84352,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -83800,6 +84371,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -84895,6 +85471,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=Bl org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -84909,6 +85490,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=By org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -86004,6 +86590,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=parent org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=parent_repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -87113,6 +87704,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -87127,6 +87723,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -88222,6 +88823,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=view_b org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=view_build_status,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -89331,6 +89937,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates, org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -89345,6 +89956,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates, org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -90440,6 +91056,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -90454,6 +91075,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -91549,6 +92175,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_col org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -91563,6 +92194,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_col org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -92658,6 +93294,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -92672,6 +93313,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -93767,6 +94413,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -93781,6 +94432,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -94876,6 +95532,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -94890,6 +95551,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -95985,6 +96651,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -95999,6 +96670,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -97094,6 +97770,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,na org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -97108,6 +97789,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,na org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -98203,6 +98889,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -98217,6 +98908,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -99312,6 +100008,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -99326,6 +100027,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -100421,6 +101127,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -100435,6 +101146,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -101530,6 +102246,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,na org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -101544,6 +102265,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,na org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -102636,13 +103362,6 @@ org.apache.cassandra.metrics:type=Table,name=BloomFilterOffHeapMemoryUsed: - name: objectName parameters: [] returnType: javax.management.ObjectName -org.apache.cassandra.metrics:type=Table,name=BytesAnticompacted: - attributes: - - {access: read-only, name: Value, type: java.lang.Object} - operations: - - name: objectName - parameters: [] - returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesFlushed: attributes: - {access: read-only, name: Value, type: java.lang.Object} @@ -102650,13 +103369,6 @@ org.apache.cassandra.metrics:type=Table,name=BytesFlushed: - name: objectName parameters: [] returnType: javax.management.ObjectName -org.apache.cassandra.metrics:type=Table,name=BytesMutatedAnticompaction: - attributes: - - {access: read-only, name: Value, type: java.lang.Object} - operations: - - name: objectName - parameters: [] - returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesPendingRepair: attributes: - {access: read-only, name: Value, type: java.lang.Object} From fe0b79a651d7eaae9ea6d7a365852c1a331a9085 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 18 Dec 2024 20:44:57 -0800 Subject: [PATCH 096/199] Fix diff41 --- test/data/jmxdump/cassandra-4.1-jmx.yaml | 768 ++++++++++++++++++++++- 1 file changed, 740 insertions(+), 28 deletions(-) diff --git a/test/data/jmxdump/cassandra-4.1-jmx.yaml b/test/data/jmxdump/cassandra-4.1-jmx.yaml index 14e204430b92..b2322f2f43bd 100644 --- a/test/data/jmxdump/cassandra-4.1-jmx.yaml +++ b/test/data/jmxdump/cassandra-4.1-jmx.yaml @@ -10190,6 +10190,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,n org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -10204,6 +10209,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,n org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -11095,6 +11105,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -11109,6 +11124,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12000,6 +12020,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12014,6 +12039,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12905,6 +12935,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12919,6 +12954,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -13810,6 +13850,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -13824,6 +13869,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -14715,6 +14765,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -14729,6 +14784,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -15620,6 +15680,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -15634,6 +15699,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -16525,6 +16595,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -16539,6 +16614,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -17430,6 +17510,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -17444,6 +17529,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -18335,6 +18425,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -18349,6 +18444,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -19240,6 +19340,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -19254,6 +19359,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -20145,6 +20255,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,na org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -20159,6 +20274,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,na org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21050,6 +21170,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_st org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_statements,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21064,6 +21189,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_st org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_statements,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21955,6 +22085,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21969,6 +22104,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -22860,6 +23000,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estima org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -22874,6 +23019,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estima org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23765,6 +23915,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_act org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_activity_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23779,6 +23934,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_act org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_activity_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -24670,6 +24830,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estim org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -24684,6 +24849,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estim org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -25575,6 +25745,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -25589,6 +25764,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -26480,6 +26660,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -26494,6 +26679,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -27385,6 +27575,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=view_builds org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=view_builds_in_progress,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -28290,6 +28485,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=networ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=network_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -30100,6 +30300,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_m org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_members,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -30114,6 +30319,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_m org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_members,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31005,6 +31215,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_p org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31019,6 +31234,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_p org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31910,6 +32130,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles, org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31924,6 +32149,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles, org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -33720,6 +33950,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope=repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -34625,6 +34860,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope=view_build_status,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -35530,6 +35770,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggr org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggregates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -35544,6 +35789,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggr org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggregates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -36435,6 +36685,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=colu org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -36449,6 +36704,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=colu org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -37340,6 +37600,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=drop org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=dropped_columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -38245,6 +38510,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=func org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=functions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -38259,6 +38529,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=func org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=functions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -39150,6 +39425,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=inde org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=indexes,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -39164,6 +39444,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=inde org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=indexes,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40055,6 +40340,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keys org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keyspaces,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40069,6 +40359,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keys org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keyspaces,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40960,6 +41255,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tabl org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tables,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40974,6 +41274,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tabl org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tables,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -41865,6 +42170,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=trig org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=triggers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -41879,6 +42189,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=trig org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=triggers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -42770,6 +43085,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=type org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=types,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -42784,6 +43104,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=type org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=types,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -43675,6 +44000,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=view org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -43689,6 +44019,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=view org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44580,6 +44915,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=even org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44594,6 +44934,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=even org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -45485,6 +45830,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sess org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sessions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -45499,6 +45849,11 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sess org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sessions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -46387,13 +46742,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BloomFilterOffHeapMemoryUsed - name: objectName parameters: [] returnType: javax.management.ObjectName -org.apache.cassandra.metrics:type=ColumnFamily,name=BytesAnticompacted: - attributes: - - {access: read-only, name: Value, type: java.lang.Object} - operations: - - name: objectName - parameters: [] - returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesFlushed: attributes: - {access: read-only, name: Value, type: java.lang.Object} @@ -46401,13 +46749,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BytesFlushed: - name: objectName parameters: [] returnType: javax.management.ObjectName -org.apache.cassandra.metrics:type=ColumnFamily,name=BytesMutatedAnticompaction: - attributes: - - {access: read-only, name: Value, type: java.lang.Object} - operations: - - name: objectName - parameters: [] - returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesPendingRepair: attributes: - {access: read-only, name: Value, type: java.lang.Object} @@ -58279,6 +58620,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=Blo org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -58293,6 +58639,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=Byt org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -59388,6 +59739,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -59402,6 +59758,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -60497,6 +60858,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -60511,6 +60877,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -61606,6 +61977,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=Bloom org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -61620,6 +61996,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=Bytes org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -62715,6 +63096,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -62729,6 +63115,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -63824,6 +64215,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -63838,6 +64234,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -64933,6 +65334,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -64947,6 +65353,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -66042,6 +66453,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -66056,6 +66472,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -67151,6 +67572,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -67165,6 +67591,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -68260,6 +68691,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -68274,6 +68710,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -69369,6 +69810,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -69383,6 +69829,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -70478,6 +70929,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=Bloo org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -70492,6 +70948,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=Byte org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -71587,6 +72048,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statement org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statements,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -71601,6 +72067,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statement org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statements,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -72696,6 +73167,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=Bloom org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -72710,6 +73186,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=Bytes org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -73805,6 +74286,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -73819,6 +74305,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -74914,6 +75405,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -74928,6 +75424,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -76023,6 +76524,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,na org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -76037,6 +76543,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,na org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -77132,6 +77643,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -77146,6 +77662,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -78241,6 +78762,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -78255,6 +78781,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -79350,6 +79881,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_pro org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_progress,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -79364,6 +79900,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_pro org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_progress,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -80459,6 +81000,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -80473,6 +81019,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -81568,6 +82119,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=resource_role org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=resource_role_permissons_index,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -82677,6 +83233,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members, org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -82691,6 +83252,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members, org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -83786,6 +84352,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -83800,6 +84371,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -84895,6 +85471,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=Bl org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -84909,6 +85490,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=By org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -86004,6 +86590,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=parent org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=parent_repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -87113,6 +87704,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -87127,6 +87723,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -88222,6 +88823,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=view_b org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=view_build_status,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -89331,6 +89937,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates, org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -89345,6 +89956,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates, org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -90440,6 +91056,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -90454,6 +91075,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -91549,6 +92175,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_col org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -91563,6 +92194,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_col org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -92658,6 +93294,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -92672,6 +93313,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -93767,6 +94413,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -93781,6 +94432,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -94876,6 +95532,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -94890,6 +95551,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -95985,6 +96651,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -95999,6 +96670,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -97094,6 +97770,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,na org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -97108,6 +97789,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,na org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -98203,6 +98889,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -98217,6 +98908,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -99312,6 +100008,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -99326,6 +100027,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -100421,6 +101127,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -100435,6 +101146,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -101530,6 +102246,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,na org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -101544,6 +102265,11 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,na org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -102636,13 +103362,6 @@ org.apache.cassandra.metrics:type=Table,name=BloomFilterOffHeapMemoryUsed: - name: objectName parameters: [] returnType: javax.management.ObjectName -org.apache.cassandra.metrics:type=Table,name=BytesAnticompacted: - attributes: - - {access: read-only, name: Value, type: java.lang.Object} - operations: - - name: objectName - parameters: [] - returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesFlushed: attributes: - {access: read-only, name: Value, type: java.lang.Object} @@ -102650,13 +103369,6 @@ org.apache.cassandra.metrics:type=Table,name=BytesFlushed: - name: objectName parameters: [] returnType: javax.management.ObjectName -org.apache.cassandra.metrics:type=Table,name=BytesMutatedAnticompaction: - attributes: - - {access: read-only, name: Value, type: java.lang.Object} - operations: - - name: objectName - parameters: [] - returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesPendingRepair: attributes: - {access: read-only, name: Value, type: java.lang.Object} From becdbebbd27b15c9b885edada73b323a6b7fe3dd Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Fri, 20 Dec 2024 08:05:01 -0800 Subject: [PATCH 097/199] Implement minimum repair task duration setting for auto-repair scheduler --- CHANGES.txt | 1 + .../repair/autorepair/AutoRepair.java | 14 ++++ .../repair/autorepair/AutoRepairConfig.java | 14 ++++ .../cassandra/service/AutoRepairService.java | 7 ++ .../service/AutoRepairServiceMBean.java | 2 + .../org/apache/cassandra/tools/NodeProbe.java | 5 ++ .../tools/nodetool/GetAutoRepairConfig.java | 1 + .../tools/nodetool/SetAutoRepairConfig.java | 6 +- .../autorepair/AutoRepairConfigTest.java | 10 ++- .../AutoRepairParameterizedTest.java | 64 ++++++++++++++++--- .../nodetool/SetAutoRepairConfigTest.java | 12 +++- 11 files changed, 125 insertions(+), 11 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 50c62dcc9151..c1b281600896 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1.6 + * Implement minimum repair task duration setting for auto-repair scheduler (CASSANDRA-20160) * Implement preview_repaired auto-repair type (CASSANDRA-20046) * Refresh stale paxos commit (CASSANDRA-19617) * Reduce info logging from automatic paxos repair (CASSANDRA-19445) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 709299def1e9..fc9b55e4cc78 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -304,7 +304,9 @@ public void repair(AutoRepairConfig.RepairType repairType) f = repairRunnableExecutors.get(repairType).submit(task); try { + long jobStartTime = timeFunc.get(); repairState.waitForRepairToComplete(config.getRepairSessionTimeout(repairType)); + soakAfterRepair(jobStartTime, config.getRepairTaskMinDuration().toMilliseconds()); } catch (InterruptedException e) { @@ -466,4 +468,16 @@ public AutoRepairState getRepairState(AutoRepairConfig.RepairType repairType) { return repairStates.get(repairType); } + + private void soakAfterRepair(long startTimeMilis, long minDurationMilis) + { + long currentTime = timeFunc.get(); + long timeElapsed = currentTime - startTimeMilis; + if (timeElapsed < minDurationMilis) + { + long timeToSoak = minDurationMilis - timeElapsed; + logger.info("Soaking for {} ms after repair", timeToSoak); + sleepFunc.accept(timeToSoak, TimeUnit.MILLISECONDS); + } + } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index fbcdd09d9d7c..3ec574eb955f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -50,6 +50,10 @@ public class AutoRepairConfig implements Serializable public volatile Integer repair_max_retries = 3; // the backoff time in seconds for retrying a repair session. public volatile DurationSpec.LongSecondsBound repair_retry_backoff = new DurationSpec.LongSecondsBound("30s"); + // the minimum duration for the execution of a single repair task (i.e.: RepairRunnable). + // This helps prevent the auto-repair scheduler from overwhelming the node by scheduling a large number of + // repair tasks in a short period of time. + public volatile DurationSpec.LongSecondsBound repair_task_min_duration = new DurationSpec.LongSecondsBound("5s"); // global_settings overides Options.defaultOptions for all repair types public volatile Options global_settings; @@ -134,6 +138,16 @@ public void setRepairRetryBackoff(String interval) repair_retry_backoff = new DurationSpec.LongSecondsBound(interval); } + public DurationSpec.LongSecondsBound getRepairTaskMinDuration() + { + return repair_task_min_duration; + } + + public void setRepairTaskMinDuration(String duration) + { + repair_task_min_duration = new DurationSpec.LongSecondsBound(duration); + } + public boolean isAutoRepairEnabled(RepairType repairType) { return enabled && applyOverrides(repairType, opt -> opt.enabled); diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 62393d66ebab..1e4ce07f58cf 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -19,6 +19,7 @@ import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; @@ -137,6 +138,12 @@ public void setAutoRepairRetryBackoff(String interval) config.setRepairRetryBackoff(interval); } + @Override + public void setAutoRepairMinRepairTaskDuration(String duration) + { + config.setRepairTaskMinDuration(duration); + } + @Override public void setRepairSSTableCountHigherThreshold(RepairType repairType, int sstableHigherThreshold) { diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index 5e62e5630043..f3578194236d 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -48,6 +48,8 @@ public interface AutoRepairServiceMBean public void setAutoRepairRetryBackoff(String interval); + public void setAutoRepairMinRepairTaskDuration(String duration); + public void setRepairSSTableCountHigherThreshold(RepairType repairType, int ssTableHigherThreshold); public void setAutoRepairTableMaxRepairTime(RepairType repairType, String autoRepairTableMaxRepairTime); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index e89294c5c341..d3b92f869dbb 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2200,6 +2200,11 @@ public void setAutoRepairRetryBackoff(String interval) autoRepairProxy.setAutoRepairRetryBackoff(interval); } + public void setAutoRepairMinRepairTaskDuration(String duration) + { + autoRepairProxy.setAutoRepairMinRepairTaskDuration(duration); + } + public void setRepairSSTableCountHigherThreshold(AutoRepairConfig.RepairType repairType, int ssTableHigherThreshold) { autoRepairProxy.setRepairSSTableCountHigherThreshold(repairType, ssTableHigherThreshold); diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java index 48f8d54de3f6..681b8533824b 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java @@ -49,6 +49,7 @@ public void execute(NodeProbe probe) sb.append("\n\tTTL for repair history for dead nodes: " + config.getAutoRepairHistoryClearDeleteHostsBufferInterval()); sb.append("\n\tmax retries for repair: " + config.getRepairMaxRetries()); sb.append("\n\tretry backoff: " + config.getRepairRetryBackoff()); + sb.append("\n\tmin repair job duration: " + config.getRepairTaskMinDuration().toSeconds() + " seconds"); for (RepairType repairType : RepairType.values()) { sb.append(formatRepairTypeConfig(probe, repairType, config)); diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index b467d9b82998..9e9709db5e3a 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -46,7 +46,8 @@ public class SetAutoRepairConfig extends NodeToolCmd "[number_of_repair_threads|number_of_subranges|min_repair_interval|sstable_upper_threshold" + "|enabled|table_max_repair_time|priority_hosts|forcerepair_hosts|ignore_dcs" + "|history_clear_delete_hosts_buffer_interval|repair_primary_token_range_only" + - "|parallel_repair_count|parallel_repair_percentage|mv_repair_enabled|repair_max_retries|repair_retry_backoff|repair_session_timeout]", + "|parallel_repair_count|parallel_repair_percentage|mv_repair_enabled|repair_max_retries" + + "|repair_retry_backoff|repair_session_timeout|min_repair_task_duration]", required = true) protected List args = new ArrayList<>(); @@ -82,6 +83,9 @@ public void execute(NodeProbe probe) case "repair_retry_backoff": probe.setAutoRepairRetryBackoff(paramVal); return; + case "min_repair_task_duration": + probe.setAutoRepairMinRepairTaskDuration(paramVal); + return; default: // proceed to options that require --repair-type option break; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index c42959eec32b..4d446096897c 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -90,6 +90,15 @@ public void testIsAutoRepairEnabledReturnsTrueWhenRepairIsEnabled() assertTrue(config.isAutoRepairEnabled(repairType)); } + @Test + public void testRepairMinDuration() + { + config = new AutoRepairConfig(false); + + config.setRepairTaskMinDuration("3s"); + assertEquals(3L, config.getRepairTaskMinDuration().toSeconds()); + } + @Test public void testIsAutoRepairEnabledReturnsTrueWhenRepairIsDisabledGlobally() { @@ -98,7 +107,6 @@ public void testIsAutoRepairEnabledReturnsTrueWhenRepairIsDisabledGlobally() assertFalse(config.isAutoRepairEnabled(repairType)); } - @Test public void testIsAutoRepairEnabledReturnsTrueWhenRepairIsDisabledForRepairType() { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 4e47dad53b68..65dc4686ee06 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -107,6 +107,12 @@ public class AutoRepairParameterizedTest extends CQLTester RepairRunnable repairRunnable; private static AutoRepairConfig defaultConfig; + //system_auth.role_permissions,system_auth.network_permissions,system_auth.role_members,system_auth.roles, + // system_auth.resource_role_permissons_index,system_traces.sessions,system_traces.events,ks.tbl, + // system_distributed.auto_repair_priority,system_distributed.repair_history,system_distributed.auto_repair_history, + // system_distributed.view_build_status,system_distributed.parent_repair_history,system_distributed.partition_denylist + private final int expectedTablesGoingThroughRepair = 14; + @Parameterized.Parameter() public AutoRepairConfig.RepairType repairType; @@ -182,6 +188,7 @@ public void setup() timeFuncCalls = 0; AutoRepair.timeFunc = System::currentTimeMillis; + AutoRepair.sleepFunc = (Long startTime, TimeUnit unit) -> {}; resetCounters(); resetConfig(); @@ -220,6 +227,7 @@ private void resetConfig() config.global_settings = defaultConfig.global_settings; config.history_clear_delete_hosts_buffer_interval = defaultConfig.history_clear_delete_hosts_buffer_interval; config.setRepairSubRangeNum(repairType, 1); + config.repair_task_min_duration = new DurationSpec.LongSecondsBound("0s"); } private void executeCQL() @@ -666,15 +674,10 @@ public void testRepairMaxRetries() AutoRepair.instance.repair(repairType); - //system_auth.role_permissions,system_auth.network_permissions,system_auth.role_members,system_auth.roles, - // system_auth.resource_role_permissons_index,system_traces.sessions,system_traces.events,ks.tbl, - // system_distributed.auto_repair_priority,system_distributed.repair_history,system_distributed.auto_repair_history, - // system_distributed.view_build_status,system_distributed.parent_repair_history,system_distributed.partition_denylist - int exptedTablesGoingThroughRepair = 14; - assertEquals(config.getRepairMaxRetries()*exptedTablesGoingThroughRepair, sleepCalls.get()); + assertEquals(config.getRepairMaxRetries()*expectedTablesGoingThroughRepair, sleepCalls.get()); verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); - verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(exptedTablesGoingThroughRepair); + verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(expectedTablesGoingThroughRepair); } @Test @@ -700,7 +703,7 @@ public void testRepairSuccessAfterRetry() AutoRepair.instance.repair(repairType); assertEquals(1, sleepCalls.get()); - verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(14); + verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); } @@ -751,4 +754,49 @@ public void testRepairThrowsForIRWithCDCReplay() AutoRepair.instance.repair(repairType); } } + + + @Test + public void testSoakAfterImmediateRepair() + { + when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())).thenReturn(repairRunnable); + when(autoRepairState.isSuccess()).thenReturn(true); + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.repair_task_min_duration = new DurationSpec.LongSecondsBound("1s"); + AtomicInteger sleepCalls = new AtomicInteger(); + AutoRepair.sleepFunc = (Long duration, TimeUnit unit) -> { + sleepCalls.getAndIncrement(); + assertEquals(TimeUnit.MILLISECONDS, unit); + assertTrue(config.getRepairTaskMinDuration().toMilliseconds() >= duration); + }; + config.setRepairMinInterval(repairType, "0s"); + AutoRepair.instance.repairStates.put(repairType, autoRepairState); + + AutoRepair.instance.repair(repairType); + + assertEquals(expectedTablesGoingThroughRepair, sleepCalls.get()); + verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); + verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); + } + + @Test + public void testNoSoakAfterRepair() + { + when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())).thenReturn(repairRunnable); + when(autoRepairState.isSuccess()).thenReturn(true); + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.repair_task_min_duration = new DurationSpec.LongSecondsBound("0s"); + AutoRepair.sleepFunc = (Long duration, TimeUnit unit) -> { + fail("Should not sleep after repair"); + }; + config.setRepairMinInterval(repairType, "0s"); + AutoRepair.instance.repairStates.put(repairType, autoRepairState); + + AutoRepair.instance.repair(repairType); + + verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); + verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); + } } diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index 2a69cefcace9..c756cd378bfd 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -105,7 +105,6 @@ public void testRepairMaxRetries() verify(probe, times(1)).setAutoRepairMaxRetriesCount(2); } - @Test public void testRetryBackoffInSec() { @@ -115,6 +114,17 @@ public void testRetryBackoffInSec() verify(probe, times(1)).setAutoRepairRetryBackoff("3s"); } + + @Test + public void testMinRepairDuration() + { + cmd.args = ImmutableList.of("min_repair_task_duration", "4s"); + + cmd.execute(probe); + + verify(probe, times(1)).setAutoRepairMinRepairTaskDuration("4s"); + } + } @RunWith(Parameterized.class) From 9bfecb10ae7e486f2200dd5f7fa9f0d9f53f6446 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Fri, 20 Dec 2024 08:09:45 -0800 Subject: [PATCH 098/199] Cleanup --- .../repair/autorepair/AutoRepairParameterizedTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 65dc4686ee06..c78071fdf895 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -107,13 +107,12 @@ public class AutoRepairParameterizedTest extends CQLTester RepairRunnable repairRunnable; private static AutoRepairConfig defaultConfig; - //system_auth.role_permissions,system_auth.network_permissions,system_auth.role_members,system_auth.roles, + // system_auth.role_permissions,system_auth.network_permissions,system_auth.role_members,system_auth.roles, // system_auth.resource_role_permissons_index,system_traces.sessions,system_traces.events,ks.tbl, // system_distributed.auto_repair_priority,system_distributed.repair_history,system_distributed.auto_repair_history, // system_distributed.view_build_status,system_distributed.parent_repair_history,system_distributed.partition_denylist private final int expectedTablesGoingThroughRepair = 14; - @Parameterized.Parameter() public AutoRepairConfig.RepairType repairType; From a2d3405ff33225f2026f7ade858f42b286120610 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Fri, 20 Dec 2024 08:17:22 -0800 Subject: [PATCH 099/199] Cleanup --- src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java | 2 -- .../repair/autorepair/AutoRepairParameterizedTest.java | 1 - 2 files changed, 3 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 268ec55c568f..0edb71271d27 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -325,11 +325,9 @@ private void repairKeyspace(AutoRepairConfig.RepairType repairType, boolean prim f = repairRunnableExecutors.get(repairType).submit(task); try { - long jobStartTime = timeFunc.get(); repairState.waitForRepairToComplete(config.getRepairSessionTimeout(repairType)); soakAfterRepair(jobStartTime, config.getRepairTaskMinDuration().toMilliseconds()); - } catch (InterruptedException e) { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 9a5d357fbfda..b6b7f5697c3e 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -784,7 +784,6 @@ public void testRepairThrowsForIRWithCDCReplay() } } - @Test public void testSoakAfterImmediateRepair() { From eace2b14b601e838de3cac2af320ea4471e27996 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 20 Dec 2024 11:10:38 -0800 Subject: [PATCH 100/199] Convert the metrics to TableMeter --- .../cassandra/metrics/KeyspaceMetrics.java | 13 ++++++++ .../cassandra/metrics/TableMetrics.java | 31 +++++++++++-------- test/data/jmxdump/cassandra-4.0-jmx.yaml | 24 ++++++++++++++ test/data/jmxdump/cassandra-4.1-jmx.yaml | 24 ++++++++++++++ 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index 776027e395a1..50f420bec50f 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -168,6 +168,13 @@ public class KeyspaceMetrics public final Meter rowIndexSizeAborts; public final Histogram rowIndexSize; + public final Meter bytesAnticompacted; + public final Meter bytesMutatedAnticompaction; + public final Meter bytesPreviewed; + public final Meter tokenRangesPreviewedDesynchronized; + public final Meter bytesPreviewedDesynchronized; + + public final MetricNameFactory factory; private Keyspace keyspace; @@ -265,6 +272,12 @@ public KeyspaceMetrics(final Keyspace ks) rowIndexSizeWarnings = createKeyspaceMeter("RowIndexSizeWarnings"); rowIndexSizeAborts = createKeyspaceMeter("RowIndexSizeAborts"); rowIndexSize = createKeyspaceHistogram("RowIndexSize", false); + + bytesAnticompacted = createKeyspaceMeter("BytesAnticompacted"); + bytesMutatedAnticompaction = createKeyspaceMeter("BytesMutatedAnticompaction"); + bytesPreviewed = createKeyspaceMeter("BytesPreviewed"); + tokenRangesPreviewedDesynchronized = createKeyspaceMeter("TokenRangesPreviewedDesynchronized"); + bytesPreviewedDesynchronized = createKeyspaceMeter("BytesPreviewedDesynchronized"); } /** diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 568920875a86..28d8913901a0 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -201,15 +201,15 @@ public class TableMetrics /** number of partitions read creating merkle trees */ public final TableHistogram partitionsValidated; /** number of bytes read while doing anticompaction */ - public final Meter bytesAnticompacted; + public final TableMeter bytesAnticompacted; /** number of bytes where the whole sstable was contained in a repairing range so that we only mutated the repair status */ - public final Meter bytesMutatedAnticompaction; + public final TableMeter bytesMutatedAnticompaction; /** number of bytes that were scanned during preview repair */ - public final Meter bytesPreviewed; + public final TableMeter bytesPreviewed; /** number of desynchronized token ranges that were detected during preview repair */ - public final Meter tokenRangesPreviewedDesynchronized; + public final TableMeter tokenRangesPreviewedDesynchronized; /** number of desynchronized bytes that were detected during preview repair */ - public final Meter bytesPreviewedDesynchronized; + public final TableMeter bytesPreviewedDesynchronized; /** ratio of how much we anticompact vs how much we could mutate the repair status*/ public final Gauge mutatedAnticompactionGauge; @@ -904,15 +904,15 @@ protected double getDenominator() bytesValidated = createTableHistogram("BytesValidated", cfs.keyspace.metric.bytesValidated, false); partitionsValidated = createTableHistogram("PartitionsValidated", cfs.keyspace.metric.partitionsValidated, false); - bytesAnticompacted = createTableMeter("BytesAnticompacted"); - bytesMutatedAnticompaction = createTableMeter("BytesMutatedAnticompaction"); - bytesPreviewed = createTableMeter("BytesPreviewed"); - tokenRangesPreviewedDesynchronized = createTableMeter("TokenRangesPreviewedDesynchronized"); - bytesPreviewedDesynchronized = createTableMeter("BytesPreviewedDesynchronized"); + bytesAnticompacted = createTableMeter("BytesAnticompacted", cfs.keyspace.metric.bytesAnticompacted); + bytesMutatedAnticompaction = createTableMeter("BytesMutatedAnticompaction", cfs.keyspace.metric.bytesMutatedAnticompaction); + bytesPreviewed = createTableMeter("BytesPreviewed", cfs.keyspace.metric.bytesPreviewed); + tokenRangesPreviewedDesynchronized = createTableMeter("TokenRangesPreviewedDesynchronized", cfs.keyspace.metric.tokenRangesPreviewedDesynchronized); + bytesPreviewedDesynchronized = createTableMeter("BytesPreviewedDesynchronized", cfs.keyspace.metric.bytesPreviewedDesynchronized); mutatedAnticompactionGauge = createTableGauge("MutatedAnticompactionGauge", () -> { - double bytesMutated = bytesMutatedAnticompaction.getCount(); - double bytesAnticomp = bytesAnticompacted.getCount(); + double bytesMutated = bytesMutatedAnticompaction.table.getCount(); + double bytesAnticomp = bytesAnticompacted.table.getCount(); if (bytesAnticomp + bytesMutated > 0) return bytesMutated / (bytesAnticomp + bytesMutated); return 0.0; @@ -1246,10 +1246,15 @@ private TableMeter(Meter table, Meter keyspace, Meter global) } public void mark() + { + mark(1L); + } + + public void mark(long val) { for (Meter meter : all) { - meter.mark(); + meter.mark(val); } } } diff --git a/test/data/jmxdump/cassandra-4.0-jmx.yaml b/test/data/jmxdump/cassandra-4.0-jmx.yaml index d99a183c67d4..a9841988c72a 100644 --- a/test/data/jmxdump/cassandra-4.0-jmx.yaml +++ b/test/data/jmxdump/cassandra-4.0-jmx.yaml @@ -103362,6 +103362,18 @@ org.apache.cassandra.metrics:type=Table,name=BloomFilterOffHeapMemoryUsed: - name: objectName parameters: [] returnType: javax.management.ObjectName +org.apache.cassandra.metrics:type=Table,name=BytesAnticompacted: + attributes: + - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} + operations: + - name: objectName + parameters: [] + returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesFlushed: attributes: - {access: read-only, name: Value, type: java.lang.Object} @@ -103369,6 +103381,18 @@ org.apache.cassandra.metrics:type=Table,name=BytesFlushed: - name: objectName parameters: [] returnType: javax.management.ObjectName +org.apache.cassandra.metrics:type=Table,name=BytesMutatedAnticompaction: + attributes: + - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} + operations: + - name: objectName + parameters: [] + returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesPendingRepair: attributes: - {access: read-only, name: Value, type: java.lang.Object} diff --git a/test/data/jmxdump/cassandra-4.1-jmx.yaml b/test/data/jmxdump/cassandra-4.1-jmx.yaml index b2322f2f43bd..e47da56ec400 100644 --- a/test/data/jmxdump/cassandra-4.1-jmx.yaml +++ b/test/data/jmxdump/cassandra-4.1-jmx.yaml @@ -103362,6 +103362,18 @@ org.apache.cassandra.metrics:type=Table,name=BloomFilterOffHeapMemoryUsed: - name: objectName parameters: [] returnType: javax.management.ObjectName +org.apache.cassandra.metrics:type=Table,name=BytesAnticompacted: + attributes: + - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} + operations: + - name: objectName + parameters: [] + returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesFlushed: attributes: - {access: read-only, name: Value, type: java.lang.Object} @@ -103369,6 +103381,18 @@ org.apache.cassandra.metrics:type=Table,name=BytesFlushed: - name: objectName parameters: [] returnType: javax.management.ObjectName +org.apache.cassandra.metrics:type=Table,name=BytesMutatedAnticompaction: + attributes: + - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} + operations: + - name: objectName + parameters: [] + returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesPendingRepair: attributes: - {access: read-only, name: Value, type: java.lang.Object} From 67be3c3a5174b0639e99ca7297e4b43ef1133662 Mon Sep 17 00:00:00 2001 From: Jaydeepkumar Chovatia Date: Fri, 20 Dec 2024 11:15:40 -0800 Subject: [PATCH 101/199] Add a global type=ColumnFamily --- test/data/jmxdump/cassandra-4.0-jmx.yaml | 24 ++++++++++++++++++++++++ test/data/jmxdump/cassandra-4.1-jmx.yaml | 24 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/test/data/jmxdump/cassandra-4.0-jmx.yaml b/test/data/jmxdump/cassandra-4.0-jmx.yaml index a9841988c72a..05351a5aa2ec 100644 --- a/test/data/jmxdump/cassandra-4.0-jmx.yaml +++ b/test/data/jmxdump/cassandra-4.0-jmx.yaml @@ -46742,6 +46742,18 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BloomFilterOffHeapMemoryUsed - name: objectName parameters: [] returnType: javax.management.ObjectName +org.apache.cassandra.metrics:type=ColumnFamily,name=BytesAnticompacted: + attributes: + - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} + operations: + - name: objectName + parameters: [] + returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesFlushed: attributes: - {access: read-only, name: Value, type: java.lang.Object} @@ -46749,6 +46761,18 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BytesFlushed: - name: objectName parameters: [] returnType: javax.management.ObjectName +org.apache.cassandra.metrics:type=ColumnFamily,name=BytesMutatedAnticompaction: + attributes: + - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} + operations: + - name: objectName + parameters: [] + returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesPendingRepair: attributes: - {access: read-only, name: Value, type: java.lang.Object} diff --git a/test/data/jmxdump/cassandra-4.1-jmx.yaml b/test/data/jmxdump/cassandra-4.1-jmx.yaml index e47da56ec400..f858b0387069 100644 --- a/test/data/jmxdump/cassandra-4.1-jmx.yaml +++ b/test/data/jmxdump/cassandra-4.1-jmx.yaml @@ -46742,6 +46742,18 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BloomFilterOffHeapMemoryUsed - name: objectName parameters: [] returnType: javax.management.ObjectName +org.apache.cassandra.metrics:type=ColumnFamily,name=BytesAnticompacted: + attributes: + - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} + operations: + - name: objectName + parameters: [] + returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesFlushed: attributes: - {access: read-only, name: Value, type: java.lang.Object} @@ -46749,6 +46761,18 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BytesFlushed: - name: objectName parameters: [] returnType: javax.management.ObjectName +org.apache.cassandra.metrics:type=ColumnFamily,name=BytesMutatedAnticompaction: + attributes: + - {access: read-only, name: Count, type: long} + - {access: read-only, name: FifteenMinuteRate, type: double} + - {access: read-only, name: FiveMinuteRate, type: double} + - {access: read-only, name: MeanRate, type: double} + - {access: read-only, name: OneMinuteRate, type: double} + - {access: read-only, name: RateUnit, type: java.lang.String} + operations: + - name: objectName + parameters: [] + returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesPendingRepair: attributes: - {access: read-only, name: Value, type: java.lang.Object} From bd94a3de1e393c7e4f642c51ead1dcc87bd62b2e Mon Sep 17 00:00:00 2001 From: Jaydeepkumar Chovatia Date: Fri, 20 Dec 2024 11:18:21 -0800 Subject: [PATCH 102/199] Remove an extra empty line --- src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index 50f420bec50f..0d2a6051f323 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -174,7 +174,6 @@ public class KeyspaceMetrics public final Meter tokenRangesPreviewedDesynchronized; public final Meter bytesPreviewedDesynchronized; - public final MetricNameFactory factory; private Keyspace keyspace; From fcf2e338abfc045e6a0eea24afd23928c94820e8 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 20 Dec 2024 14:22:29 -0800 Subject: [PATCH 103/199] Revert the cassandra-4.x.xml changes --- test/data/jmxdump/cassandra-4.0-jmx.yaml | 768 +---------------------- test/data/jmxdump/cassandra-4.1-jmx.yaml | 768 +---------------------- 2 files changed, 8 insertions(+), 1528 deletions(-) diff --git a/test/data/jmxdump/cassandra-4.0-jmx.yaml b/test/data/jmxdump/cassandra-4.0-jmx.yaml index 05351a5aa2ec..c69487909d8b 100644 --- a/test/data/jmxdump/cassandra-4.0-jmx.yaml +++ b/test/data/jmxdump/cassandra-4.0-jmx.yaml @@ -10190,11 +10190,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,n org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -10209,11 +10204,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,n org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -11105,11 +11095,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -11124,11 +11109,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12020,11 +12000,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12039,11 +12014,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12935,11 +12905,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12954,11 +12919,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -13850,11 +13810,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -13869,11 +13824,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -14765,11 +14715,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -14784,11 +14729,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -15680,11 +15620,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -15699,11 +15634,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -16595,11 +16525,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -16614,11 +16539,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -17510,11 +17430,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -17529,11 +17444,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -18425,11 +18335,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -18444,11 +18349,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -19340,11 +19240,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -19359,11 +19254,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -20255,11 +20145,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,na org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -20274,11 +20159,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,na org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21170,11 +21050,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_st org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_statements,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21189,11 +21064,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_st org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_statements,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -22085,11 +21955,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -22104,11 +21969,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23000,11 +22860,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estima org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23019,11 +22874,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estima org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23915,11 +23765,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_act org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_activity,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23934,11 +23779,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_act org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_activity,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -24830,11 +24670,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estim org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -24849,11 +24684,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estim org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -25745,11 +25575,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -25764,11 +25589,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -26660,11 +26480,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -26679,11 +26494,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -27575,11 +27385,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=view_builds org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=view_builds_in_progress,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -28485,11 +28290,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=networ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=network_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -30300,11 +30100,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_m org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_members,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -30319,11 +30114,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_m org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_members,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31215,11 +31005,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_p org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31234,11 +31019,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_p org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -32130,11 +31910,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles, org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -32149,11 +31924,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles, org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -33950,11 +33720,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope=repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -34860,11 +34625,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope=view_build_status,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -35770,11 +35530,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggr org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggregates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -35789,11 +35544,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggr org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggregates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -36685,11 +36435,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=colu org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -36704,11 +36449,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=colu org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -37600,11 +37340,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=drop org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=dropped_columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -38510,11 +38245,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=func org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=functions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -38529,11 +38259,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=func org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=functions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -39425,11 +39150,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=inde org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=indexes,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -39444,11 +39164,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=inde org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=indexes,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40340,11 +40055,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keys org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keyspaces,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40359,11 +40069,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keys org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keyspaces,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -41255,11 +40960,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tabl org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tables,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -41274,11 +40974,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tabl org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tables,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -42170,11 +41865,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=trig org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=triggers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -42189,11 +41879,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=trig org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=triggers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -43085,11 +42770,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=type org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=types,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -43104,11 +42784,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=type org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=types,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44000,11 +43675,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=view org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44019,11 +43689,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=view org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44915,11 +44580,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=even org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44934,11 +44594,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=even org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -45830,11 +45485,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sess org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sessions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -45849,11 +45499,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sess org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sessions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -46744,12 +46389,7 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BloomFilterOffHeapMemoryUsed returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesAnticompacted: attributes: - - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} + - {access: read-only, name: Value, type: java.lang.Object} operations: - name: objectName parameters: [] @@ -46763,12 +46403,7 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BytesFlushed: returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesMutatedAnticompaction: attributes: - - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} + - {access: read-only, name: Value, type: java.lang.Object} operations: - name: objectName parameters: [] @@ -58644,11 +58279,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=Blo org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -58663,11 +58293,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=Byt org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -59763,11 +59388,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -59782,11 +59402,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -60882,11 +60497,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -60901,11 +60511,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -62001,11 +61606,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=Bloom org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -62020,11 +61620,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=Bytes org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -63120,11 +62715,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -63139,11 +62729,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -64239,11 +63824,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -64258,11 +63838,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -65358,11 +64933,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -65377,11 +64947,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -66477,11 +66042,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -66496,11 +66056,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -67596,11 +67151,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -67615,11 +67165,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -68715,11 +68260,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -68734,11 +68274,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -69834,11 +69369,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -69853,11 +69383,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -70953,11 +70478,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=Bloo org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -70972,11 +70492,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=Byte org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -72072,11 +71587,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statement org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statements,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -72091,11 +71601,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statement org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statements,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -73191,11 +72696,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=Bloom org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -73210,11 +72710,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=Bytes org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -74310,11 +73805,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -74329,11 +73819,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -75429,11 +74914,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -75448,11 +74928,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -76548,11 +76023,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,na org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -76567,11 +76037,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,na org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -77667,11 +77132,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -77686,11 +77146,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -78786,11 +78241,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -78805,11 +78255,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -79905,11 +79350,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_pro org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_progress,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -79924,11 +79364,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_pro org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_progress,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -81024,11 +80459,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -81043,11 +80473,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -82143,11 +81568,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=resource_role org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=resource_role_permissons_index,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -83257,11 +82677,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members, org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -83276,11 +82691,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members, org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -84376,11 +83786,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -84395,11 +83800,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -85495,11 +84895,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=Bl org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -85514,11 +84909,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=By org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -86614,11 +86004,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=parent org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=parent_repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -87728,11 +87113,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -87747,11 +87127,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -88847,11 +88222,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=view_b org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=view_build_status,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -89961,11 +89331,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates, org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -89980,11 +89345,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates, org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -91080,11 +90440,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -91099,11 +90454,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -92199,11 +91549,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_col org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -92218,11 +91563,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_col org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -93318,11 +92658,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -93337,11 +92672,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -94437,11 +93767,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -94456,11 +93781,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -95556,11 +94876,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -95575,11 +94890,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -96675,11 +95985,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -96694,11 +95999,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -97794,11 +97094,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,na org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -97813,11 +97108,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,na org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -98913,11 +98203,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -98932,11 +98217,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -100032,11 +99312,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -100051,11 +99326,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -101151,11 +100421,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -101170,11 +100435,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -102270,11 +101530,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,na org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -102289,11 +101544,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,na org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -103388,12 +102638,7 @@ org.apache.cassandra.metrics:type=Table,name=BloomFilterOffHeapMemoryUsed: returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesAnticompacted: attributes: - - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} + - {access: read-only, name: Value, type: java.lang.Object} operations: - name: objectName parameters: [] @@ -103407,12 +102652,7 @@ org.apache.cassandra.metrics:type=Table,name=BytesFlushed: returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesMutatedAnticompaction: attributes: - - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} + - {access: read-only, name: Value, type: java.lang.Object} operations: - name: objectName parameters: [] diff --git a/test/data/jmxdump/cassandra-4.1-jmx.yaml b/test/data/jmxdump/cassandra-4.1-jmx.yaml index f858b0387069..14e204430b92 100644 --- a/test/data/jmxdump/cassandra-4.1-jmx.yaml +++ b/test/data/jmxdump/cassandra-4.1-jmx.yaml @@ -10190,11 +10190,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,n org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -10209,11 +10204,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,n org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=IndexInfo,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -11105,11 +11095,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -11124,11 +11109,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12020,11 +12000,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12039,11 +12014,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_r org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=available_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12935,11 +12905,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -12954,11 +12919,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=batches,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -13850,11 +13810,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -13869,11 +13824,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=built_views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -14765,11 +14715,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -14784,11 +14729,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=compaction_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -15680,11 +15620,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -15699,11 +15634,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=local,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -16595,11 +16525,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -16614,11 +16539,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=paxos,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -17510,11 +17430,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -17529,11 +17444,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -18425,11 +18335,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -18444,11 +18349,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peer_events_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -19340,11 +19240,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -19359,11 +19254,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name= org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -20255,11 +20145,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,na org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -20274,11 +20159,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,na org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=peers_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21170,11 +21050,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_st org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_statements,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -21189,11 +21064,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_st org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=prepared_statements,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -22085,11 +21955,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -22104,11 +21969,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,nam org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=repairs,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23000,11 +22860,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estima org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23019,11 +22874,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estima org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=size_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23915,11 +23765,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_act org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_activity_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -23934,11 +23779,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_act org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=sstable_activity_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -24830,11 +24670,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estim org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -24849,11 +24684,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estim org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=table_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -25745,11 +25575,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -25764,11 +25589,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -26660,11 +26480,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -26679,11 +26494,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=transferred_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -27575,11 +27385,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=view_builds org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system,scope=view_builds_in_progress,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -28485,11 +28290,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=networ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=network_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -30300,11 +30100,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_m org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_members,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -30319,11 +30114,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_m org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_members,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31215,11 +31005,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_p org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -31234,11 +31019,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_p org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=role_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -32130,11 +31910,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles, org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -32149,11 +31924,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles, org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_auth,scope=roles,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -33950,11 +33720,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope=repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -34860,11 +34625,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_distributed,scope=view_build_status,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -35770,11 +35530,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggr org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggregates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -35789,11 +35544,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggr org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=aggregates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -36685,11 +36435,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=colu org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -36704,11 +36449,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=colu org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -37600,11 +37340,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=drop org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=dropped_columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -38510,11 +38245,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=func org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=functions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -38529,11 +38259,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=func org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=functions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -39425,11 +39150,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=inde org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=indexes,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -39444,11 +39164,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=inde org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=indexes,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40340,11 +40055,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keys org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keyspaces,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -40359,11 +40069,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keys org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=keyspaces,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -41255,11 +40960,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tabl org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tables,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -41274,11 +40974,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tabl org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=tables,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -42170,11 +41865,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=trig org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=triggers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -42189,11 +41879,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=trig org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=triggers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -43085,11 +42770,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=type org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=types,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -43104,11 +42784,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=type org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=types,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44000,11 +43675,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=view org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44019,11 +43689,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=view org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_schema,scope=views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44915,11 +44580,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=even org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -44934,11 +44594,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=even org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -45830,11 +45485,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sess org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sessions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -45849,11 +45499,6 @@ org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sess org.apache.cassandra.metrics:type=ColumnFamily,keyspace=system_traces,scope=sessions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -46744,12 +46389,7 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BloomFilterOffHeapMemoryUsed returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesAnticompacted: attributes: - - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} + - {access: read-only, name: Value, type: java.lang.Object} operations: - name: objectName parameters: [] @@ -46763,12 +46403,7 @@ org.apache.cassandra.metrics:type=ColumnFamily,name=BytesFlushed: returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=ColumnFamily,name=BytesMutatedAnticompaction: attributes: - - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} + - {access: read-only, name: Value, type: java.lang.Object} operations: - name: objectName parameters: [] @@ -58644,11 +58279,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=Blo org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -58663,11 +58293,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=Byt org.apache.cassandra.metrics:type=Table,keyspace=system,scope=IndexInfo,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -59763,11 +59388,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -59782,11 +59402,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,n org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -60882,11 +60497,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -60901,11 +60511,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=available_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -62001,11 +61606,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=Bloom org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -62020,11 +61620,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=Bytes org.apache.cassandra.metrics:type=Table,keyspace=system,scope=batches,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -63120,11 +62715,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -63139,11 +62729,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=built_views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -64239,11 +63824,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -64258,11 +63838,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history org.apache.cassandra.metrics:type=Table,keyspace=system,scope=compaction_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -65358,11 +64933,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -65377,11 +64947,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=local,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -66477,11 +66042,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -66496,11 +66056,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=paxos,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -67596,11 +67151,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -67615,11 +67165,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=B org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -68715,11 +68260,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -68734,11 +68274,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peer_events_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -69834,11 +69369,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BloomFi org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -69853,11 +69383,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesFl org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -70953,11 +70478,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=Bloo org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -70972,11 +70492,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=Byte org.apache.cassandra.metrics:type=Table,keyspace=system,scope=peers_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -72072,11 +71587,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statement org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statements,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -72091,11 +71601,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statement org.apache.cassandra.metrics:type=Table,keyspace=system,scope=prepared_statements,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -73191,11 +72696,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=Bloom org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -73210,11 +72710,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=Bytes org.apache.cassandra.metrics:type=Table,keyspace=system,scope=repairs,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -74310,11 +73805,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -74329,11 +73819,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,nam org.apache.cassandra.metrics:type=Table,keyspace=system,scope=size_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -75429,11 +74914,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -75448,11 +74928,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity_v org.apache.cassandra.metrics:type=Table,keyspace=system,scope=sstable_activity_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -76548,11 +76023,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,na org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -76567,11 +76037,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,na org.apache.cassandra.metrics:type=Table,keyspace=system,scope=table_estimates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -77667,11 +77132,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -77686,11 +77146,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -78786,11 +78241,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges_v2,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -78805,11 +78255,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges org.apache.cassandra.metrics:type=Table,keyspace=system,scope=transferred_ranges_v2,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -79905,11 +79350,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_pro org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_progress,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -79924,11 +79364,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_pro org.apache.cassandra.metrics:type=Table,keyspace=system,scope=view_builds_in_progress,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -81024,11 +80459,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -81043,11 +80473,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=network_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -82143,11 +81568,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=resource_role org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=resource_role_permissons_index,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -83257,11 +82677,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members, org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -83276,11 +82691,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members, org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_members,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -84376,11 +83786,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -84395,11 +83800,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissi org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=role_permissions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -85495,11 +84895,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=Bl org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -85514,11 +84909,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=By org.apache.cassandra.metrics:type=Table,keyspace=system_auth,scope=roles,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -86614,11 +86004,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=parent org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=parent_repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -87728,11 +87113,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair_history,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -87747,11 +87127,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=repair_history,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -88847,11 +88222,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=view_b org.apache.cassandra.metrics:type=Table,keyspace=system_distributed,scope=view_build_status,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -89961,11 +89331,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates, org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -89980,11 +89345,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates, org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=aggregates,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -91080,11 +90440,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -91099,11 +90454,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -92199,11 +91549,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_col org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_columns,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -92218,11 +91563,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_col org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=dropped_columns,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -93318,11 +92658,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -93337,11 +92672,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=functions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -94437,11 +93767,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -94456,11 +93781,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,nam org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=indexes,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -95556,11 +94876,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -95575,11 +94890,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,n org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=keyspaces,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -96675,11 +95985,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -96694,11 +95999,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=tables,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -97794,11 +97094,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,na org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -97813,11 +97108,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,na org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=triggers,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -98913,11 +98203,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -98932,11 +98217,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=types,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -100032,11 +99312,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -100051,11 +99326,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name= org.apache.cassandra.metrics:type=Table,keyspace=system_schema,scope=views,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -101151,11 +100421,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -101170,11 +100435,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=events,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -102270,11 +101530,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,na org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,name=BytesAnticompacted: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -102289,11 +101544,6 @@ org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,na org.apache.cassandra.metrics:type=Table,keyspace=system_traces,scope=sessions,name=BytesMutatedAnticompaction: attributes: - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} operations: - name: objectName parameters: [] @@ -103388,12 +102638,7 @@ org.apache.cassandra.metrics:type=Table,name=BloomFilterOffHeapMemoryUsed: returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesAnticompacted: attributes: - - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} + - {access: read-only, name: Value, type: java.lang.Object} operations: - name: objectName parameters: [] @@ -103407,12 +102652,7 @@ org.apache.cassandra.metrics:type=Table,name=BytesFlushed: returnType: javax.management.ObjectName org.apache.cassandra.metrics:type=Table,name=BytesMutatedAnticompaction: attributes: - - {access: read-only, name: Count, type: long} - - {access: read-only, name: FifteenMinuteRate, type: double} - - {access: read-only, name: FiveMinuteRate, type: double} - - {access: read-only, name: MeanRate, type: double} - - {access: read-only, name: OneMinuteRate, type: double} - - {access: read-only, name: RateUnit, type: java.lang.String} + - {access: read-only, name: Value, type: java.lang.Object} operations: - name: objectName parameters: [] From 8256fdf442a51baf6ef80347bd812dcacf163192 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Fri, 20 Dec 2024 21:54:44 -0800 Subject: [PATCH 104/199] gauge compatible meter --- .../metrics/CassandraMetricsRegistry.java | 67 +++++++++++++++++-- .../cassandra/metrics/TableMetrics.java | 16 +++-- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java b/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java index 37c37e357b40..eb776b5cb2dc 100644 --- a/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java +++ b/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java @@ -68,20 +68,30 @@ public Counter counter(MetricName name, MetricName alias) } public Meter meter(MetricName name) + { + return meter(name, false); + } + + public Meter meter(MetricName name, boolean gaugeCompatible) { Meter meter = meter(name.getMetricName()); - registerMBean(meter, name.getMBeanName()); + registerMBean(meter, name.getMBeanName(), gaugeCompatible); return meter; } - public Meter meter(MetricName name, MetricName alias) + public Meter meter(MetricName name, MetricName alias, boolean gaugeCompatible) { - Meter meter = meter(name); - registerAlias(name, alias); + Meter meter = meter(name, gaugeCompatible); + registerAlias(name, alias, gaugeCompatible); return meter; } + public Meter meter(MetricName name, MetricName alias) + { + return meter(name, alias, false); + } + public Histogram histogram(MetricName name, boolean considerZeroes) { Histogram histogram = register(name, new ClearableHistogram(new DecayingEstimatedHistogramReservoir(considerZeroes))); @@ -217,6 +227,11 @@ public boolean remove(MetricName name, MetricName... aliases) } public void registerMBean(Metric metric, ObjectName name) + { + registerMBean(metric, name, false); + } + + public void registerMBean(Metric metric, ObjectName name, boolean gaugeCompatible) { AbstractBean mbean; @@ -229,7 +244,18 @@ else if (metric instanceof Histogram) else if (metric instanceof Timer) mbean = new JmxTimer((Timer) metric, name, TimeUnit.SECONDS, DEFAULT_TIMER_UNIT); else if (metric instanceof Metered) - mbean = new JmxMeter((Metered) metric, name, TimeUnit.SECONDS); + { + // If a gauge compatible meter is requested, create a special implementation which + // also yields a 'Value' attribute for backwards compatibility. + if (gaugeCompatible) + { + mbean = new JmxMeterGaugeCompatible((Metered) metric, name, TimeUnit.SECONDS); + } + else + { + mbean = new JmxMeter((Metered) metric, name, TimeUnit.SECONDS); + } + } else throw new IllegalArgumentException("Unknown metric type: " + metric.getClass()); @@ -238,11 +264,16 @@ else if (metric instanceof Metered) } private void registerAlias(MetricName existingName, MetricName aliasName) + { + registerAlias(existingName, aliasName, false); + } + + private void registerAlias(MetricName existingName, MetricName aliasName, boolean gaugeCompatible) { Metric existing = Metrics.getMetrics().get(existingName.getMetricName()); assert existing != null : existingName + " not registered"; - registerMBean(existing, aliasName.getMBeanName()); + registerMBean(existing, aliasName.getMBeanName(), gaugeCompatible); } private void removeAlias(MetricName name) @@ -531,6 +562,30 @@ private String calculateRateUnit(TimeUnit unit) } } + public interface JmxMeterGaugeCompatibleMBean extends JmxMeterMBean, JmxGaugeMBean {} + + /** + * An implementation of {@link JmxMeter} that is compatible with {@link JmxGaugeMBean} in that it also + * implements {@link JmxGaugeMBean}. This is useful for metrics that were migrated from {@link JmxGauge} + * to {@link JmxMeter} like {@link TableMetrics#bytesAnticompacted} and + * {@link TableMetrics#bytesMutatedAnticompaction}. + */ + private static class JmxMeterGaugeCompatible extends JmxMeter implements JmxMeterGaugeCompatibleMBean + { + + private JmxMeterGaugeCompatible(Metered metric, ObjectName objectName, TimeUnit rateUnit) + { + super(metric, objectName, rateUnit); + } + + @Override + public Object getValue() + { + return getCount(); + } + } + + public interface JmxTimerMBean extends JmxMeterMBean { double getMin(); diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 28d8913901a0..e5d544891c1c 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -904,8 +904,8 @@ protected double getDenominator() bytesValidated = createTableHistogram("BytesValidated", cfs.keyspace.metric.bytesValidated, false); partitionsValidated = createTableHistogram("PartitionsValidated", cfs.keyspace.metric.partitionsValidated, false); - bytesAnticompacted = createTableMeter("BytesAnticompacted", cfs.keyspace.metric.bytesAnticompacted); - bytesMutatedAnticompaction = createTableMeter("BytesMutatedAnticompaction", cfs.keyspace.metric.bytesMutatedAnticompaction); + bytesAnticompacted = createTableMeter("BytesAnticompacted", cfs.keyspace.metric.bytesAnticompacted, true); + bytesMutatedAnticompaction = createTableMeter("BytesMutatedAnticompaction", cfs.keyspace.metric.bytesMutatedAnticompaction, true); bytesPreviewed = createTableMeter("BytesPreviewed", cfs.keyspace.metric.bytesPreviewed); tokenRangesPreviewedDesynchronized = createTableMeter("TokenRangesPreviewedDesynchronized", cfs.keyspace.metric.tokenRangesPreviewedDesynchronized); bytesPreviewedDesynchronized = createTableMeter("BytesPreviewedDesynchronized", cfs.keyspace.metric.bytesPreviewedDesynchronized); @@ -1164,17 +1164,23 @@ protected SnapshottingTimer createTableTimer(String name) protected TableMeter createTableMeter(String name, Meter keyspaceMeter) { - return createTableMeter(name, name, keyspaceMeter); + return createTableMeter(name, keyspaceMeter, false); } - protected TableMeter createTableMeter(String name, String alias, Meter keyspaceMeter) + protected TableMeter createTableMeter(String name, Meter keyspaceMeter, boolean globalMeterGaugeCompatible) + { + return createTableMeter(name, name, keyspaceMeter, globalMeterGaugeCompatible); + } + + protected TableMeter createTableMeter(String name, String alias, Meter keyspaceMeter, boolean globalMeterGaugeCompatible) { Meter meter = Metrics.meter(factory.createMetricName(name), aliasFactory.createMetricName(alias)); register(name, alias, meter); return new TableMeter(meter, keyspaceMeter, Metrics.meter(GLOBAL_FACTORY.createMetricName(name), - GLOBAL_ALIAS_FACTORY.createMetricName(alias))); + GLOBAL_ALIAS_FACTORY.createMetricName(alias), + globalMeterGaugeCompatible)); } private LatencyMetrics createLatencyMetrics(String namePrefix, LatencyMetrics ... parents) From 5d47501adbe8fde916e20bb0a316239bfd540adc Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Sat, 21 Dec 2024 08:29:55 -0800 Subject: [PATCH 105/199] Address comment --- .../repair/autorepair/AutoRepairParameterizedTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index b6b7f5697c3e..70bb33859ba9 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -790,19 +790,20 @@ public void testSoakAfterImmediateRepair() when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())).thenReturn(repairRunnable); when(autoRepairState.isSuccess()).thenReturn(true); AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - config.repair_task_min_duration = new DurationSpec.LongSecondsBound("1s"); + config.repair_task_min_duration = new DurationSpec.LongSecondsBound("10s"); AtomicInteger sleepCalls = new AtomicInteger(); AutoRepair.sleepFunc = (Long duration, TimeUnit unit) -> { sleepCalls.getAndIncrement(); assertEquals(TimeUnit.MILLISECONDS, unit); assertTrue(config.getRepairTaskMinDuration().toMilliseconds() >= duration); + config.repair_task_min_duration = new DurationSpec.LongSecondsBound("0s"); }; config.setRepairMinInterval(repairType, "0s"); AutoRepair.instance.repairStates.put(repairType, autoRepairState); AutoRepair.instance.repair(repairType); - assertEquals(expectedTablesGoingThroughRepair, sleepCalls.get()); + assertEquals(1, sleepCalls.get()); verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); From e2e182082eda72bc4fb3a34ae11e5356520f9d32 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sat, 21 Dec 2024 10:42:11 -0800 Subject: [PATCH 106/199] Remove unused imports --- src/java/org/apache/cassandra/service/AutoRepairService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 5e8bbee6eb90..167e4507a7c1 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -18,7 +18,6 @@ package org.apache.cassandra.service; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; From 1d8acfdfeb87b32b1daac17f8e1657991ce9e431 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Thu, 2 Jan 2025 14:56:35 -0600 Subject: [PATCH 107/199] Make RepairTokenRangeSplitter the default implementation Summary of impacting changes: * RepairTokenRangeSplitter is now the default in favor of FixedSplitTokenRangeSplitter. * number_of_subranges moved from a repair_type_override/global config to a property of FixedSplitTokenRangeSplitter. * [get|set]autorepairconfig usability changes Detailed breakdown of changes: 0. IAutoRepairTokenRangeSplitter changes: * Move RepairType from an parameter in getRepairAssignments to a parameter in the constructor for implementations. This was done because the RepairType is always the same for a splitter instance. * Add setParameter and getParameter methods which are used by setautorepairconfig and getautorepairconfig to dynamically update splitter configuration. 1. [get|set]autorepairconfig changes: * getautorepairconfig output now shows property names instead of human readable names (e.g. repair_check_interval instead of repair eligibility check interval). This was done to make it more intuitive to know what properties to use for setautorepairconfig. * getautorepairconfig and setautorepairconfig now support viewing and changing splitter properties, e.g.: setautorepairconfig token_range_splitter.max_bytes_per_schedule 500GiB -t full 2. RepairTokenRangeSplitter changes: * Renames RepairRangeSplitter to RepairTokenRangeSplitter and makes it the default implementation. * Establishes defaults for each repair type to be sensible. * Improve javadocs detailing the primary goal of the splitter, its configuration and its defaults and the justifications for using them. * Rename variables to be consistent with their setting names. 3. FixedSplitTokenRangeSplitter changes: * Renames DefaultAutoRepairTokenSplitter to FixedSplitTokenRangeSplitter as it is no longer the default. * Move number_of_subranges from a global config to a property for this splitter. 4. RepairAssignmentIterator * Refactored common code from both splitter implementations into RepairAssignmentIterator with the aim to reduce the amount of boiler plate custom splitter implementations need to implement. 5. Test changes * Fix AutoRepairParameterizedTest to use fixed splitter so we get a deterministic repair plan. * Allow splitter to be changed programmatically, only expect it to be used for tests. * Rename CassandraSreamReceiverTest and fix it Whether streaming cdc/mvs into commitlogs was previously dependent on system properties; update the test to account for the new yaml properties. * Fix dtest after CASSANDRA-20160 The introduction of repair_task_min_duration causes repairs to take cumulatively longer for a node than 2 minutes. To resolve this, set that to 0s, and also enable repair_by_keyspace and set subranges to 1 to reduce the overall number of repairs. Patch by Andy Tolbert; reviewed by ___ for CASSANDRA-20179 --- .../db/streaming/CassandraStreamReceiver.java | 2 +- .../repair/autorepair/AutoRepair.java | 53 +- .../repair/autorepair/AutoRepairConfig.java | 95 +++- .../repair/autorepair/AutoRepairUtils.java | 2 +- ...java => FixedSplitTokenRangeSplitter.java} | 109 ++-- .../IAutoRepairTokenRangeSplitter.java | 48 +- .../autorepair/PrioritizedRepairPlan.java | 2 +- .../autorepair/RepairAssignmentIterator.java | 84 +++ ...ter.java => RepairTokenRangeSplitter.java} | 513 ++++++++++++++---- .../cassandra/service/AutoRepairService.java | 19 +- .../service/AutoRepairServiceMBean.java | 7 +- .../org/apache/cassandra/tools/NodeProbe.java | 15 +- .../tools/nodetool/GetAutoRepairConfig.java | 71 ++- .../tools/nodetool/SetAutoRepairConfig.java | 22 +- .../apache/cassandra/utils/FBUtilities.java | 29 - .../test/repair/AutoRepairSchedulerTest.java | 8 +- .../config/DatabaseDescriptorRefTest.java | 10 +- ....java => CassandraStreamReceiverTest.java} | 48 +- .../autorepair/AutoRepairConfigTest.java | 25 +- ...DefaultTokenSplitterParameterizedTest.java | 13 +- .../AutoRepairParameterizedTest.java | 11 +- ...java => RepairTokenRangeSplitterTest.java} | 74 +-- .../service/AutoRepairServiceTest.java | 1 - .../nodetool/SetAutoRepairConfigTest.java | 21 +- 24 files changed, 894 insertions(+), 388 deletions(-) rename src/java/org/apache/cassandra/repair/autorepair/{DefaultAutoRepairTokenSplitter.java => FixedSplitTokenRangeSplitter.java} (55%) create mode 100644 src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java rename src/java/org/apache/cassandra/repair/autorepair/{RepairRangeSplitter.java => RepairTokenRangeSplitter.java} (57%) rename test/unit/org/apache/cassandra/db/streaming/{CassandraSreamReceiverTest.java => CassandraStreamReceiverTest.java} (71%) rename test/unit/org/apache/cassandra/repair/autorepair/{RepairRangeSplitterTest.java => RepairTokenRangeSplitterTest.java} (78%) diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index eab07abf56dd..731e9eae528b 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -173,7 +173,7 @@ private boolean hasCDC(ColumnFamilyStore cfs) return cfs.metadata().params.cdc; } - // returns true iif it is a cdc table and cdc on repair is enabled. + // returns true if it is a cdc table and cdc on repair is enabled. private boolean cdcRequiresWriteCommitLog(ColumnFamilyStore cfs) { return DatabaseDescriptor.isCDCOnRepairEnabled() && hasCDC(cfs); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 0edb71271d27..5be067b3a4e0 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -19,6 +19,7 @@ package org.apache.cassandra.repair.autorepair; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumMap; import java.util.HashSet; import java.util.Iterator; @@ -37,6 +38,7 @@ import com.google.common.util.concurrent.Uninterruptibles; import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.utils.Clock; @@ -91,8 +93,6 @@ public class AutoRepair @VisibleForTesting protected static BiConsumer sleepFunc = Uninterruptibles::sleepUninterruptibly; - protected final Map tokenRangeSplitters = new EnumMap<>(AutoRepairConfig.RepairType.class); - private boolean isSetupDone = false; @VisibleForTesting @@ -107,7 +107,43 @@ protected AutoRepair() repairExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-Repair-" + repairType.getConfigName(), Thread.NORM_PRIORITY)); repairRunnableExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-RepairRunnable-" + repairType.getConfigName(), Thread.NORM_PRIORITY)); repairStates.put(repairType, AutoRepairConfig.RepairType.getAutoRepairState(repairType)); - tokenRangeSplitters.put(repairType, FBUtilities.newAutoRepairTokenRangeSplitter(config.getTokenRangeSplitter(repairType))); + } + } + + @VisibleForTesting + static IAutoRepairTokenRangeSplitter newAutoRepairTokenRangeSplitter(AutoRepairConfig.RepairType repairType, ParameterizedClass parameterizedClass) throws ConfigurationException + { + try + { + Class tokenRangeSplitterClass; + final String className; + if (parameterizedClass.class_name != null && !parameterizedClass.class_name.isEmpty()) + { + className = parameterizedClass.class_name.contains(".") ? + parameterizedClass.class_name : + "org.apache.cassandra.repair.autorepair." + parameterizedClass.class_name; + tokenRangeSplitterClass = FBUtilities.classForName(className, "token_range_splitter"); + } + else + { + // If token_range_splitter.class_name is not defined, just use default, this is for convenience. + tokenRangeSplitterClass = AutoRepairConfig.DEFAULT_SPLITTER; + } + try + { + Map parameters = parameterizedClass.parameters != null ? parameterizedClass.parameters : Collections.emptyMap(); + // first attempt to initialize with RepairType and Map arguments. + return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor(AutoRepairConfig.RepairType.class, Map.class).newInstance(repairType, parameters); + } + catch (NoSuchMethodException nsme) + { + // fall back on no argument constructor. + return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor().newInstance(); + } + } + catch (Exception ex) + { + throw new ConfigurationException("Unable to create instance of IAutoRepairTokenRangeSplitter", ex); } } @@ -149,15 +185,6 @@ public void repairAsync(AutoRepairConfig.RepairType repairType) repairExecutors.get(repairType).submit(() -> repair(repairType)); } - /** - * @return The priority of the given table if defined, otherwise 0. - */ - private int getPriority(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName) - { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); - return cfs != null ? cfs.metadata().params.autoRepair.priority() : 0; - } - // repair runs a repair session of the given type synchronously. public void repair(AutoRepairConfig.RepairType repairType) { @@ -228,7 +255,7 @@ public void repair(AutoRepairConfig.RepairType repairType) List repairPlans = PrioritizedRepairPlan.build(keyspacesAndTablesToRepair, repairType, shuffleFunc); // calculate the repair assignments for each priority:keyspace. - Iterator repairAssignmentsIterator = tokenRangeSplitters.get(repairType).getRepairAssignments(repairType, primaryRangeOnly, repairPlans); + Iterator repairAssignmentsIterator = config.getTokenRangeSplitterInstance(repairType).getRepairAssignments(primaryRangeOnly, repairPlans); while (repairAssignmentsIterator.hasNext()) { diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index dc17f26ef1f3..2ac27f265f0f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -20,7 +20,9 @@ import java.io.Serializable; import java.util.Collections; +import java.util.EnumMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.function.Function; @@ -32,6 +34,8 @@ import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.config.ParameterizedClass; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.utils.FBUtilities; public class AutoRepairConfig implements Serializable { @@ -59,6 +63,11 @@ public class AutoRepairConfig implements Serializable // global_settings overides Options.defaultOptions for all repair types public volatile Options global_settings; + public static final Class DEFAULT_SPLITTER = RepairTokenRangeSplitter.class; + + // make transient so gets consturcted in the implementation. + private final transient Map tokenRangeSplitters = new EnumMap<>(AutoRepairConfig.RepairType.class); + public enum RepairType implements Serializable { FULL, @@ -203,16 +212,6 @@ public void setRepairThreads(RepairType repairType, int repairThreads) getOptions(repairType).number_of_repair_threads = repairThreads; } - public int getRepairSubRangeNum(RepairType repairType) - { - return applyOverrides(repairType, opt -> opt.number_of_subranges); - } - - public void setRepairSubRangeNum(RepairType repairType, int repairSubRanges) - { - getOptions(repairType).number_of_subranges = repairSubRanges; - } - public DurationSpec.IntSecondsBound getRepairMinInterval(RepairType repairType) { return applyOverrides(repairType, opt -> opt.min_repair_interval); @@ -308,6 +307,26 @@ public ParameterizedClass getTokenRangeSplitter(RepairType repairType) return applyOverrides(repairType, opt -> opt.token_range_splitter); } + /** + * Set a new token range splitter, this is not meant to be used other than for testing. + */ + public void setTokenRangeSplitter(RepairType repairType, ParameterizedClass tokenRangeSplitter) + { + getOptions(repairType).token_range_splitter = tokenRangeSplitter; + tokenRangeSplitters.remove(repairType); + } + + public IAutoRepairTokenRangeSplitter getTokenRangeSplitterInstance(RepairType repairType) + { + IAutoRepairTokenRangeSplitter splitter = tokenRangeSplitters.get(repairType); + if (splitter == null) + { + splitter = newAutoRepairTokenRangeSplitter(repairType, getTokenRangeSplitter(repairType)); + tokenRangeSplitters.put(repairType, splitter); + } + return splitter; + } + public void setInitialSchedulerDelay(RepairType repairType, String initialSchedulerDelay) { getOptions(repairType).initial_scheduler_delay = new DurationSpec.IntSecondsBound(initialSchedulerDelay); @@ -328,6 +347,43 @@ public void setRepairSessionTimeout(RepairType repairType, String repairSessionT getOptions(repairType).repair_session_timeout = new DurationSpec.IntSecondsBound(repairSessionTimeout); } + @VisibleForTesting + static IAutoRepairTokenRangeSplitter newAutoRepairTokenRangeSplitter(AutoRepairConfig.RepairType repairType, ParameterizedClass parameterizedClass) throws ConfigurationException + { + try + { + Class tokenRangeSplitterClass; + final String className; + if (parameterizedClass.class_name != null && !parameterizedClass.class_name.isEmpty()) + { + className = parameterizedClass.class_name.contains(".") ? + parameterizedClass.class_name : + "org.apache.cassandra.repair.autorepair." + parameterizedClass.class_name; + tokenRangeSplitterClass = FBUtilities.classForName(className, "token_range_splitter"); + } + else + { + // If token_range_splitter.class_name is not defined, just use default, this is for convenience. + tokenRangeSplitterClass = AutoRepairConfig.DEFAULT_SPLITTER; + } + try + { + Map parameters = parameterizedClass.parameters != null ? parameterizedClass.parameters : Collections.emptyMap(); + // first attempt to initialize with RepairType and Map arguments. + return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor(AutoRepairConfig.RepairType.class, Map.class).newInstance(repairType, parameters); + } + catch (NoSuchMethodException nsme) + { + // fall back on no argument constructor. + return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor().newInstance(); + } + } + catch (Exception ex) + { + throw new ConfigurationException("Unable to create instance of IAutoRepairTokenRangeSplitter", ex); + } + } + // Options configures auto-repair behavior for a given repair type. // All fields can be modified dynamically. public static class Options implements Serializable @@ -347,7 +403,6 @@ protected static Options getDefaultOptions() opts.enabled = false; opts.repair_by_keyspace = false; - opts.number_of_subranges = 16; opts.number_of_repair_threads = 1; opts.parallel_repair_count = 3; opts.parallel_repair_percentage = 3; @@ -358,7 +413,7 @@ protected static Options getDefaultOptions() opts.force_repair_new_node = false; opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h"); opts.mv_repair_enabled = false; - opts.token_range_splitter = new ParameterizedClass(DefaultAutoRepairTokenSplitter.class.getName(), Collections.emptyMap()); + opts.token_range_splitter = new ParameterizedClass(DEFAULT_SPLITTER.getName(), Collections.emptyMap()); opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("5m"); // 5 minutes opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h"); // 3 hours @@ -369,13 +424,6 @@ protected static Options getDefaultOptions() public volatile Boolean enabled; // auto repair is default repair table by table, if this is enabled, the framework will repair all the tables in a keyspace in one go. public volatile Boolean repair_by_keyspace; - // the number of subranges to split each to-be-repaired token range into, - // the higher this number, the smaller the repair sessions will be - // How many subranges to divide one range into? The default is 1. - // If you are using v-node, say 256, then the repair will always go one v-node range at a time, this parameter, additionally, will let us further subdivide a given v-node range into sub-ranges. - // With the value “1” and v-nodes of 256, a given table on a node will undergo the repair 256 times. But with a value “2,” the same table on a node will undergo a repair 512 times because every v-node range will be further divided by two. - // If you do not use v-nodes or the number of v-nodes is pretty small, say 8, setting this value to a higher number, say 16, will be useful to repair on a smaller range, and the chance of succeeding is higher. - public volatile Integer number_of_subranges; // the number of repair threads to run for a given invoked Repair Job. // Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. // This is similar to -j for repair options for the nodetool repair command. @@ -422,8 +470,12 @@ protected static Options getDefaultOptions() // the default is 'true'. // This flag determines whether the auto-repair framework needs to run anti-entropy, a.k.a, repair on the MV table or not. public volatile Boolean mv_repair_enabled; - // the default is DefaultAutoRepairTokenSplitter. The class should implement IAutoRepairTokenRangeSplitter. - // The default implementation splits the tokens based on the token ranges owned by this node divided by the number of 'number_of_subranges' + /** + * Splitter implementation to use for generating repair assignments. + *

+ * The default is {@link RepairTokenRangeSplitter}. The class should implement {@link IAutoRepairTokenRangeSplitter} + * and have a constructor accepting ({@link RepairType}, {@link java.util.Map}) + */ public volatile ParameterizedClass token_range_splitter; // the minimum delay after a node starts before the scheduler starts running repair public volatile DurationSpec.IntSecondsBound initial_scheduler_delay; @@ -436,7 +488,6 @@ public String toString() return "Options{" + "enabled=" + enabled + ", repair_by_keyspace=" + repair_by_keyspace + - ", number_of_subranges=" + number_of_subranges + ", number_of_repair_threads=" + number_of_repair_threads + ", parallel_repair_count=" + parallel_repair_count + ", parallel_repair_percentage=" + parallel_repair_percentage + diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index f5678d1e88fe..c387d5a7345f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -430,7 +430,7 @@ protected static TreeSet getHostIdsInCurrentRing(RepairType repairType, Se } else { - logger.info("Node is not present in Gossipe cache node {}, node data center {}", node, nodeDC); + logger.info("Node is not present in Gossip cache node {}, node data center {}", node, nodeDC); } } return hostIdsInCurrentRing; diff --git a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java similarity index 55% rename from src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java rename to src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index 35d1b7ca2252..d95fdc9959b5 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/DefaultAutoRepairTokenSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -23,7 +23,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.NoSuchElementException; +import java.util.Map; import org.apache.cassandra.service.AutoRepairService; @@ -33,68 +33,49 @@ import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; -public class DefaultAutoRepairTokenSplitter implements IAutoRepairTokenRangeSplitter +public class FixedSplitTokenRangeSplitter implements IAutoRepairTokenRangeSplitter { - @Override - public Iterator getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, List repairPlans) + /** + * The number of subranges to split each to-be-repaired token range into, + * the higher this number, the smaller the repair sessions will be + * How many subranges to divide one range into? The default is 1. + * If you are using v-node, say 256, then the repair will always go one v-node range at a time, this parameter, additionally, will let us further subdivide a given v-node range into sub-ranges. + * With the value “1” and v-nodes of 256, a given table on a node will undergo the repair 256 times. But with a value “2,” the same table on a node will undergo a repair 512 times because every v-node range will be further divided by two. + * If you do not use v-nodes or the number of v-nodes is pretty small, say 8, setting this value to a higher number, say 16, will be useful to repair on a smaller range, and the chance of succeeding is higher. + */ + static final String NUMBER_OF_SUBRANGES = "number_of_subranges"; + + private final AutoRepairConfig.RepairType repairType; + private int numberOfSubranges; + + public FixedSplitTokenRangeSplitter(AutoRepairConfig.RepairType repairType, Map parameters) { - return new RepairAssignmentIterator(repairType, primaryRangeOnly, repairPlans); - } - - private class RepairAssignmentIterator implements Iterator - { - - private final AutoRepairConfig.RepairType repairType; - private final boolean primaryRangeOnly; + this.repairType = repairType; - private final Iterator repairPlanIterator; - - private Iterator currentIterator = null; - private PrioritizedRepairPlan currentPlan = null; - - RepairAssignmentIterator(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, List repairPlans) + if (parameters.containsKey(NUMBER_OF_SUBRANGES)) { - this.repairType = repairType; - this.primaryRangeOnly = primaryRangeOnly; - this.repairPlanIterator = repairPlans.iterator(); + numberOfSubranges = Integer.parseInt(parameters.get(NUMBER_OF_SUBRANGES)); } - - private synchronized Iterator currentIterator() + else { - if (currentIterator == null || !currentIterator.hasNext()) - { - // Advance the repair plan iterator if the current repair plan is exhausted, but only - // if there are more repair plans. - if (repairPlanIterator.hasNext()) - { - currentPlan = repairPlanIterator.next(); - currentIterator = currentPlan.getKeyspaceRepairPlans().iterator(); - } - } - - return currentIterator; + numberOfSubranges = 1; } + } - @Override - public boolean hasNext() - { - return currentIterator().hasNext(); - } + @Override + public Iterator getRepairAssignments(boolean primaryRangeOnly, List repairPlans) + { + return new RepairAssignmentIterator(repairPlans) { - @Override - public KeyspaceRepairAssignments next() - { - if (!currentIterator.hasNext()) + @Override + KeyspaceRepairAssignments nextInternal(int priority, KeyspaceRepairPlan repairPlan) { - throw new NoSuchElementException("No remaining repair plans"); + return getRepairAssignmentsForKeyspace(primaryRangeOnly, priority, repairPlan); } - - final KeyspaceRepairPlan repairPlan = currentIterator().next(); - return getRepairAssignmentsForKeyspace(repairType, primaryRangeOnly, currentPlan.getPriority(), repairPlan); - } + }; } - private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, int priority, KeyspaceRepairPlan repairPlan) + private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(boolean primaryRangeOnly, int priority, KeyspaceRepairPlan repairPlan) { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); List repairAssignments = new ArrayList<>(); @@ -107,7 +88,6 @@ private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(AutoRepairConf // if we need to repair non-primary token ranges, then change the tokens accrodingly tokens = StorageService.instance.getLocalReplicas(keyspaceName).ranges(); } - int numberOfSubranges = config.getRepairSubRangeNum(repairType); boolean byKeyspace = config.getRepairByKeyspace(repairType); // collect all token ranges. @@ -139,4 +119,31 @@ private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(AutoRepairConf return new KeyspaceRepairAssignments(priority, keyspaceName, repairAssignments); } + + @Override + public void setParameter(String key, String value) + { + if (key.equals(NUMBER_OF_SUBRANGES)) + { + setNumberOfSubranges(Integer.parseInt(value)); + } + throw new IllegalArgumentException("Unexpected parameter '" + key + "', must be " + NUMBER_OF_SUBRANGES); + } + + @Override + public Map getParameters() + { + return Map.of(NUMBER_OF_SUBRANGES, Integer.toString(getNumberOfSubranges())); + } + + public void setNumberOfSubranges(int numberOfSubranges) + { + this.numberOfSubranges = numberOfSubranges; + } + + public int getNumberOfSubranges() + { + return this.numberOfSubranges; + } + } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java index 854bc78a036d..86bac32c96bb 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java @@ -17,19 +17,55 @@ */ package org.apache.cassandra.repair.autorepair; +import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; +import org.apache.cassandra.config.ParameterizedClass; + +/** + * Interface that defines how to generate {@link KeyspaceRepairAssignments}. + *

+ * The default is {@link RepairTokenRangeSplitter} which aims to provide sensible defaults for all repair types. + *

+ * Custom implementations class should require a constructor accepting + * ({@link AutoRepairConfig.RepairType}, {@link java.util.Map}) with the {@link java.util.Map} parameter accepting + * custom configuration for your splitter. If such a constructor does not exist, + * {@link AutoRepair#newAutoRepairTokenRangeSplitter(AutoRepairConfig.RepairType, ParameterizedClass)} + * will fall back on invoking a default zero argument constructor. + */ public interface IAutoRepairTokenRangeSplitter { - /** * Split the token range you wish to repair into multiple assignments. - * The autorepair framework will repair the list of returned subrange in a sequence. - * @param repairType The type of repair being executed + * The autorepair framework will repair the assignments from returned subrange iterator in the sequence it's + * provided. * @param primaryRangeOnly Whether to repair only this node's primary ranges or all of its ranges. * @param repairPlans A list of ordered prioritized repair plans to generate assignments for in order. - * @return Iterator of repair assignments, with each element representing a grouping of repair assignments for a given keyspace. + * @return iterator of repair assignments, with each element representing a grouping of repair assignments for a given keyspace. + * The iterator is traversed lazily {@link KeyspaceRepairAssignments} at a time with the intent to try to get the + * most up-to-date representation of your data (e.g. how much data exists and is unrepaired at a given time). + */ + Iterator getRepairAssignments(boolean primaryRangeOnly, List repairPlans); + + /** + * Update a configuration parameter. This is meant to be used by nodetool setautorepairconfig to + * update configuration dynamically. + * @param key parameter to update + * @param value The value to set to. + */ + default void setParameter(String key, String value) + { + throw new IllegalArgumentException(this.getClass().getName() + " does not support custom configuration"); + } + + /** + * @return custom configuration. This is meant to be used by nodetool getautorepairconfig for + * retrieving the splitter configuration. */ - Iterator getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, List repairPlans); -} \ No newline at end of file + default Map getParameters() + { + return Collections.emptyMap(); + } +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java index 857fbe19b406..fbedb71b7c3c 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java +++ b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java @@ -33,7 +33,7 @@ /** * Encapsulates a devised plan to repair tables, grouped by their keyspace and a given priority. This is used * by {@link AutoRepair} to pass in an organized plan to - * {@link IAutoRepairTokenRangeSplitter#getRepairAssignments(AutoRepairConfig.RepairType, boolean, List)} which + * {@link IAutoRepairTokenRangeSplitter#getRepairAssignments(boolean, List)} which * can iterate over this plan in order to generate {@link RepairAssignment}s. */ public class PrioritizedRepairPlan diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java new file mode 100644 index 000000000000..5e6f8b61c3b8 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +/** + * Convenience {@link Iterator} implementation for to assist implementations of + * {@link IAutoRepairTokenRangeSplitter#getRepairAssignments(boolean, List)} by passing {@link KeyspaceRepairPlan} + * to a custom {@link #nextInternal(int, KeyspaceRepairPlan)} in priority order. + */ +public abstract class RepairAssignmentIterator implements Iterator +{ + private final Iterator repairPlanIterator; + + private Iterator currentIterator = null; + private PrioritizedRepairPlan currentPlan = null; + + RepairAssignmentIterator(List repairPlans) + { + this.repairPlanIterator = repairPlans.iterator(); + } + + private synchronized Iterator currentIterator() + { + if (currentIterator == null || !currentIterator.hasNext()) + { + // Advance the repair plan iterator if the current repair plan is exhausted, but only + // if there are more repair plans. + if (repairPlanIterator.hasNext()) + { + currentPlan = repairPlanIterator.next(); + currentIterator = currentPlan.getKeyspaceRepairPlans().iterator(); + } + } + + return currentIterator; + } + + @Override + public boolean hasNext() + { + return currentIterator().hasNext(); + } + + @Override + public KeyspaceRepairAssignments next() + { + if (!currentIterator.hasNext()) + { + throw new NoSuchElementException("No remaining repair plans"); + } + + final KeyspaceRepairPlan repairPlan = currentIterator().next(); + return nextInternal(currentPlan.getPriority(), repairPlan); + } + + /** + * Invoked by {@link #next()} with the next {@link KeyspaceRepairPlan} for the given priority. + * @param priority current priority being processed. + * @param repairPlan the next keyspace repair plan to process + * @return assignments for the given keyspace at this priority. Should never return null, if one desires to + * short-circuit the iterator, override {@link #hasNext()}. + */ + abstract KeyspaceRepairAssignments nextInternal(int priority, KeyspaceRepairPlan repairPlan); +} diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java similarity index 57% rename from src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java rename to src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index 31cc97590137..19f8a07af371 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -22,11 +22,13 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -61,163 +63,258 @@ import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; /** - * In Apache Cassandra, tuning the repair ranges has three main goals: - *

- * 1. **Create smaller, consistent repair times**: Long repairs, such as those lasting 15 hours, can be problematic. - * If a node fails 14 hours into the repair, the entire process must be restarted. The goal is to reduce the impact - * of disturbances or failures. However, making the repairs too short can lead to overhead from repair orchestration - * becoming the main bottleneck. - *

- * 2. **Minimize the impact on hosts**: Repairs should not heavily affect the host systems. For incremental repairs, - * this might involve anti-compaction work. In full repairs, streaming large amounts of data—especially with wide - * partitions—can lead to issues with disk usage and higher compaction costs. - *

- * 3. **Reduce overstreaming**: The Merkle tree, which represents data within each partition and range, has a maximum size. - * If a repair covers too many partitions, the tree’s leaves represent larger data ranges. Even a small change in a leaf - * can trigger excessive data streaming, making the process inefficient. - *

- * Additionally, if there are many small tables, it's beneficial to batch these tables together under a single parent repair. - * This prevents the repair overhead from becoming a bottleneck, especially when dealing with hundreds of tables. Running - * individual repairs for each table can significantly impact performance and efficiency. - *

- * To manage these issues, the strategy involves estimating the size and number of partitions within a range and splitting - * it accordingly to bound the size of the range splits. + * In Apache Cassandra, tuning repair ranges has four main goals: + *

    + *
  1. + * Create smaller, consistent repair times: Long repairs, such as those lasting 15 hours, can be problematic. + * If a node fails 14 hours into the repair, the entire process must be restarted. The goal is to reduce the impact + * of disturbances or failures. However, making the repairs too short can lead to overhead from repair orchestration + * becoming the main bottleneck. + *
    2. + *
  2. + * Minimize the impact on hosts: Repairs should not heavily affect the host systems. For incremental repairs, + * this might involve anti-compaction work. In full repairs, streaming large amounts of data—especially with wide + * partitions—can lead to issues with disk usage and higher compaction costs. + *
    3. + *
  3. + * Reduce overstreaming: The Merkle tree, which represents data within each partition and range, + * has a maximum size. If a repair covers too many partitions, the tree’s leaves represent larger data ranges. + * Even a small change in a leaf can trigger excessive data streaming, making the process inefficient. + *
    4. + *
  4. + * Reduce number of repairs: If there are many small tables, it's beneficial to batch these tables together + * under a single parent repair. This prevents the repair overhead from becoming a bottleneck, especially when + * dealing with hundreds of tables. Running individual repairs for each table can significantly impact performance + * and efficiency. + *
    5. + *
+ * To manage these issues, the strategy involves estimating the size and number of partitions within a range and + * splitting it accordingly to bound the size of the range splits. This is established by iterating over SSTable + * index files to estimate the amount of bytes and partitions involved in the ranges being repaired and by what + * repair type is beinb invoked. + *

+ * While this splitter has a lot of tuning parameters, the expectation is that the established default configuration + * shall be sensible for all {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType}'s. The following + * configuration parameters are offered. + *

    + *
  • + * bytes_per_assigment: The target and maximum amount of bytes that should be included in a repair + * assignment. This is meant to scope the amount of work involved in a repair. For incremental repair, this + * involves the total number of bytes in all SSTables containing unrepaired data involving the ranges being + * repaired, including data that doesn't cover the range. This is to account for the amount of anticompaction + * that is expected. For all other repair types, this involves the amount of data covering the range being + * repaired. + *
    • + *
  • + * partitions_per_assignment: The target number of partitions that should be included in a repair + * assignment. This configuration exists to reduce excessive overstreaming. + *
    • + *
  • + * max_tables_per_assignment: The maximum number of tables that can be included in a repair assignment. + * This aims to reduce the number of repairs, especially in cases where a large amount of tables exists for + * a keyspace. Note that the splitter will avoid batching tables together if they exceed the other + * configuration paramters such as bytes_per_assignment and partitions_per_assignment. + *
    • + *
  • + * max_bytes_per_schedule: The maximum number of bytes to cover an individiual schedule. This serves + * as a mechanism for throttling the amount of work that can be done on each repair cycle. One may opt to + * reduce this value if the impact of repairs is causing too many load on the cluster, or increase it if + * writes outpace the amount of data being repaired. Alternatively, one may want to choose tuning down or up + * the min_repair_interval. + *
    • + *
+ * Given the impact of what each repair type accomplishes, different defaults are established per repair type. + *
    + *
  • + * full: Configured in a way that attempts to accomplish repairing all data in a schedule, with + * individual repairs targeting at most 200GiB of data and 1048576 partitions. + *
      + *
    • bytes_per_assignment: 200GiB
      • + *
    • partitions_per_assignment: 2^repair_session_max_tree_depth (2^20 == 1048576 by default)
      • + *
    • max_tables_per_assignment: 64
      • + *
    • max_bytes_per_schedule: 100TiB
      • + *
    + *
    • + *
  • + * incremental: Configured in a way that attempts to repair 50GiB of data per repair, and 500GiB per + * schedule. This attempts to throttle the amount of IR and anticompaction done per schedule after turning + * incremental on for the first time. You may want to consider increasing max_bytes_per_schedule + * more than this much data is written per min_repair_interval. + *
      + *
    • bytes_per_assignment: 50GiB
      • + *
    • partitions_per_assignment: 2^repair_session_max_tree_depth (2^20 == 1048576 by default)
      • + *
    • max_tables_per_assignment: 64
      • + *
    • max_bytes_per_schedule: 500GiB
      • + *
    + *
    • + *
  • + * preview_repaired: Configured in a way that attempts to accomplish previewing all data in a schedule, + * with previews targeting at most 200GiB of data and 1048576 partitions. + *
      + *
    • bytes_per_assignment: 200GiB
      • + *
    • partitions_per_assignment: 2^repair_session_max_tree_depth (2^20 == 1048576 by default)
      • + *
    • max_tables_per_assignment: 64
      • + *
    • max_bytes_per_schedule: 100TiB
      • + *
    + *
    • + *
*/ - -public class RepairRangeSplitter implements IAutoRepairTokenRangeSplitter +public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter { - private static final Logger logger = LoggerFactory.getLogger(RepairRangeSplitter.class); + private static final Logger logger = LoggerFactory.getLogger(RepairTokenRangeSplitter.class); + + // Default max bytes to 100TiB, which is much more readable than Long.MAX_VALUE + private static final DataStorageSpec.LongBytesBound MAX_BYTES = new DataStorageSpec.LongBytesBound(100_000, DataStorageSpec.DataStorageUnit.GIBIBYTES); - static final String SUBRANGE_SIZE = "bytes_per_assignment"; - static final String PARTITION_COUNT = "partitions_per_assignment"; - static final String TABLE_BATCH_LIMIT = "max_tables_per_assignment"; + static final String BYTES_PER_ASSIGNMENT = "bytes_per_assignment"; + static final String PARTITIONS_PER_ASSIGNMENT = "partitions_per_assignment"; + static final String MAX_TABLES_PER_ASSIGNMENT = "max_tables_per_assignment"; static final String MAX_BYTES_PER_SCHEDULE = "max_bytes_per_schedule"; - private final int tablesPerAssignmentLimit; - private final long maxBytesPerSchedule; - private final long bytesPerSubrange; - private final long partitionsPerSubrange; + static final List PARAMETERS = List.of(BYTES_PER_ASSIGNMENT, PARTITIONS_PER_ASSIGNMENT, MAX_TABLES_PER_ASSIGNMENT, MAX_BYTES_PER_SCHEDULE); + + private final AutoRepairConfig.RepairType repairType; + + private final Map givenParameters = new HashMap<>(); - private static final long DEFAULT_SUBRANGE_SIZE = new DataStorageSpec.LongBytesBound("100GiB").toBytes(); - private static final long DEFAULT_MAX_BYTES_PER_SCHEDULE = Long.MAX_VALUE; - private static final long DEFAULT_PARTITION_LIMIT = (long) Math.pow(2, DatabaseDescriptor.getRepairSessionMaxTreeDepth()); - private static final int DEFAULT_TABLE_BATCH_LIMIT = 64; + private DataStorageSpec.LongBytesBound bytesPerAssignment; + private long partitionsPerAssignment; + private int maxTablesPerAssignment; + private DataStorageSpec.LongBytesBound maxBytesPerSchedule; - public RepairRangeSplitter(Map parameters) + /** + * Established default for each {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType}, meant to + * choose sensible defaults for each. + *

+ * Defaults if not specified for the given repair type: + *

  • + *
      bytes_per_assignment: 200GiB
    + *
      partitions_per_assignment: 2^repair_session_max_tree_depth
    + *
      max_tables_per_assignment: 64
    + *
      max_bytes_per_schedule: 1000GiB
    + *
    • + * It's expected that these defaults should work well for everything except incremental, so we confine + * bytes_per_assignment to 50GiB and max_bytes_per_schedule to 500GiB. This should strike a good balance + * between the amount of data that will be repaired during an initial migration to incremental repair and should + * move the entire repaired set from unrepaired to repaired at steady state, assuming not more the 500GiB of + * data is written to a node per min_repair_interval. + */ + private static final Map DEFAULTS_BY_REPAIR_TYPE = new EnumMap<>(AutoRepairConfig.RepairType.class) {{ + put(AutoRepairConfig.RepairType.FULL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.FULL) + .build()); + // Restrict incremental repair to 50GB bytes per assignment to confine the amount of possible autocompaction. + put(AutoRepairConfig.RepairType.INCREMENTAL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.INCREMENTAL) + .withBytesPerAssignment(new DataStorageSpec.LongBytesBound("50GiB")) + .withMaxBytesPerSchedule(new DataStorageSpec.LongBytesBound("500GiB")) + .build()); + put(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.PREVIEW_REPAIRED) + .build()); + }}; + + + public RepairTokenRangeSplitter(AutoRepairConfig.RepairType repairType, Map parameters) { - if (parameters.containsKey(SUBRANGE_SIZE)) + this.repairType = repairType; + this.givenParameters.putAll(parameters); + RepairTypeDefaults defaults = DEFAULTS_BY_REPAIR_TYPE.get(repairType); + + if (parameters.containsKey(BYTES_PER_ASSIGNMENT)) { - bytesPerSubrange = new DataStorageSpec.LongBytesBound(parameters.get(SUBRANGE_SIZE)).toBytes(); + bytesPerAssignment = new DataStorageSpec.LongBytesBound(parameters.get(BYTES_PER_ASSIGNMENT)); } else { - bytesPerSubrange = DEFAULT_SUBRANGE_SIZE; + bytesPerAssignment = defaults.bytesPerAssignment; } - if (parameters.containsKey(MAX_BYTES_PER_SCHEDULE)) + if (parameters.containsKey(PARTITIONS_PER_ASSIGNMENT)) { - maxBytesPerSchedule = new DataStorageSpec.LongBytesBound(parameters.get(MAX_BYTES_PER_SCHEDULE)).toBytes(); + partitionsPerAssignment = Long.parseLong(parameters.get(PARTITIONS_PER_ASSIGNMENT)); } else { - maxBytesPerSchedule = DEFAULT_MAX_BYTES_PER_SCHEDULE; + partitionsPerAssignment = defaults.partitionsPerAssignment; } - if (parameters.containsKey(PARTITION_COUNT)) + if (parameters.containsKey(MAX_TABLES_PER_ASSIGNMENT)) { - partitionsPerSubrange = Long.parseLong(parameters.get(PARTITION_COUNT)); + maxTablesPerAssignment = Integer.parseInt(parameters.get(MAX_TABLES_PER_ASSIGNMENT)); } - else - { - partitionsPerSubrange = DEFAULT_PARTITION_LIMIT; + else{ + maxTablesPerAssignment = defaults.maxTablesPerAssignment; } - if (parameters.containsKey(TABLE_BATCH_LIMIT)) + if (parameters.containsKey(MAX_BYTES_PER_SCHEDULE)) { - tablesPerAssignmentLimit = Integer.parseInt(parameters.get(TABLE_BATCH_LIMIT)); + maxBytesPerSchedule = new DataStorageSpec.LongBytesBound(parameters.get(MAX_BYTES_PER_SCHEDULE)); } else { - tablesPerAssignmentLimit = DEFAULT_TABLE_BATCH_LIMIT; + maxBytesPerSchedule = defaults.maxBytesPerSchedule; } - if (bytesPerSubrange > maxBytesPerSchedule) + + if (bytesPerAssignment.toBytes() > maxBytesPerSchedule.toBytes()) { - throw new IllegalArgumentException(String.format("%s='%s' cannot be greater than %s='%s'", - SUBRANGE_SIZE, - FileUtils.stringifyFileSize(bytesPerSubrange), + throw new IllegalArgumentException(String.format("%s='%s' cannot be greater than %s='%s' for %s", + BYTES_PER_ASSIGNMENT, + bytesPerAssignment.toBytes(), MAX_BYTES_PER_SCHEDULE, - FileUtils.stringifyFileSize(maxBytesPerSchedule))); + maxBytesPerSchedule, + repairType.getConfigName())); } - logger.info("Configured {} with {}={}, {}={}, {}={}, {}={}", RepairRangeSplitter.class.getName(), - SUBRANGE_SIZE, FileUtils.stringifyFileSize(bytesPerSubrange), - MAX_BYTES_PER_SCHEDULE, FileUtils.stringifyFileSize(maxBytesPerSchedule), - PARTITION_COUNT, partitionsPerSubrange, TABLE_BATCH_LIMIT, tablesPerAssignmentLimit); + logger.info("Configured {}[{}] with {}={}, {}={}, {}={}, {}={}", RepairTokenRangeSplitter.class.getName(), + repairType.getConfigName(), + BYTES_PER_ASSIGNMENT, bytesPerAssignment, + PARTITIONS_PER_ASSIGNMENT, partitionsPerAssignment, + MAX_TABLES_PER_ASSIGNMENT, maxTablesPerAssignment, + MAX_BYTES_PER_SCHEDULE, maxBytesPerSchedule); } - @Override - public Iterator getRepairAssignments(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, List repairPlans) + public AutoRepairConfig.RepairType getRepairType() { - return new RepairAssignmentIterator(repairType, primaryRangeOnly, repairPlans); + return repairType; } - private class RepairAssignmentIterator implements Iterator - { - private final AutoRepairConfig.RepairType repairType; - private final boolean primaryRangeOnly; + @Override + public Iterator getRepairAssignments(boolean primaryRangeOnly, List repairPlans) + { + return new BytesBasedRepairAssignmentIterator(primaryRangeOnly, repairPlans); + } - private final Iterator repairPlanIterator; + /** + * A custom {@link RepairAssignmentIterator} that confines the number of repair assignments to + * max_bytes_per_schedule. + */ + private class BytesBasedRepairAssignmentIterator extends RepairAssignmentIterator { - private Iterator currentIterator = null; - private PrioritizedRepairPlan currentPlan = null; + private final boolean primaryRangeOnly; private long bytesSoFar = 0; - RepairAssignmentIterator(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, List repairPlans) + BytesBasedRepairAssignmentIterator(boolean primaryRangeOnly, List repairPlans) { - this.repairType = repairType; + super(repairPlans); this.primaryRangeOnly = primaryRangeOnly; - this.repairPlanIterator = repairPlans.iterator(); - } - - private synchronized Iterator currentIterator() - { - if (currentIterator == null || !currentIterator.hasNext()) - { - // Advance the repair plan iterator if the current repair plan is exhausted, but only - // if there are more repair plans. - if (repairPlanIterator.hasNext()) - { - currentPlan = repairPlanIterator.next(); - currentIterator = currentPlan.getKeyspaceRepairPlans().iterator(); - } - } - - return currentIterator; - } - - @Override - public boolean hasNext() - { - return currentIterator().hasNext(); } @Override - public KeyspaceRepairAssignments next() + KeyspaceRepairAssignments nextInternal(int priority, KeyspaceRepairPlan repairPlan) { - // Should not happen unless violating the contract of iterator of checking hasNext first. - if (!currentIterator.hasNext()) + // short circuit if we've accumulated too many bytes by returning a KeyspaceRepairAssignments with + // no assignments. We do this rather than returning false in hasNext() because we want to signal + // to AutoRepair that a keyspace generated no assignments. + if (bytesSoFar >= maxBytesPerSchedule.toBytes()) { - throw new NoSuchElementException("No remaining repair plans"); + return new KeyspaceRepairAssignments(priority, repairPlan.getKeyspaceName(), Collections.emptyList()); } - final KeyspaceRepairPlan repairPlan = currentIterator().next(); Collection> tokenRanges = getTokenRanges(primaryRangeOnly, repairPlan.getKeyspaceName()); List repairAssignments = getRepairAssignmentsForKeyspace(repairType, repairPlan.getKeyspaceName(), repairPlan.getTableNames(), tokenRanges); - FilteredRepairAssignments filteredRepairAssignments = filterRepairAssignments(repairType, currentPlan.getPriority(), repairPlan.getKeyspaceName(), repairAssignments, bytesSoFar); + FilteredRepairAssignments filteredRepairAssignments = filterRepairAssignments(priority, repairPlan.getKeyspaceName(), repairAssignments, bytesSoFar); bytesSoFar = filteredRepairAssignments.newBytesSoFar; - return new KeyspaceRepairAssignments(currentPlan.getPriority(), repairPlan.getKeyspaceName(), filteredRepairAssignments.repairAssignments); + return new KeyspaceRepairAssignments(priority, repairPlan.getKeyspaceName(), filteredRepairAssignments.repairAssignments); } } @@ -245,7 +342,7 @@ List getRepairAssignmentsForKeyspace(AutoRepairConfig.Rep for (String tableName : tableNames) { - List tableAssignments = getRepairAssignmentsForTable(repairType, keyspaceName, tableName, tokenRanges); + List tableAssignments = getRepairAssignmentsForTable(keyspaceName, tableName, tokenRanges); if (tableAssignments.isEmpty()) continue; @@ -257,13 +354,13 @@ List getRepairAssignmentsForKeyspace(AutoRepairConfig.Rep } // If the table assignments are for the same token range, and we have room to add more tables to the current assignment else if (tableAssignments.size() == 1 && - currentAssignments.size() < tablesPerAssignmentLimit && - (currentAssignments.isEmpty() || currentAssignments.get(0).getTokenRange().equals(tableAssignments.get(0).getTokenRange()))) + currentAssignments.size() < maxTablesPerAssignment && + (currentAssignments.isEmpty() || currentAssignments.get(0).getTokenRange().equals(tableAssignments.get(0).getTokenRange()))) { long currentAssignmentsBytes = getEstimatedBytes(currentAssignments); long tableAssignmentsBytes = getEstimatedBytes(tableAssignments); // only add assignments together if they don't exceed max bytes per schedule. - if (currentAssignmentsBytes + tableAssignmentsBytes < maxBytesPerSchedule) { + if (currentAssignmentsBytes + tableAssignmentsBytes < maxBytesPerSchedule.toBytes()) { currentAssignments.addAll(tableAssignments); } else @@ -291,13 +388,12 @@ else if (tableAssignments.size() == 1 && /** * Given a repair type and map of sized-based repair assignments, confine them by maxBytesPerSchedule. - * @param repairType used to determine underyling table priorities * @param repairAssignments the assignments to filter. * @param bytesSoFar repair assignment bytes accumulated so far. * @return A list of repair assignments confined by maxBytesPerSchedule. */ @VisibleForTesting - FilteredRepairAssignments filterRepairAssignments(AutoRepairConfig.RepairType repairType, int priority, String keyspaceName, List repairAssignments, long bytesSoFar) + FilteredRepairAssignments filterRepairAssignments(int priority, String keyspaceName, List repairAssignments, long bytesSoFar) { // Confine repair assignments by maxBytesPerSchedule. long bytesSoFarThisIteration = 0L; @@ -311,7 +407,7 @@ FilteredRepairAssignments filterRepairAssignments(AutoRepairConfig.RepairType re { totalAssignments++; // skip any repair assignments that would accumulate us past the maxBytesPerSchedule - if (bytesSoFar + repairAssignment.getEstimatedBytes() > maxBytesPerSchedule) + if (bytesSoFar + repairAssignment.getEstimatedBytes() > maxBytesPerSchedule.toBytes()) { // log that repair assignment was skipped. bytesNotRepaired += repairAssignment.getEstimatedBytes(); @@ -379,11 +475,12 @@ private FilteredRepairAssignments(List repairAssignments, long } } - private String getBytesOfMaxBytesPerSchedule(long bytes) { - if (maxBytesPerSchedule == Long.MAX_VALUE) + private String getBytesOfMaxBytesPerSchedule(long bytes) + { + if (maxBytesPerSchedule.equals(MAX_BYTES)) return FileUtils.stringifyFileSize(bytes); else - return String.format("%s of %s", FileUtils.stringifyFileSize(bytes), FileUtils.stringifyFileSize(maxBytesPerSchedule)); + return String.format("%s of %s", FileUtils.stringifyFileSize(bytes), maxBytesPerSchedule); } /** @@ -427,9 +524,9 @@ static SizedRepairAssignment merge(List assignments) } @VisibleForTesting - protected List getRepairAssignmentsForTable(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName, Collection> tokenRanges) + protected List getRepairAssignmentsForTable(String keyspaceName, String tableName, Collection> tokenRanges) { - List sizeEstimates = getRangeSizeEstimate(repairType, keyspaceName, tableName, tokenRanges); + List sizeEstimates = getRangeSizeEstimate(keyspaceName, tableName, tokenRanges); return getRepairAssignments(sizeEstimates); } @@ -461,7 +558,7 @@ protected List getRepairAssignments(List si else { // Check if the estimate needs splitting based on the criteria - boolean needsSplitting = estimate.sizeForRepair > bytesPerSubrange || estimate.partitions > partitionsPerSubrange; + boolean needsSplitting = estimate.sizeForRepair > bytesPerAssignment.toBytes() || estimate.partitions > partitionsPerAssignment; if (needsSplitting) { int numberOfSplits = calculateNumberOfSplits(estimate); @@ -491,8 +588,8 @@ protected List getRepairAssignments(List si private int calculateNumberOfSplits(SizeEstimate estimate) { // Calculate the number of splits needed for size and partitions - int splitsForSize = (int) Math.ceil((double) estimate.sizeForRepair / bytesPerSubrange); - int splitsForPartitions = (int) Math.ceil((double) estimate.partitions / partitionsPerSubrange); + int splitsForSize = (int) Math.ceil((double) estimate.sizeForRepair / bytesPerAssignment.toBytes()); + int splitsForPartitions = (int) Math.ceil((double) estimate.partitions / partitionsPerAssignment); // Split the token range into subranges based on whichever (partitions, bytes) would generate the most splits. boolean splitBySize = splitsForSize > splitsForPartitions; @@ -536,7 +633,7 @@ private Collection> getTokenRanges(boolean primaryRangeOnly, String return ranges; } - private List getRangeSizeEstimate(AutoRepairConfig.RepairType repairType, String keyspace, String table, Collection> tokenRanges) + private List getRangeSizeEstimate(String keyspace, String table, Collection> tokenRanges) { List sizeEstimates = new ArrayList<>(); for (Range tokenRange : tokenRanges) @@ -638,6 +735,111 @@ static Refs getSSTableReaderRefs(AutoRepairConfig.RepairType repa return refs; } + @Override + public void setParameter(String key, String value) + { + switch (key) + { + case BYTES_PER_ASSIGNMENT: + setBytesPerAssignment(new DataStorageSpec.LongBytesBound(value)); + givenParameters.put(key, value); + return; + case PARTITIONS_PER_ASSIGNMENT: + setPartitionsPerAssignment(Long.parseLong(value)); + givenParameters.put(key, value); + return; + case MAX_TABLES_PER_ASSIGNMENT: + setMaxTablesPerAssignment(Integer.parseInt(value)); + givenParameters.put(key, value); + return; + case MAX_BYTES_PER_SCHEDULE: + setMaxBytesPerSchedule(new DataStorageSpec.LongBytesBound(value)); + givenParameters.put(key, value); + return; + default: + throw new IllegalArgumentException("Unexpected parameter '" + key + "', must be one of " + + PARAMETERS); + } + } + + @Override + public Map getParameters() + { + final Map parameters = new LinkedHashMap<>(); + for (String parameter : PARAMETERS) + { + // Use the parameter as configured if it exists, user would likely to prefer to see something more readable. + if (givenParameters.containsKey(parameter)) + { + parameters.put(parameter, givenParameters.get(parameter)); + continue; + } + + switch (parameter) + { + case BYTES_PER_ASSIGNMENT: + parameters.put(parameter, getBytesPerAssignment().toString()); + continue; + case PARTITIONS_PER_ASSIGNMENT: + parameters.put(parameter, Long.toString(getPartitionsPerAssignment())); + continue; + case MAX_TABLES_PER_ASSIGNMENT: + parameters.put(parameter, Integer.toString(getMaxTablesPerAssignment())); + continue; + case MAX_BYTES_PER_SCHEDULE: + parameters.put(parameter, getMaxBytesPerSchedule().toString()); + continue; + default: + // not expected + parameters.put(parameter, ""); + } + } + return parameters; + } + + public DataStorageSpec.LongBytesBound getBytesPerAssignment() + { + return bytesPerAssignment; + } + + public void setBytesPerAssignment(DataStorageSpec.LongBytesBound bytesPerAssignment) + { + this.bytesPerAssignment = bytesPerAssignment; + } + + public long getPartitionsPerAssignment() + { + return partitionsPerAssignment; + } + + public void setPartitionsPerAssignment(long partitionsPerAssignment) + { + this.partitionsPerAssignment = partitionsPerAssignment; + } + + public int getMaxTablesPerAssignment() + { + return maxTablesPerAssignment; + } + + public void setMaxTablesPerAssignment(int maxTablesPerAssignment) + { + this.maxTablesPerAssignment = maxTablesPerAssignment; + } + + public DataStorageSpec.LongBytesBound getMaxBytesPerSchedule() + { + return maxBytesPerSchedule; + } + + public void setMaxBytesPerSchedule(DataStorageSpec.LongBytesBound maxBytesPerSchedule) + { + this.maxBytesPerSchedule = maxBytesPerSchedule; + } + + /** + * Represents a size estimate by both bytes and partition count for a given keyspace and table for a token range. + */ @VisibleForTesting protected static class SizeEstimate { @@ -754,4 +956,79 @@ public String toString() '}'; } } + + /** + * Conveinence builder for establishing defaults by repair type. + */ + protected static class RepairTypeDefaults + { + final AutoRepairConfig.RepairType repairType; + final DataStorageSpec.LongBytesBound bytesPerAssignment; + final long partitionsPerAssignment; + final int maxTablesPerAssignment; + final DataStorageSpec.LongBytesBound maxBytesPerSchedule; + + public RepairTypeDefaults(AutoRepairConfig.RepairType repairType, + DataStorageSpec.LongBytesBound bytesPerAssignment, + long partitionsPerAssignment, + int maxTablesPerAssignment, + DataStorageSpec.LongBytesBound maxBytesPerSchedule) + { + this.repairType = repairType; + this.bytesPerAssignment = bytesPerAssignment; + this.partitionsPerAssignment = partitionsPerAssignment; + this.maxTablesPerAssignment = maxTablesPerAssignment; + this.maxBytesPerSchedule = maxBytesPerSchedule; + } + + static RepairTypeDefaultsBuilder builder(AutoRepairConfig.RepairType repairType) + { + return new RepairTypeDefaultsBuilder(repairType); + } + + static class RepairTypeDefaultsBuilder + { + private final AutoRepairConfig.RepairType repairType; + private DataStorageSpec.LongBytesBound bytesPerAssignment = new DataStorageSpec.LongBytesBound("200GiB"); + private long partitionsPerAssignment = (long) Math.pow(2, DatabaseDescriptor.getRepairSessionMaxTreeDepth()); + private int maxTablesPerAssignment = 64; + private DataStorageSpec.LongBytesBound maxBytesPerSchedule = MAX_BYTES; + + private RepairTypeDefaultsBuilder(AutoRepairConfig.RepairType repairType) + { + this.repairType = repairType; + } + + public RepairTypeDefaultsBuilder withBytesPerAssignment(DataStorageSpec.LongBytesBound bytesPerAssignment) + { + this.bytesPerAssignment = bytesPerAssignment; + return this; + } + + @SuppressWarnings("unused") + public RepairTypeDefaultsBuilder withPartitionsPerAssignment(long partitionsPerAssignment) + { + this.partitionsPerAssignment = partitionsPerAssignment; + return this; + } + + @SuppressWarnings("unused") + public RepairTypeDefaultsBuilder withMaxTablesPerAssignment(int maxTablesPerAssignment) + { + this.maxTablesPerAssignment = maxTablesPerAssignment; + return this; + } + + public RepairTypeDefaultsBuilder withMaxBytesPerSchedule(DataStorageSpec.LongBytesBound maxBytesPerSchedule) + { + this.maxBytesPerSchedule = maxBytesPerSchedule; + return this; + } + + public RepairTokenRangeSplitter.RepairTypeDefaults build() + { + return new RepairTypeDefaults(repairType, bytesPerAssignment, partitionsPerAssignment, maxTablesPerAssignment, maxBytesPerSchedule); + } + } + } } diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 167e4507a7c1..1a9c010928a7 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -27,6 +27,7 @@ import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; @@ -107,12 +108,6 @@ public void setForceRepairForHosts(RepairType repairType, Set getOnGoingRepairHostIds(RepairType rType) } return hostIds; } + + @Override + public Map getAutoRepairTokenRangeSplitterParameters(RepairType repairType) + { + return config.getTokenRangeSplitterInstance(repairType).getParameters(); + } + + @Override + public void setAutoRepairTokenRangeSplitterParameter(RepairType repairType, String key, String value) + { + config.getTokenRangeSplitterInstance(repairType).setParameter(key, value); + } } diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index cac2680c3f01..e69a45482bc0 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -21,6 +21,7 @@ import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import java.util.Map; import java.util.Set; public interface AutoRepairServiceMBean @@ -38,8 +39,6 @@ public interface AutoRepairServiceMBean public Set getRepairHostPriority(RepairType repairType); - public void setRepairSubRangeNum(RepairType repairType, int repairSubRangeNum); - public void setRepairMinInterval(RepairType repairType, String minRepairInterval); void startScheduler(); @@ -71,4 +70,8 @@ public interface AutoRepairServiceMBean public void setRepairSessionTimeout(RepairType repairType, String timeout); public Set getOnGoingRepairHostIds(RepairType rType); + + public Map getAutoRepairTokenRangeSplitterParameters(RepairType repairType); + + public void setAutoRepairTokenRangeSplitterParameter(AutoRepairConfig.RepairType repairType, String key, String value); } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index ea2fc6acde34..313ce99944a6 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2151,6 +2151,16 @@ public AutoRepairConfig getAutoRepairConfig() { return autoRepairProxy.getAutoRepairConfig(); } + public Map getAutoRepairTokenRangeSplitterParameters(AutoRepairConfig.RepairType repairType) + { + return autoRepairProxy.getAutoRepairTokenRangeSplitterParameters(repairType); + } + + public void setAutoRepairTokenRangeSplitterParameter(AutoRepairConfig.RepairType repairType, String key, String value) + { + autoRepairProxy.setAutoRepairTokenRangeSplitterParameter(repairType, key, value); + } + public void setAutoRepairEnabled(AutoRepairConfig.RepairType repairType, boolean enabled) { autoRepairProxy.setAutoRepairEnabled(repairType, enabled); @@ -2175,11 +2185,6 @@ public void setForceRepairForHosts(AutoRepairConfig.RepairType repairType, Set priorityHosts = probe.getRepairPriorityForHosts(repairType); + if (!priorityHosts.isEmpty()) + { + appendConfig(sb, "priority_hosts", Joiner.on(',').skipNulls().join(priorityHosts)); + } + + appendConfig(sb , "min_repair_interval", config.getRepairMinInterval(repairType)); + appendConfig(sb , "repair_by_keyspace", config.getRepairByKeyspace(repairType)); + appendConfig(sb , "number_of_repair_threads", config.getRepairThreads(repairType)); + appendConfig(sb , "sstable_upper_threshold", config.getRepairSSTableCountHigherThreshold(repairType)); + appendConfig(sb , "table_max_repair_time", config.getAutoRepairTableMaxRepairTime(repairType)); + appendConfig(sb , "ignore_dcs", config.getIgnoreDCs(repairType)); + appendConfig(sb , "repair_primary_token_range_only", config.getRepairPrimaryTokenRangeOnly(repairType)); + appendConfig(sb , "parallel_repair_count", config.getParallelRepairCount(repairType)); + appendConfig(sb , "parallel_repair_percentage", config.getParallelRepairPercentage(repairType)); + appendConfig(sb , "mv_repair_enabled", config.getMVRepairEnabled(repairType)); + appendConfig(sb , "initial_scheduler_delay", config.getInitialSchedulerDelay(repairType)); + appendConfig(sb , "repair_session_timeout", config.getRepairSessionTimeout(repairType)); + appendConfig(sb , "force_repair_new_node", config.getForceRepairNewNode(repairType)); + + final ParameterizedClass splitterClass = config.getTokenRangeSplitter(repairType); + final String splitterClassName = splitterClass.class_name != null ? splitterClass.class_name : AutoRepairConfig.DEFAULT_SPLITTER.getName(); + appendConfig(sb, "token_range_splitter", splitterClassName); + Map tokenRangeSplitterParameters = probe.getAutoRepairTokenRangeSplitterParameters(repairType); + if (!tokenRangeSplitterParameters.isEmpty()) + { + for (Map.Entry param : tokenRangeSplitterParameters.entrySet()) + { + appendConfig(sb, String.format("token_range_splitter.%s", param.getKey()), param.getValue()); + } + } + } return sb.toString(); } + + private void appendConfig(StringBuilder sb, String config, T value) + { + sb.append(String.format("%s%s: %s", "\n\t", config, value)); + } } diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index 75cdea6cd1c0..d62b5b1d95db 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -43,21 +43,23 @@ public class SetAutoRepairConfig extends NodeToolCmd @VisibleForTesting @Arguments(title = " ", usage = " ", description = "autorepair param and value.\nPossible autorepair parameters are as following: " + - "[start_scheduler|number_of_repair_threads|number_of_subranges|min_repair_interval|sstable_upper_threshold" + + "[start_scheduler|number_of_repair_threads|min_repair_interval|sstable_upper_threshold" + "|enabled|table_max_repair_time|priority_hosts|forcerepair_hosts|ignore_dcs" + "|history_clear_delete_hosts_buffer_interval|repair_primary_token_range_only" + "|parallel_repair_count|parallel_repair_percentage|mv_repair_enabled|repair_max_retries" + - "|repair_retry_backoff|repair_session_timeout|min_repair_task_duration]", + "|repair_retry_backoff|repair_session_timeout|min_repair_task_duration|token_range_splitter.]", required = true) protected List args = new ArrayList<>(); @VisibleForTesting @Option(title = "repair type", name = { "-t", "--repair-type" }, description = "Repair type") - protected RepairType repairType; + protected String repairTypeStr; @VisibleForTesting protected PrintStream out = System.out; + private static final String TOKEN_RANGE_SPLITTER_PROPERTY_PREFIX = "token_range_splitter."; + @Override public void execute(NodeProbe probe) { @@ -98,7 +100,16 @@ public void execute(NodeProbe probe) } // options below require --repair-type option - checkArgument(repairType != null, "--repair-type is required for this parameter."); + checkArgument(repairTypeStr != null, "--repair-type is required for this parameter."); + final RepairType repairType = RepairType.valueOf(repairTypeStr.toUpperCase()); + + if(paramType.startsWith(TOKEN_RANGE_SPLITTER_PROPERTY_PREFIX)) + { + final String key = paramType.replace(TOKEN_RANGE_SPLITTER_PROPERTY_PREFIX, ""); + probe.setAutoRepairTokenRangeSplitterParameter(repairType, key, paramVal); + return; + } + Set hosts; switch (paramType) { @@ -108,9 +119,6 @@ public void execute(NodeProbe probe) case "number_of_repair_threads": probe.setRepairThreads(repairType, Integer.parseInt(paramVal)); break; - case "number_of_subranges": - probe.setRepairSubRangeNum(repairType, Integer.parseInt(paramVal)); - break; case "min_repair_interval": probe.setRepairMinInterval(repairType, paramVal); break; diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index c9b8233c6ffc..3d881ddf3f59 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -50,11 +50,9 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileInputStreamPlus; import org.apache.cassandra.io.util.FileOutputStreamPlus; -import org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter; import org.apache.cassandra.utils.concurrent.*; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -687,33 +685,6 @@ public static ISslContextFactory newSslContextFactory(String className, Map tokenRangeSplitterClass = Class.forName(className); - try - { - Map parameters = parameterizedClass.parameters != null ? parameterizedClass.parameters : Collections.emptyMap(); - // first attempt to initialize with Map arguments. - return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor(Map.class).newInstance(parameters); - } - catch (NoSuchMethodException nsme) - { - // fall back on no argument constructor. - return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor().newInstance(); - } - } - catch (Exception ex) - { - throw new ConfigurationException("Unable to create instance of IAutoRepairTokenRangeSplitter for " + className, ex); - } - } - /** * @return The Class for the given name. * @param classname Fully qualified classname. diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index 6cf55299abc9..31664fb6a886 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -75,6 +75,8 @@ public static void init() throws IOException "parallel_repair_percentage", "0", "min_repair_interval", "1s")))) .set("auto_repair.enabled", "true") + .set("auto_repair.global_settings.repair_by_keyspace", "true") + .set("auto_repair.repair_task_min_duration", "0s") .set("auto_repair.repair_check_interval", "10s")).start(); cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); @@ -123,12 +125,8 @@ private void validate(String repairType) throws ParseException sdf.parse(row[2].toString()); sdf.parse(row[3].toString()); // the reason why the repair was scheduled + Assert.assertNotNull(row[4]); Assert.assertEquals("MY_TURN", row[4].toString()); - for (Object col : row) - { - System.out.println("Data:" + col); - } - System.out.println("====================================="); } } } diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java index 48cadcbc64ca..c0893d1c6b46 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java @@ -91,12 +91,14 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.config.Config$CorruptedTombstoneStrategy", "org.apache.cassandra.repair.autorepair.AutoRepairConfig", "org.apache.cassandra.repair.autorepair.AutoRepairConfig$Options", - "org.apache.cassandra.repair.autorepair.DefaultAutoRepairTokenSplitter", - "org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter", - "org.apache.cassandra.repair.autorepair.FullRepairState", - "org.apache.cassandra.repair.autorepair.IncrementalRepairState", "org.apache.cassandra.repair.autorepair.AutoRepairConfig$RepairType", "org.apache.cassandra.repair.autorepair.AutoRepairState", + "org.apache.cassandra.repair.autorepair.FixedSplitTokenRangeSplitter", + "org.apache.cassandra.repair.autorepair.FullRepairState", + "org.apache.cassandra.repair.autorepair.IAutoRepairTokenRangeSplitter", + "org.apache.cassandra.repair.autorepair.IncrementalRepairState", + "org.apache.cassandra.repair.autorepair.PreviewRepairedState", + "org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter", "org.apache.cassandra.config.DatabaseDescriptor$ByteUnit", "org.apache.cassandra.config.DataRateSpec", "org.apache.cassandra.config.DataRateSpec$DataRateUnit", diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraSreamReceiverTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamReceiverTest.java similarity index 71% rename from test/unit/org/apache/cassandra/db/streaming/CassandraSreamReceiverTest.java rename to test/unit/org/apache/cassandra/db/streaming/CassandraStreamReceiverTest.java index 43020feac991..3b9b3b271206 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraSreamReceiverTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamReceiverTest.java @@ -19,9 +19,9 @@ package org.apache.cassandra.db.streaming; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.db.ColumnFamilyStore; @@ -35,7 +35,7 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.when; -public class CassandraSreamReceiverTest extends CQLTester +public class CassandraStreamReceiverTest extends CQLTester { @Mock private StreamSession session; @@ -45,15 +45,15 @@ public class CassandraSreamReceiverTest extends CQLTester private static final String CDC_MV_TABLE = "cdc_mv_table"; private static final String NO_CDC_MV_TABLE = "no_cdc_mv_table"; - @BeforeClass - public static void beforeClass() - { - System.setProperty("cassandra.streaming.requires_cdc_replay", "false"); - } - @Before public void setup() { + // Set cdc_on_repair_enabled materialized_views_on_repair to true + DatabaseDescriptor.setCDCOnRepairEnabled(true); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); + // Enable materialized views + DatabaseDescriptor.setMaterializedViewsEnabled(true); + MockitoAnnotations.initMocks(this); QueryProcessor.executeInternal(String.format("CREATE TABLE IF NOT EXISTS %s.%s (pk int PRIMARY KEY, v int) WITH cdc=true", KEYSPACE, CDC_TABLE)); QueryProcessor.executeInternal(String.format("CREATE TABLE IF NOT EXISTS %s.%s (pk int PRIMARY KEY, v int) WITH cdc=false", KEYSPACE, MV_TABLE)); @@ -66,26 +66,35 @@ public void setup() @Test public void testRequiresWritePathRepair() { + // given a CDC table with a materialized view attached to it. ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CDC_MV_TABLE); when(session.streamOperation()).thenReturn(StreamOperation.REPAIR); CassandraStreamReceiver receiver = new CassandraStreamReceiver(cfs, session, 1); + // Should require write path since cdc_on_repair_enabled and materialized_views_on_repair_enabled are both true. assertTrue(receiver.requiresWritePath(cfs)); } @Test public void testRequiresWritePathBulkLoad() { + // given a CDC table with a materialized view attached to it. ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CDC_MV_TABLE); when(session.streamOperation()).thenReturn(StreamOperation.BULK_LOAD); CassandraStreamReceiver receiver = new CassandraStreamReceiver(cfs, session, 1); + // Should require write path since cdc_on_repair_enabled and materialized_views_on_repair_enabled are both true. assertTrue(receiver.requiresWritePath(cfs)); } @Test - public void testRequiresWritePathNoCDCOrMV() + public void testDoesNotRequireWritePathNoCDCOrMV() { + // Given cdc_on_repaired_enabled and materialized_views_on_repair_enabled are false + // requiresWritePath should still return false for a non-CDC table. + DatabaseDescriptor.setCDCOnRepairEnabled(false); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(NO_CDC_MV_TABLE); when(session.streamOperation()).thenReturn(StreamOperation.BULK_LOAD); CassandraStreamReceiver receiver = new CassandraStreamReceiver(cfs, session, 1); @@ -96,7 +105,8 @@ public void testRequiresWritePathNoCDCOrMV() @Test public void testRequiresWritePathRepairMVOnly() { - // Access the private field using reflection + // Given cdc_on_repaired_enabled and materialized_views_on_repair_enabled are true + // requiresWritePath should return true for a table with materialized views. ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(MV_TABLE); when(session.streamOperation()).thenReturn(StreamOperation.REPAIR); CassandraStreamReceiver receiver = new CassandraStreamReceiver(cfs, session, 1); @@ -105,10 +115,10 @@ public void testRequiresWritePathRepairMVOnly() } @Test - public void testRequiresWritePathRepairCDCWithSystemProp() + public void testRequiresWritePathRepairCDCOnRepairEnabled() { - System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); - + // Given cdc_on_repaired_enabled and materialized_views_on_repair_enabled are true + // requiresWritePath should return true for a table with CDC enabled. ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CDC_TABLE); when(session.streamOperation()).thenReturn(StreamOperation.REPAIR); CassandraStreamReceiver receiver = new CassandraStreamReceiver(cfs, session, 1); @@ -119,16 +129,22 @@ public void testRequiresWritePathRepairCDCWithSystemProp() } @Test - public void testDoesNotRequiresWritePathRepairCDCOnly() + public void testDoesNotRequireWritePathRepairCDCOnRepairEnabledFalse() { + // Given cdc_on_repaired_enabled and materialized_views_on_repair_enabled are false + // requiresWritePath should return false for a table with CDC enabled. + DatabaseDescriptor.setCDCOnRepairEnabled(false); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CDC_TABLE); when(session.streamOperation()).thenReturn(StreamOperation.BULK_LOAD); CassandraStreamReceiver receiver1 = new CassandraStreamReceiver(cfs, session, 1); assertFalse(receiver1.requiresWritePath(cfs)); - System.setProperty("cassandra.streaming.requires_cdc_replay", "true"); + // When flipping cdc_on_repair_enabled to true + // requiresWritePath should return true. + DatabaseDescriptor.setCDCOnRepairEnabled(true); CassandraStreamReceiver receiver2 = new CassandraStreamReceiver(cfs, session, 1); assertTrue(receiver2.requiresWritePath(cfs)); - } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 79b371fca347..2ba4ff9fbf3f 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -35,7 +35,6 @@ import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.Options; -import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -161,24 +160,6 @@ public void testGetRepairThreads() assertEquals(5, result); } - @Test - public void testGetRepairSubRangeNum() - { - config.global_settings.number_of_subranges = 5; - - int result = config.getRepairSubRangeNum(repairType); - - assertEquals(5, result); - } - - @Test - public void testSetRepairSubRangeNum() - { - config.setRepairSubRangeNum(repairType, 5); - - assert config.getOptions(repairType).number_of_subranges == 5; - } - @Test public void testGetRepairMinFrequencyInHours() { @@ -390,16 +371,16 @@ public void testGetDefaultOptionsTokenRangeSplitter() { Options defaultOptions = Options.getDefaultOptions(); - ParameterizedClass expectedDefault = new ParameterizedClass(DefaultAutoRepairTokenSplitter.class.getName(), Collections.emptyMap()); + ParameterizedClass expectedDefault = new ParameterizedClass(RepairTokenRangeSplitter.class.getName(), Collections.emptyMap()); assertEquals(expectedDefault, defaultOptions.token_range_splitter); - assertEquals(DefaultAutoRepairTokenSplitter.class.getName(), FBUtilities.newAutoRepairTokenRangeSplitter(defaultOptions.token_range_splitter).getClass().getName()); + assertEquals(RepairTokenRangeSplitter.class.getName(), AutoRepair.newAutoRepairTokenRangeSplitter(repairType, defaultOptions.token_range_splitter).getClass().getName()); } @Test(expected = ConfigurationException.class) public void testInvalidTokenRangeSplitter() { - FBUtilities.newAutoRepairTokenRangeSplitter(new ParameterizedClass("invalid-class", Collections.emptyMap())); + AutoRepair.newAutoRepairTokenRangeSplitter(repairType, new ParameterizedClass("invalid-class", Collections.emptyMap())); } @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java index eccf162bbb46..daff1f568219 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; import org.junit.BeforeClass; import org.junit.Test; @@ -95,12 +96,10 @@ public void testTokenRangesSplitByTable() } } - AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - config.setRepairSubRangeNum(repairType, numberOfSplits); - List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); - Iterator keyspaceAssignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, plan); + Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Map.of(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSplits))) + .getRepairAssignments(true, plan); // should be only 1 entry for the keyspace. assertTrue(keyspaceAssignments.hasNext()); @@ -148,12 +147,10 @@ public void testTokenRangesSplitByKeyspace() expectedToken.addAll(AutoRepairUtils.split(range, numberOfSplits)); } - AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - config.setRepairSubRangeNum(repairType, numberOfSplits); - List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); - Iterator keyspaceAssignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, plan); + Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Map.of(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSplits))) + .getRepairAssignments(true, plan); // should be only 1 entry for the keyspace. assertTrue(keyspaceAssignments.hasNext()); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 70bb33859ba9..178add44558d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -21,9 +21,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -33,6 +35,7 @@ import com.google.common.collect.Sets; import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.cql3.statements.schema.TableAttributes; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -251,8 +254,11 @@ private void resetConfig() config.repair_type_overrides = defaultConfig.repair_type_overrides; config.global_settings = defaultConfig.global_settings; config.history_clear_delete_hosts_buffer_interval = defaultConfig.history_clear_delete_hosts_buffer_interval; - config.setRepairSubRangeNum(repairType, 1); config.repair_task_min_duration = new DurationSpec.LongSecondsBound("0s"); + // Use fixed splitter so we get a deterministic amount of repairs. + config.setTokenRangeSplitter(repairType, + new ParameterizedClass(FixedSplitTokenRangeSplitter.class.getName(), + Map.of(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, "1"))); } private void executeCQL() @@ -605,7 +611,7 @@ public void testTokenRangesNoSplit() List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE); - Iterator keyspaceAssignments = new DefaultAutoRepairTokenSplitter().getRepairAssignments(repairType, true, plan); + Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.emptyMap()).getRepairAssignments(true, plan); // should be only 1 entry for the keyspace. assertTrue(keyspaceAssignments.hasNext()); @@ -703,6 +709,7 @@ public void testRepairMaxRetries() AutoRepair.instance.repair(repairType); + // Expect configured retries for each table expected to be repaired assertEquals(config.getRepairMaxRetries()*expectedTablesGoingThroughRepair, sleepCalls.get()); verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java similarity index 78% rename from test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java rename to test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index 9baaecca545e..3b7e45af3291 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -42,21 +42,21 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; -import org.apache.cassandra.repair.autorepair.RepairRangeSplitter.FilteredRepairAssignments; -import org.apache.cassandra.repair.autorepair.RepairRangeSplitter.SizeEstimate; -import org.apache.cassandra.repair.autorepair.RepairRangeSplitter.SizedRepairAssignment; +import org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.FilteredRepairAssignments; +import org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.SizeEstimate; +import org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.SizedRepairAssignment; import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.utils.concurrent.Refs; -import static org.apache.cassandra.repair.autorepair.RepairRangeSplitter.MAX_BYTES_PER_SCHEDULE; -import static org.apache.cassandra.repair.autorepair.RepairRangeSplitter.SUBRANGE_SIZE; -import static org.apache.cassandra.repair.autorepair.RepairRangeSplitter.TABLE_BATCH_LIMIT; +import static org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.MAX_BYTES_PER_SCHEDULE; +import static org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.BYTES_PER_ASSIGNMENT; +import static org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.MAX_TABLES_PER_ASSIGNMENT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class RepairRangeSplitterTest extends CQLTester +public class RepairTokenRangeSplitterTest extends CQLTester { - private RepairRangeSplitter repairRangeSplitter; + private RepairTokenRangeSplitter repairRangeSplitter; private String tableName; private static Range FULL_RANGE; @@ -72,7 +72,7 @@ public static void setUpClass() @Before public void setUp() { - repairRangeSplitter = new RepairRangeSplitter(Collections.emptyMap()); + repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.emptyMap()); tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); } @@ -80,9 +80,9 @@ public void setUp() public void testSizePartitionCount() throws Throwable { insertAndFlushTable(tableName, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10); - Refs sstables = RepairRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE); + Refs sstables = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE); assertEquals(10, sstables.iterator().next().getEstimatedPartitionSize().count()); - SizeEstimate sizes = RepairRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE, sstables); + SizeEstimate sizes = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE, sstables); assertEquals(10, sizes.partitions); } @@ -98,10 +98,10 @@ public void testSizePartitionCountSplit() throws Throwable Range tokenRange2 = range.next(); Assert.assertFalse(range.hasNext()); - Refs sstables1 = RepairRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange1); - Refs sstables2 = RepairRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange2); - SizeEstimate sizes1 = RepairRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange1, sstables1); - SizeEstimate sizes2 = RepairRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange2, sstables2); + Refs sstables1 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange1); + Refs sstables2 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange2); + SizeEstimate sizes1 = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange1, sstables1); + SizeEstimate sizes2 = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange2, sstables2); // +-5% because HLL merge and the applying of range size approx ratio causes estimation errors assertTrue(Math.abs(10000 - (sizes1.partitions + sizes2.partitions)) <= 60); } @@ -110,7 +110,7 @@ public void testSizePartitionCountSplit() throws Throwable public void testGetRepairAssignmentsForTable_NoSSTables() { Collection> ranges = Collections.singleton(new Range<>(Murmur3Partitioner.instance.getMinimumToken(), Murmur3Partitioner.instance.getMaximumToken())); - List assignments = repairRangeSplitter.getRepairAssignmentsForTable(RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(CQLTester.KEYSPACE, tableName, ranges); assertEquals(0, assignments.size()); } @@ -118,15 +118,15 @@ public void testGetRepairAssignmentsForTable_NoSSTables() public void testGetRepairAssignmentsForTable_Single() throws Throwable { Collection> ranges = Collections.singleton(new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken())); - insertAndFlushSingleTable(tableName); - List assignments = repairRangeSplitter.getRepairAssignmentsForTable(RepairType.FULL, CQLTester.KEYSPACE, tableName, ranges); + insertAndFlushSingleTable(); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(CQLTester.KEYSPACE, tableName, ranges); assertEquals(1, assignments.size()); } @Test public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable { - repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "2")); + repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "2")); Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); @@ -141,7 +141,7 @@ public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable @Test public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable { - repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "2")); + repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "2")); Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(2); @@ -155,7 +155,7 @@ public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable @Test public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable { - repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "1")); + repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "1")); Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); @@ -167,7 +167,7 @@ public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable @Test public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable { - repairRangeSplitter = new RepairRangeSplitter(Collections.singletonMap(TABLE_BATCH_LIMIT, "100")); + repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "100")); Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(5); @@ -181,7 +181,7 @@ public void testMergeEmptyAssignments() { // Test when the list of assignments is empty List emptyAssignments = Collections.emptyList(); - RepairRangeSplitter.merge(emptyAssignments); + RepairTokenRangeSplitter.merge(emptyAssignments); } @Test @@ -194,7 +194,7 @@ public void testMergeSingleAssignment() SizedRepairAssignment assignment = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames); List assignments = Collections.singletonList(assignment); - SizedRepairAssignment result = RepairRangeSplitter.merge(assignments); + SizedRepairAssignment result = RepairTokenRangeSplitter.merge(assignments); assertEquals(FULL_RANGE, result.getTokenRange()); assertEquals(keyspaceName, result.getKeyspaceName()); @@ -213,7 +213,7 @@ public void testMergeMultipleAssignmentsWithSameTokenRangeAndKeyspace() SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames2); List assignments = Arrays.asList(assignment1, assignment2); - SizedRepairAssignment result = RepairRangeSplitter.merge(assignments); + SizedRepairAssignment result = RepairTokenRangeSplitter.merge(assignments); assertEquals(FULL_RANGE, result.getTokenRange()); assertEquals(keyspaceName, result.getKeyspaceName()); @@ -236,7 +236,7 @@ public void testMergeDifferentTokenRange() SizedRepairAssignment assignment2 = new SizedRepairAssignment(tokenRange2, keyspaceName, tableNames); List assignments = Arrays.asList(assignment1, assignment2); - RepairRangeSplitter.merge(assignments); // Should throw IllegalStateException + RepairTokenRangeSplitter.merge(assignments); // Should throw IllegalStateException } @Test(expected = IllegalStateException.class) @@ -249,7 +249,7 @@ public void testMergeDifferentKeyspaceName() SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, "keyspace2", tableNames); List assignments = Arrays.asList(assignment1, assignment2); - RepairRangeSplitter.merge(assignments); // Should throw IllegalStateException + RepairTokenRangeSplitter.merge(assignments); // Should throw IllegalStateException } @Test @@ -264,7 +264,7 @@ public void testMergeWithDuplicateTables() SizedRepairAssignment assignment2 = new SizedRepairAssignment(FULL_RANGE, keyspaceName, tableNames2); List assignments = Arrays.asList(assignment1, assignment2); - RepairAssignment result = RepairRangeSplitter.merge(assignments); + RepairAssignment result = RepairTokenRangeSplitter.merge(assignments); // The merged result should contain all unique table names assertEquals(new HashSet<>(Arrays.asList("table1", "table2", "table3")), new HashSet<>(result.getTableNames())); @@ -275,12 +275,12 @@ public void testGetRepairAssignmentsSplitsBySubrangeSizeAndFilterLimitsByMaxByte { // Ensures that getRepairAssignments splits by SUBRANGE_SIZE and filterRepairAssignments limits by MAX_BYTES_PER_SCHEDULE. Map parameters = new HashMap<>(); - parameters.put(SUBRANGE_SIZE, "50GiB"); + parameters.put(BYTES_PER_ASSIGNMENT, "50GiB"); parameters.put(MAX_BYTES_PER_SCHEDULE, "100GiB"); - repairRangeSplitter = new RepairRangeSplitter(parameters); + repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.INCREMENTAL, parameters); // Given a size estimate of 1024GiB, we should expect 21 splits (50GiB*21 = 1050GiB < 1024GiB) - SizeEstimate sizeEstimate = sizeEstimateByBytes(RepairType.INCREMENTAL, new LongMebibytesBound("1024GiB")); + SizeEstimate sizeEstimate = sizeEstimateByBytes(new LongMebibytesBound("1024GiB")); List assignments = repairRangeSplitter.getRepairAssignments(Collections.singletonList(sizeEstimate)); @@ -294,24 +294,24 @@ public void testGetRepairAssignmentsSplitsBySubrangeSizeAndFilterLimitsByMaxByte } // When filtering we should only get 2 assignments back (48.76 * 2 < 100GiB) - FilteredRepairAssignments filteredRepairAssignments = repairRangeSplitter.filterRepairAssignments(RepairType.INCREMENTAL, 0, KEYSPACE, assignments, 0); + FilteredRepairAssignments filteredRepairAssignments = repairRangeSplitter.filterRepairAssignments(0, KEYSPACE, assignments, 0); List finalRepairAssignments = filteredRepairAssignments.repairAssignments; assertEquals(2, finalRepairAssignments.size()); assertEquals(expectedBytes*2, filteredRepairAssignments.newBytesSoFar); } - private SizeEstimate sizeEstimateByBytes(RepairType repairType, LongMebibytesBound totalSize) + private SizeEstimate sizeEstimateByBytes(LongMebibytesBound totalSize) { - return sizeEstimateByBytes(repairType, totalSize, totalSize); + return sizeEstimateByBytes(totalSize, totalSize); } - private SizeEstimate sizeEstimateByBytes(RepairType repairType, LongMebibytesBound sizeInRange, LongMebibytesBound totalSize) + private SizeEstimate sizeEstimateByBytes(LongMebibytesBound sizeInRange, LongMebibytesBound totalSize) { - return new SizeEstimate(repairType, KEYSPACE, "table1", FULL_RANGE, 1, sizeInRange.toBytes(), totalSize.toBytes()); + return new SizeEstimate(RepairType.INCREMENTAL, KEYSPACE, "table1", FULL_RANGE, 1, sizeInRange.toBytes(), totalSize.toBytes()); } - private void insertAndFlushSingleTable(String tableName) throws Throwable + private void insertAndFlushSingleTable() throws Throwable { execute("INSERT INTO %s (k, v) values (?, ?)", 1, 1); flush(); diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index 068700ac774d..432c09556d88 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -231,7 +231,6 @@ public static Collection testCases() return Stream.of( forEachRepairType(true, AutoRepairService.instance::setAutoRepairEnabled, config::isAutoRepairEnabled), forEachRepairType(100, AutoRepairService.instance::setRepairThreads, config::getRepairThreads), - forEachRepairType(200, AutoRepairService.instance::setRepairSubRangeNum, config::getRepairSubRangeNum), forEachRepairType(400, AutoRepairService.instance::setRepairSSTableCountHigherThreshold, config::getRepairSSTableCountHigherThreshold), forEachRepairType(ImmutableSet.of("dc1", "dc2"), AutoRepairService.instance::setIgnoreDCs, config::getIgnoreDCs), forEachRepairType(true, AutoRepairService.instance::setPrimaryTokenRangeOnly, config::getRepairPrimaryTokenRangeOnly), diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index 7433b6b1e8ce..6a068dc55f15 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -175,7 +175,7 @@ public void setUp() throws Exception @Test(expected = IllegalArgumentException.class) public void testNoArgs() { - cmd.repairType = repairType; + cmd.repairTypeStr = repairType.name(); cmd.execute(probe); } @@ -183,7 +183,7 @@ public void testNoArgs() public void testRepairSchedulingDisabled() { when(probe.getAutoRepairConfig()).thenReturn(new AutoRepairConfig(false)); - cmd.repairType = repairType; + cmd.repairTypeStr = repairType.name(); cmd.args = ImmutableList.of("threads", "1"); cmd.execute(probe); @@ -196,7 +196,7 @@ public void testRepairSchedulingDisabled() public void testRepairTypeDisabled() { config.setAutoRepairEnabled(repairType, false); - cmd.repairType = repairType; + cmd.repairTypeStr = repairType.name(); cmd.args = ImmutableList.of("number_of_repair_threads", "1"); cmd.execute(probe); @@ -208,7 +208,7 @@ public void testRepairTypeDisabled() @Test public void testV2FlagMissing() { - cmd.repairType = repairType; + cmd.repairTypeStr = repairType.name(); cmd.args = ImmutableList.of("threads", "1"); try @@ -228,17 +228,16 @@ public void testV2FlagMissing() @Test(expected = IllegalArgumentException.class) public void testInvalidParamType() { - cmd.repairType = repairType; + cmd.repairTypeStr = repairType.name(); cmd.args = ImmutableList.of("unknown_type", "1"); cmd.execute(probe); } - @Test public void testPriorityHosts() { - cmd.repairType = repairType; + cmd.repairTypeStr = repairType.name(); cmd.args = ImmutableList.of("priority_hosts", String.join(",", localEndpoint.toString().substring(1), otherEndpoint.toString().substring(1))); cmd.execute(probe); @@ -249,7 +248,7 @@ public void testPriorityHosts() @Test public void testForceRepairHosts() { - cmd.repairType = repairType; + cmd.repairTypeStr = repairType.name(); cmd.args = ImmutableList.of("forcerepair_hosts", String.join(",", localEndpoint.toString().substring(1), otherEndpoint.toString().substring(1))); cmd.execute(probe); @@ -279,7 +278,6 @@ public static Collection testCases() return Stream.of( forEachRepairType("enabled", "true", (type) -> verify(probe, times(1)).setAutoRepairEnabled(type, true)), forEachRepairType("number_of_repair_threads", "1", (type) -> verify(probe, times(1)).setRepairThreads(type, 1)), - forEachRepairType("number_of_subranges", "2", (type) -> verify(probe, times(1)).setRepairSubRangeNum(type, 2)), forEachRepairType("min_repair_interval", "3h", (type) -> verify(probe, times(1)).setRepairMinInterval(type, "3h")), forEachRepairType("sstable_upper_threshold", "4", (type) -> verify(probe, times(1)).setRepairSSTableCountHigherThreshold(type, 4)), forEachRepairType("table_max_repair_time", "5s", (type) -> verify(probe, times(1)).setAutoRepairTableMaxRepairTime(type, "5s")), @@ -287,7 +285,8 @@ public static Collection testCases() forEachRepairType("parallel_repair_count", "6", (type) -> verify(probe, times(1)).setParallelRepairCount(type, 6)), forEachRepairType("parallel_repair_percentage", "7", (type) -> verify(probe, times(1)).setParallelRepairPercentage(type, 7)), forEachRepairType("mv_repair_enabled", "true", (type) -> verify(probe, times(1)).setMVRepairEnabled(type, true)), - forEachRepairType("ignore_dcs", "dc1,dc2", (type) -> verify(probe, times(1)).setAutoRepairIgnoreDCs(type, ImmutableSet.of("dc1", "dc2"))) + forEachRepairType("ignore_dcs", "dc1,dc2", (type) -> verify(probe, times(1)).setAutoRepairIgnoreDCs(type, ImmutableSet.of("dc1", "dc2"))), + forEachRepairType("token_range_splitter.max_bytes_per_schedule", "500GiB", (type) -> verify(probe, times(1)).setAutoRepairTokenRangeSplitterParameter(type, "max_bytes_per_schedule", "500GiB")) ).flatMap(Function.identity()).collect(Collectors.toList()); } @@ -318,7 +317,7 @@ public void setUp() @Test public void test() { - cmd.repairType = repairType; + cmd.repairTypeStr = repairType.name(); cmd.args = ImmutableList.of(paramType, paramVal); cmd.execute(probe); From ebb36cb914f4232c373004c932b2248e27a998c2 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:36:57 -0600 Subject: [PATCH 108/199] JDK 8 support and test rename --- .../autorepair/FixedSplitTokenRangeSplitter.java | 6 +++++- .../repair/autorepair/RepairTokenRangeSplitter.java | 6 +++--- .../autorepair/AutoRepairParameterizedTest.java | 3 +-- ...st.java => FixedSplitTokenRangeSplitterTest.java} | 12 +++++++++--- 4 files changed, 18 insertions(+), 9 deletions(-) rename test/unit/org/apache/cassandra/repair/autorepair/{AutoRepairDefaultTokenSplitterParameterizedTest.java => FixedSplitTokenRangeSplitterTest.java} (94%) diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index d95fdc9959b5..5505a9293cbc 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -133,7 +134,10 @@ public void setParameter(String key, String value) @Override public Map getParameters() { - return Map.of(NUMBER_OF_SUBRANGES, Integer.toString(getNumberOfSubranges())); + return Collections.unmodifiableMap(new LinkedHashMap() + {{ + put(NUMBER_OF_SUBRANGES, Integer.toString(getNumberOfSubranges())); + }}); } public void setNumberOfSubranges(int numberOfSubranges) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index 19f8a07af371..d83cc0b0f1b6 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -171,7 +171,7 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter static final String MAX_TABLES_PER_ASSIGNMENT = "max_tables_per_assignment"; static final String MAX_BYTES_PER_SCHEDULE = "max_bytes_per_schedule"; - static final List PARAMETERS = List.of(BYTES_PER_ASSIGNMENT, PARTITIONS_PER_ASSIGNMENT, MAX_TABLES_PER_ASSIGNMENT, MAX_BYTES_PER_SCHEDULE); + static final String[] PARAMETERS = { BYTES_PER_ASSIGNMENT, PARTITIONS_PER_ASSIGNMENT, MAX_TABLES_PER_ASSIGNMENT, MAX_BYTES_PER_SCHEDULE }; private final AutoRepairConfig.RepairType repairType; @@ -199,7 +199,7 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter * move the entire repaired set from unrepaired to repaired at steady state, assuming not more the 500GiB of * data is written to a node per min_repair_interval. */ - private static final Map DEFAULTS_BY_REPAIR_TYPE = new EnumMap<>(AutoRepairConfig.RepairType.class) {{ + private static final Map DEFAULTS_BY_REPAIR_TYPE = new EnumMap(AutoRepairConfig.RepairType.class) {{ put(AutoRepairConfig.RepairType.FULL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.FULL) .build()); // Restrict incremental repair to 50GB bytes per assignment to confine the amount of possible autocompaction. @@ -794,7 +794,7 @@ public Map getParameters() parameters.put(parameter, ""); } } - return parameters; + return Collections.unmodifiableMap(parameters); } public DataStorageSpec.LongBytesBound getBytesPerAssignment() diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 178add44558d..1d0a35513703 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -25,7 +25,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -258,7 +257,7 @@ private void resetConfig() // Use fixed splitter so we get a deterministic amount of repairs. config.setTokenRangeSplitter(repairType, new ParameterizedClass(FixedSplitTokenRangeSplitter.class.getName(), - Map.of(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, "1"))); + ImmutableMap.of(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, "1"))); } private void executeCQL() diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java similarity index 94% rename from test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java rename to test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java index daff1f568219..87303cee375f 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairDefaultTokenSplitterParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -49,13 +50,18 @@ import static org.junit.Assert.assertTrue; @RunWith(Parameterized.class) -public class AutoRepairDefaultTokenSplitterParameterizedTest extends CQLTester +public class FixedSplitTokenRangeSplitterTest extends CQLTester { private static final String KEYSPACE = "ks"; private static final String TABLE1 = "tbl1"; private static final String TABLE2 = "tbl2"; private static final String TABLE3 = "tbl3"; + private final Map splitterParams = new LinkedHashMap() + {{ + put(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(4)); + }}; + @Parameterized.Parameter() public AutoRepairConfig.RepairType repairType; @@ -98,7 +104,7 @@ public void testTokenRangesSplitByTable() List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); - Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Map.of(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSplits))) + Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, splitterParams) .getRepairAssignments(true, plan); // should be only 1 entry for the keyspace. @@ -149,7 +155,7 @@ public void testTokenRangesSplitByKeyspace() List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); - Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Map.of(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSplits))) + Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, splitterParams) .getRepairAssignments(true, plan); // should be only 1 entry for the keyspace. From 7ed3822c41a5a14e0ae5b3baa0f5b7d7b8c770f6 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:41:18 -0600 Subject: [PATCH 109/199] Fix newAutoRepairTokenRangeSplitter duplication --- .../repair/autorepair/AutoRepair.java | 39 ------------------- .../IAutoRepairTokenRangeSplitter.java | 2 +- .../autorepair/AutoRepairConfigTest.java | 4 +- 3 files changed, 3 insertions(+), 42 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 5be067b3a4e0..4a101059b596 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -19,7 +19,6 @@ package org.apache.cassandra.repair.autorepair; import java.util.ArrayList; -import java.util.Collections; import java.util.EnumMap; import java.util.HashSet; import java.util.Iterator; @@ -38,7 +37,6 @@ import com.google.common.util.concurrent.Uninterruptibles; import org.apache.cassandra.config.DurationSpec; -import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.utils.Clock; @@ -110,43 +108,6 @@ protected AutoRepair() } } - @VisibleForTesting - static IAutoRepairTokenRangeSplitter newAutoRepairTokenRangeSplitter(AutoRepairConfig.RepairType repairType, ParameterizedClass parameterizedClass) throws ConfigurationException - { - try - { - Class tokenRangeSplitterClass; - final String className; - if (parameterizedClass.class_name != null && !parameterizedClass.class_name.isEmpty()) - { - className = parameterizedClass.class_name.contains(".") ? - parameterizedClass.class_name : - "org.apache.cassandra.repair.autorepair." + parameterizedClass.class_name; - tokenRangeSplitterClass = FBUtilities.classForName(className, "token_range_splitter"); - } - else - { - // If token_range_splitter.class_name is not defined, just use default, this is for convenience. - tokenRangeSplitterClass = AutoRepairConfig.DEFAULT_SPLITTER; - } - try - { - Map parameters = parameterizedClass.parameters != null ? parameterizedClass.parameters : Collections.emptyMap(); - // first attempt to initialize with RepairType and Map arguments. - return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor(AutoRepairConfig.RepairType.class, Map.class).newInstance(repairType, parameters); - } - catch (NoSuchMethodException nsme) - { - // fall back on no argument constructor. - return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor().newInstance(); - } - } - catch (Exception ex) - { - throw new ConfigurationException("Unable to create instance of IAutoRepairTokenRangeSplitter", ex); - } - } - public void setup() { // Ensure setup is done only once; this is only for unit tests diff --git a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java index 86bac32c96bb..8b82eac296db 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java @@ -32,7 +32,7 @@ * Custom implementations class should require a constructor accepting * ({@link AutoRepairConfig.RepairType}, {@link java.util.Map}) with the {@link java.util.Map} parameter accepting * custom configuration for your splitter. If such a constructor does not exist, - * {@link AutoRepair#newAutoRepairTokenRangeSplitter(AutoRepairConfig.RepairType, ParameterizedClass)} + * {@link AutoRepairConfig#newAutoRepairTokenRangeSplitter(AutoRepairConfig.RepairType, ParameterizedClass)} * will fall back on invoking a default zero argument constructor. */ public interface IAutoRepairTokenRangeSplitter diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 2ba4ff9fbf3f..8f02baa52f57 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -374,13 +374,13 @@ public void testGetDefaultOptionsTokenRangeSplitter() ParameterizedClass expectedDefault = new ParameterizedClass(RepairTokenRangeSplitter.class.getName(), Collections.emptyMap()); assertEquals(expectedDefault, defaultOptions.token_range_splitter); - assertEquals(RepairTokenRangeSplitter.class.getName(), AutoRepair.newAutoRepairTokenRangeSplitter(repairType, defaultOptions.token_range_splitter).getClass().getName()); + assertEquals(RepairTokenRangeSplitter.class.getName(), AutoRepairConfig.newAutoRepairTokenRangeSplitter(repairType, defaultOptions.token_range_splitter).getClass().getName()); } @Test(expected = ConfigurationException.class) public void testInvalidTokenRangeSplitter() { - AutoRepair.newAutoRepairTokenRangeSplitter(repairType, new ParameterizedClass("invalid-class", Collections.emptyMap())); + AutoRepairConfig.newAutoRepairTokenRangeSplitter(repairType, new ParameterizedClass("invalid-class", Collections.emptyMap())); } @Test From 76a3fcbb1d382c880c1d19c3c327361ee340e5db Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Fri, 3 Jan 2025 11:03:56 -0600 Subject: [PATCH 110/199] Nit-fixing in existing code * Move curly brackets to new line * Remove unused config declaration in constructor --- .../cassandra/cql3/UntypedResultSet.java | 3 +- .../repair/autorepair/AutoRepair.java | 1 - .../cassandra/service/AutoRepairService.java | 3 +- .../cassandra/service/StorageService.java | 9 ++- .../org/apache/cassandra/tools/NodeProbe.java | 12 ++- .../AutoRepairStateFactoryTest.java | 15 ++-- .../autorepair/AutoRepairStateTest.java | 79 +++++++++++++------ 7 files changed, 81 insertions(+), 41 deletions(-) diff --git a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java index 5862444e37f8..00f10f29e3aa 100644 --- a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java +++ b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java @@ -434,7 +434,8 @@ public long getLong(String column) // this function will return the default value if the row doesn't have that column or the column data is null // This function is used to avoid the nullpointerexception - public long getLong(String column, long ifNull) { + public long getLong(String column, long ifNull) + { ByteBuffer bytes = data.get(column); return bytes == null ? ifNull : LongType.instance.compose(bytes); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 4a101059b596..9ee955d07fed 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -96,7 +96,6 @@ public class AutoRepair @VisibleForTesting protected AutoRepair() { - AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); repairExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class); repairRunnableExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class); repairStates = new EnumMap<>(AutoRepairConfig.RepairType.class); diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 1a9c010928a7..4056e6e6a930 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -98,7 +98,8 @@ public void setRepairPriorityForHosts(RepairType repairType, Set getRepairHostPriority(RepairType repairType) { + public Set getRepairHostPriority(RepairType repairType) + { return AutoRepairUtils.getPriorityHosts(repairType); } diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 90df7854f513..37e7a547e906 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -7072,7 +7072,8 @@ public void setEnforceNativeDeadlineForHints(boolean value) DatabaseDescriptor.setEnforceNativeDeadlineForHints(value); } - public List getTablesForKeyspace(String keyspace) { + public List getTablesForKeyspace(String keyspace) + { return Keyspace.open(keyspace).getColumnFamilyStores().stream().map(cfs -> cfs.name).collect(Collectors.toList()); } @@ -7080,7 +7081,8 @@ public List mutateSSTableRepairedState(boolean repaired, boolean preview { Map tables = Keyspace.open(keyspace).getColumnFamilyStores() .stream().collect(Collectors.toMap(c -> c.name, c -> c)); - for (String tableName : tableNames) { + for (String tableName : tableNames) + { if (!tables.containsKey(tableName)) throw new InvalidRequestException("Table " + tableName + " does not exist in keyspace " + keyspace); } @@ -7091,7 +7093,8 @@ public List mutateSSTableRepairedState(boolean repaired, boolean preview // mutate SSTables long repairedAt = !repaired ? 0 : currentTimeMillis(); List sstablesTouched = new ArrayList<>(); - for (String tableName : tableNames) { + for (String tableName : tableNames) + { ColumnFamilyStore table = tables.get(tableName); Set result = table.runWithCompactionsDisabled(() -> { Set sstables = table.getLiveSSTables().stream().filter(predicate).collect(Collectors.toSet()); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 313ce99944a6..02e77bf00b9b 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2147,7 +2147,8 @@ public int getDefaultKeyspaceReplicationFactor() return ssProxy.getDefaultKeyspaceReplicationFactor(); } - public AutoRepairConfig getAutoRepairConfig() { + public AutoRepairConfig getAutoRepairConfig() + { return autoRepairProxy.getAutoRepairConfig(); } @@ -2230,11 +2231,13 @@ public void setAutoRepairIgnoreDCs(AutoRepairConfig.RepairType repairType, Set mutateSSTableRepairedState(boolean repair, boolean preview, return ssProxy.mutateSSTableRepairedState(repair, preview, keyspace, tables); } - public List getTablesForKeyspace(String keyspace) { + public List getTablesForKeyspace(String keyspace) + { return ssProxy.getTablesForKeyspace(keyspace); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java index a45583b9bf87..15d62c536a06 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java @@ -29,7 +29,8 @@ public class AutoRepairStateFactoryTest { @Test - public void testGetRepairState() { + public void testGetRepairState() + { AutoRepairState state = RepairType.getAutoRepairState(RepairType.FULL); assertTrue(state instanceof FullRepairState); @@ -44,12 +45,16 @@ public void testGetRepairState() { } @Test - public void testGetRepairStateSupportsAllRepairTypes() { - for (RepairType repairType : RepairType.values()) { - try { + public void testGetRepairStateSupportsAllRepairTypes() + { + for (RepairType repairType : RepairType.values()) + { + try + { AutoRepairState state = RepairType.getAutoRepairState(repairType); assertNotNull(state); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException e) + { assertNull(e); } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java index f0974dd83c1a..63e1ee4aaa0d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -67,14 +67,16 @@ public static Collection repairTypes() } @Before - public void setUp() { + public void setUp() + { AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); initMocks(this); createTable(String.format("CREATE TABLE IF NOT EXISTS %s.%s (pk int PRIMARY KEY, v int)", KEYSPACE, testTable)); } @Test - public void testGetRepairRunnable() { + public void testGetRepairRunnable() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); AutoRepairService.setup(); @@ -138,7 +140,8 @@ public void testWaitForRepairToComplete() throws Exception } @Test - public void testGetLastRepairTime() { + public void testGetLastRepairTime() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.lastRepairTimeInMs = 1; @@ -146,7 +149,8 @@ public void testGetLastRepairTime() { } @Test - public void testSetTotalTablesConsideredForRepair() { + public void testSetTotalTablesConsideredForRepair() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.setTotalTablesConsideredForRepair(1); @@ -155,7 +159,8 @@ public void testSetTotalTablesConsideredForRepair() { } @Test - public void testGetTotalTablesConsideredForRepair() { + public void testGetTotalTablesConsideredForRepair() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.totalTablesConsideredForRepair = 1; @@ -163,7 +168,8 @@ public void testGetTotalTablesConsideredForRepair() { } @Test - public void testSetLastRepairTimeInMs() { + public void testSetLastRepairTimeInMs() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.setLastRepairTime(1); @@ -172,7 +178,8 @@ public void testSetLastRepairTimeInMs() { } @Test - public void testGetClusterRepairTimeInSec() { + public void testGetClusterRepairTimeInSec() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.clusterRepairTimeInSec = 1; @@ -180,7 +187,8 @@ public void testGetClusterRepairTimeInSec() { } @Test - public void testGetNodeRepairTimeInSec() { + public void testGetNodeRepairTimeInSec() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.nodeRepairTimeInSec = 1; @@ -188,7 +196,8 @@ public void testGetNodeRepairTimeInSec() { } @Test - public void testSetRepairInProgress() { + public void testSetRepairInProgress() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.setRepairInProgress(true); @@ -197,7 +206,8 @@ public void testSetRepairInProgress() { } @Test - public void testIsRepairInProgress() { + public void testIsRepairInProgress() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.repairInProgress = true; @@ -205,7 +215,8 @@ public void testIsRepairInProgress() { } @Test - public void testSetSkippedTokenRangesCount() { + public void testSetSkippedTokenRangesCount() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.setSkippedTokenRangesCount(1); @@ -214,7 +225,8 @@ public void testSetSkippedTokenRangesCount() { } @Test - public void testGetSkippedTokenRangesCount() { + public void testGetSkippedTokenRangesCount() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.skippedTokenRangesCount = 1; @@ -222,7 +234,8 @@ public void testGetSkippedTokenRangesCount() { } @Test - public void testGetLongestUnrepairedSecNull() { + public void testGetLongestUnrepairedSecNull() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.longestUnrepairedNode = null; @@ -235,7 +248,8 @@ public void testGetLongestUnrepairedSecNull() { } @Test - public void testGetLongestUnrepairedSec() { + public void testGetLongestUnrepairedSec() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.longestUnrepairedNode = new AutoRepairHistory(UUID.randomUUID(), "", 0, 1000, null, 0, false); @@ -250,7 +264,8 @@ public void testGetLongestUnrepairedSec() { } @Test - public void testSetTotalMVTablesConsideredForRepair() { + public void testSetTotalMVTablesConsideredForRepair() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.setTotalMVTablesConsideredForRepair(1); @@ -259,7 +274,8 @@ public void testSetTotalMVTablesConsideredForRepair() { } @Test - public void testGetTotalMVTablesConsideredForRepair() { + public void testGetTotalMVTablesConsideredForRepair() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.totalMVTablesConsideredForRepair = 1; @@ -267,7 +283,8 @@ public void testGetTotalMVTablesConsideredForRepair() { } @Test - public void testSetNodeRepairTimeInSec() { + public void testSetNodeRepairTimeInSec() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.setNodeRepairTimeInSec(1); @@ -276,7 +293,8 @@ public void testSetNodeRepairTimeInSec() { } @Test - public void testSetClusterRepairTimeInSec() { + public void testSetClusterRepairTimeInSec() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.setClusterRepairTimeInSec(1); @@ -285,15 +303,18 @@ public void testSetClusterRepairTimeInSec() { } @Test - public void testSetRepairKeyspaceCount() { + public void testSetRepairKeyspaceCount() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.setRepairKeyspaceCount(1); assertEquals(1, state.repairKeyspaceCount); } + @Test - public void testGetRepairKeyspaceCount() { + public void testGetRepairKeyspaceCount() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.repairKeyspaceCount = 1; @@ -301,7 +322,8 @@ public void testGetRepairKeyspaceCount() { } @Test - public void testSetLongestUnrepairedNode() { + public void testSetLongestUnrepairedNode() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); AutoRepairHistory history = new AutoRepairHistory(UUID.randomUUID(), "", 0, 0, null, 0, false); @@ -311,7 +333,8 @@ public void testSetLongestUnrepairedNode() { } @Test - public void testSetSucceededTokenRangesCount() { + public void testSetSucceededTokenRangesCount() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.setSucceededTokenRangesCount(1); @@ -320,7 +343,8 @@ public void testSetSucceededTokenRangesCount() { } @Test - public void testGetSucceededTokenRangesCount() { + public void testGetSucceededTokenRangesCount() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.succeededTokenRangesCount = 1; @@ -328,7 +352,8 @@ public void testGetSucceededTokenRangesCount() { } @Test - public void testSetFailedTokenRangesCount() { + public void testSetFailedTokenRangesCount() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.setFailedTokenRangesCount(1); @@ -337,7 +362,8 @@ public void testSetFailedTokenRangesCount() { } @Test - public void testGetFailedTokenRangesCount() { + public void testGetFailedTokenRangesCount() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.failedTokenRangesCount = 1; @@ -345,7 +371,8 @@ public void testGetFailedTokenRangesCount() { } @Test - public void isSuccess() { + public void isSuccess() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.success = true; From aa6581b34065cbcc0e0a2af41389348e33fda133 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Fri, 3 Jan 2025 11:04:35 -0600 Subject: [PATCH 111/199] Process feedback Also remove redundant AutoRepairConfig.RepairType for RepairType in AutoRepairConfig. --- .../repair/autorepair/AutoRepairConfig.java | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 2ac27f265f0f..e69f98318499 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -66,7 +66,7 @@ public class AutoRepairConfig implements Serializable public static final Class DEFAULT_SPLITTER = RepairTokenRangeSplitter.class; // make transient so gets consturcted in the implementation. - private final transient Map tokenRangeSplitters = new EnumMap<>(AutoRepairConfig.RepairType.class); + private final transient Map tokenRangeSplitters = new EnumMap<>(RepairType.class); public enum RepairType implements Serializable { @@ -81,11 +81,6 @@ public enum RepairType implements Serializable this.configName = name().toLowerCase(); } - RepairType(String configName) - { - this.configName = configName; - } - /** * @return Format of the repair type as it should be represented in configuration. * Canonically this is the enum name in lowerCase. @@ -310,7 +305,8 @@ public ParameterizedClass getTokenRangeSplitter(RepairType repairType) /** * Set a new token range splitter, this is not meant to be used other than for testing. */ - public void setTokenRangeSplitter(RepairType repairType, ParameterizedClass tokenRangeSplitter) + @VisibleForTesting + void setTokenRangeSplitter(RepairType repairType, ParameterizedClass tokenRangeSplitter) { getOptions(repairType).token_range_splitter = tokenRangeSplitter; tokenRangeSplitters.remove(repairType); @@ -318,13 +314,8 @@ public void setTokenRangeSplitter(RepairType repairType, ParameterizedClass toke public IAutoRepairTokenRangeSplitter getTokenRangeSplitterInstance(RepairType repairType) { - IAutoRepairTokenRangeSplitter splitter = tokenRangeSplitters.get(repairType); - if (splitter == null) - { - splitter = newAutoRepairTokenRangeSplitter(repairType, getTokenRangeSplitter(repairType)); - tokenRangeSplitters.put(repairType, splitter); - } - return splitter; + return tokenRangeSplitters.computeIfAbsent(repairType, + key -> newAutoRepairTokenRangeSplitter(key, getTokenRangeSplitter(key))); } public void setInitialSchedulerDelay(RepairType repairType, String initialSchedulerDelay) @@ -348,11 +339,11 @@ public void setRepairSessionTimeout(RepairType repairType, String repairSessionT } @VisibleForTesting - static IAutoRepairTokenRangeSplitter newAutoRepairTokenRangeSplitter(AutoRepairConfig.RepairType repairType, ParameterizedClass parameterizedClass) throws ConfigurationException + static IAutoRepairTokenRangeSplitter newAutoRepairTokenRangeSplitter(RepairType repairType, ParameterizedClass parameterizedClass) throws ConfigurationException { try { - Class tokenRangeSplitterClass; + Class tokenRangeSplitterClass; final String className; if (parameterizedClass.class_name != null && !parameterizedClass.class_name.isEmpty()) { @@ -370,12 +361,12 @@ static IAutoRepairTokenRangeSplitter newAutoRepairTokenRangeSplitter(AutoRepairC { Map parameters = parameterizedClass.parameters != null ? parameterizedClass.parameters : Collections.emptyMap(); // first attempt to initialize with RepairType and Map arguments. - return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor(AutoRepairConfig.RepairType.class, Map.class).newInstance(repairType, parameters); + return tokenRangeSplitterClass.getConstructor(RepairType.class, Map.class).newInstance(repairType, parameters); } catch (NoSuchMethodException nsme) { // fall back on no argument constructor. - return (IAutoRepairTokenRangeSplitter) tokenRangeSplitterClass.getConstructor().newInstance(); + return tokenRangeSplitterClass.getConstructor().newInstance(); } } catch (Exception ex) From 5eb55b26edd00608152a31fd8d38bbf7d72f7e46 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sat, 4 Jan 2025 16:02:49 -0800 Subject: [PATCH 112/199] Fix the the total expected sub-ranges calculation keeping v-nodes in mind --- .../autorepair/RepairTokenRangeSplitter.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index d83cc0b0f1b6..6a45c7e6b927 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -538,6 +538,18 @@ protected List getRepairAssignments(List si // since its possible for us to hit maxBytesPerSchedule before seeing all ranges, shuffle so there is chance // at least of hitting all the ranges _eventually_ for the worst case scenarios Collections.shuffle(sizeEstimates); + int totalExpectedSubRanges = 0; + for (SizeEstimate estimate : sizeEstimates) + { + if (estimate.sizeForRepair != 0) + { + boolean needsSplitting = estimate.sizeForRepair > bytesPerAssignment.toBytes() || estimate.partitions > partitionsPerAssignment; + if (needsSplitting) + { + totalExpectedSubRanges += calculateNumberOfSplits(estimate); + } + } + } for (SizeEstimate estimate : sizeEstimates) { if (estimate.sizeForRepair == 0) @@ -567,7 +579,7 @@ protected List getRepairAssignments(List si for (Range subrange : subranges) { SizedRepairAssignment assignment = new SizedRepairAssignment(subrange, estimate.keyspace, Collections.singletonList(estimate.table), - String.format("subrange %d of %d", repairAssignments.size()+1, numberOfSplits), + String.format("subrange %d of %d", repairAssignments.size()+1, totalExpectedSubRanges), approximateBytesPerSplit); repairAssignments.add(assignment); } From 736c150dbd9513e4b636ebfaf4f61ae1b4b556fd Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 5 Jan 2025 18:35:24 -0600 Subject: [PATCH 113/199] Feedback and update tests * Promote node not being present in gossip to warn * Clean up NUMBER_OF_SUBRANGES doc * Simplify default map parameter parsing in both splitters * Doc cleanup and make RepairAssignmentIterator fully public * Remove unnecessary getters and setters in Splitters * Handle case where ColumnFamilyStore not retrievable, in this case return no assignments as we can assume deleted. * Always return a RepairAssignment for a table, even if empty, in RepairTokenRangeSplitter as node may have missed writes * Add set|getParameters tests * Update AutoRepairParameterizedTest to use RepairTokenRangeSplitter * Move no-split specific test to FixedSplitTokenRangeSplitterTest --- .../repair/autorepair/AutoRepairUtils.java | 2 +- .../FixedSplitTokenRangeSplitter.java | 62 +++--- .../autorepair/RepairAssignmentIterator.java | 15 +- .../autorepair/RepairTokenRangeSplitter.java | 190 +++++++----------- .../service/AutoRepairServiceMBean.java | 2 +- .../AutoRepairParameterizedTest.java | 58 +----- .../FixedSplitTokenRangeSplitterTest.java | 34 +++- .../RepairTokenRangeSplitterTest.java | 46 ++++- 8 files changed, 187 insertions(+), 222 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index c387d5a7345f..6057e8254587 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -430,7 +430,7 @@ protected static TreeSet getHostIdsInCurrentRing(RepairType repairType, Se } else { - logger.info("Node is not present in Gossip cache node {}, node data center {}", node, nodeDC); + logger.warn("Node is not present in Gossip cache node {}, node data center {}", node, nodeDC); } } return hostIdsInCurrentRing; diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index 5505a9293cbc..b6cdd04498cf 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -22,10 +22,12 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.dht.Range; @@ -36,13 +38,22 @@ public class FixedSplitTokenRangeSplitter implements IAutoRepairTokenRangeSplitter { + private static final Logger logger = LoggerFactory.getLogger(FixedSplitTokenRangeSplitter.class); + /** - * The number of subranges to split each to-be-repaired token range into, - * the higher this number, the smaller the repair sessions will be - * How many subranges to divide one range into? The default is 1. - * If you are using v-node, say 256, then the repair will always go one v-node range at a time, this parameter, additionally, will let us further subdivide a given v-node range into sub-ranges. - * With the value “1” and v-nodes of 256, a given table on a node will undergo the repair 256 times. But with a value “2,” the same table on a node will undergo a repair 512 times because every v-node range will be further divided by two. - * If you do not use v-nodes or the number of v-nodes is pretty small, say 8, setting this value to a higher number, say 16, will be useful to repair on a smaller range, and the chance of succeeding is higher. + * The number of subranges to split each to-be-repaired token range into. Defaults to 1. + *

    + * The higher this number, the smaller the repair sessions will be. + *

    + * If you are using vnodes, say 256, then the repair will always go one vnode range at a time. This parameter, + * additionally, will let us further subdivide a given vnode range into subranges. + *

    + * With the value "1" and vnodes of 256, a given table on a node will undergo the repair 256 times. But with a + * value "2", the same table on a node will undergo a repair 512 times because every vnode range will be further + * divided by two. + *

    + * If you do not use vnodes or the number of vnodes is pretty small, say 8, setting this value to a higher number, + * such as 16, will be useful to repair on a smaller range, and the chance of succeeding is higher. */ static final String NUMBER_OF_SUBRANGES = "number_of_subranges"; @@ -53,14 +64,7 @@ public FixedSplitTokenRangeSplitter(AutoRepairConfig.RepairType repairType, Map< { this.repairType = repairType; - if (parameters.containsKey(NUMBER_OF_SUBRANGES)) - { - numberOfSubranges = Integer.parseInt(parameters.get(NUMBER_OF_SUBRANGES)); - } - else - { - numberOfSubranges = 1; - } + numberOfSubranges = Integer.parseInt(parameters.getOrDefault(NUMBER_OF_SUBRANGES, "1")); } @Override @@ -69,7 +73,7 @@ public Iterator getRepairAssignments(boolean primaryR return new RepairAssignmentIterator(repairPlans) { @Override - KeyspaceRepairAssignments nextInternal(int priority, KeyspaceRepairPlan repairPlan) + protected KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repairPlan) { return getRepairAssignmentsForKeyspace(primaryRangeOnly, priority, repairPlan); } @@ -86,7 +90,7 @@ private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(boolean primar Collection> tokens = StorageService.instance.getPrimaryRanges(keyspaceName); if (!primaryRangeOnly) { - // if we need to repair non-primary token ranges, then change the tokens accrodingly + // if we need to repair non-primary token ranges, then change the tokens accordingly tokens = StorageService.instance.getLocalReplicas(keyspaceName).ranges(); } @@ -124,30 +128,18 @@ private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(boolean primar @Override public void setParameter(String key, String value) { - if (key.equals(NUMBER_OF_SUBRANGES)) + if (!key.equals(NUMBER_OF_SUBRANGES)) { - setNumberOfSubranges(Integer.parseInt(value)); + throw new IllegalArgumentException("Unexpected parameter '" + key + "', must be " + NUMBER_OF_SUBRANGES); } - throw new IllegalArgumentException("Unexpected parameter '" + key + "', must be " + NUMBER_OF_SUBRANGES); + + logger.info("Setting {} to {} for repair type {}", key, value, repairType); + this.numberOfSubranges = Integer.parseInt(value); } @Override public Map getParameters() { - return Collections.unmodifiableMap(new LinkedHashMap() - {{ - put(NUMBER_OF_SUBRANGES, Integer.toString(getNumberOfSubranges())); - }}); + return Collections.singletonMap(NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubranges)); } - - public void setNumberOfSubranges(int numberOfSubranges) - { - this.numberOfSubranges = numberOfSubranges; - } - - public int getNumberOfSubranges() - { - return this.numberOfSubranges; - } - } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java index 5e6f8b61c3b8..fc5573a48944 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java @@ -23,9 +23,9 @@ import java.util.NoSuchElementException; /** - * Convenience {@link Iterator} implementation for to assist implementations of + * Convenience {@link Iterator} implementation to assist implementations of * {@link IAutoRepairTokenRangeSplitter#getRepairAssignments(boolean, List)} by passing {@link KeyspaceRepairPlan} - * to a custom {@link #nextInternal(int, KeyspaceRepairPlan)} in priority order. + * to a custom {@link #next(int, KeyspaceRepairPlan)} method in priority order. */ public abstract class RepairAssignmentIterator implements Iterator { @@ -34,7 +34,7 @@ public abstract class RepairAssignmentIterator implements Iterator currentIterator = null; private PrioritizedRepairPlan currentPlan = null; - RepairAssignmentIterator(List repairPlans) + public RepairAssignmentIterator(List repairPlans) { this.repairPlanIterator = repairPlans.iterator(); } @@ -58,19 +58,20 @@ private synchronized Iterator currentIterator() @Override public boolean hasNext() { - return currentIterator().hasNext(); + Iterator iterator = currentIterator(); + return (iterator != null && iterator.hasNext()); } @Override public KeyspaceRepairAssignments next() { - if (!currentIterator.hasNext()) + if (!hasNext()) { throw new NoSuchElementException("No remaining repair plans"); } final KeyspaceRepairPlan repairPlan = currentIterator().next(); - return nextInternal(currentPlan.getPriority(), repairPlan); + return next(currentPlan.getPriority(), repairPlan); } /** @@ -80,5 +81,5 @@ public KeyspaceRepairAssignments next() * @return assignments for the given keyspace at this priority. Should never return null, if one desires to * short-circuit the iterator, override {@link #hasNext()}. */ - abstract KeyspaceRepairAssignments nextInternal(int priority, KeyspaceRepairPlan repairPlan); + protected abstract KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repairPlan); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index 6a45c7e6b927..7a1223626a60 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumMap; @@ -30,7 +31,9 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; @@ -136,7 +139,7 @@ * * *

  • - * incremental: Configured in a way that attempts to repair 50GiB of data per repair, and 500GiB per + * incremental: Configured in a way that attempts to repair 50GiB of data per repair, and 100GiB per * schedule. This attempts to throttle the amount of IR and anticompaction done per schedule after turning * incremental on for the first time. You may want to consider increasing max_bytes_per_schedule * more than this much data is written per min_repair_interval. @@ -144,7 +147,7 @@ *
  • bytes_per_assignment: 50GiB
    • *
  • partitions_per_assignment: 2^repair_session_max_tree_depth (2^20 == 1048576 by default)
    • *
  • max_tables_per_assignment: 64
    • - *
  • max_bytes_per_schedule: 500GiB
    • + *
  • max_bytes_per_schedule: 100GiB
    • * * *
  • @@ -171,7 +174,7 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter static final String MAX_TABLES_PER_ASSIGNMENT = "max_tables_per_assignment"; static final String MAX_BYTES_PER_SCHEDULE = "max_bytes_per_schedule"; - static final String[] PARAMETERS = { BYTES_PER_ASSIGNMENT, PARTITIONS_PER_ASSIGNMENT, MAX_TABLES_PER_ASSIGNMENT, MAX_BYTES_PER_SCHEDULE }; + static final List PARAMETERS = Arrays.asList(BYTES_PER_ASSIGNMENT, PARTITIONS_PER_ASSIGNMENT, MAX_TABLES_PER_ASSIGNMENT, MAX_BYTES_PER_SCHEDULE); private final AutoRepairConfig.RepairType repairType; @@ -194,9 +197,9 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter *
      max_bytes_per_schedule: 1000GiB
    *
    • * It's expected that these defaults should work well for everything except incremental, so we confine - * bytes_per_assignment to 50GiB and max_bytes_per_schedule to 500GiB. This should strike a good balance + * bytes_per_assignment to 50GiB and max_bytes_per_schedule to 100GiB. This should strike a good balance * between the amount of data that will be repaired during an initial migration to incremental repair and should - * move the entire repaired set from unrepaired to repaired at steady state, assuming not more the 500GiB of + * move the entire repaired set from unrepaired to repaired at steady state, assuming not more the 100GiB of * data is written to a node per min_repair_interval. */ private static final Map DEFAULTS_BY_REPAIR_TYPE = new EnumMap(AutoRepairConfig.RepairType.class) {{ @@ -205,65 +208,44 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter // Restrict incremental repair to 50GB bytes per assignment to confine the amount of possible autocompaction. put(AutoRepairConfig.RepairType.INCREMENTAL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.INCREMENTAL) .withBytesPerAssignment(new DataStorageSpec.LongBytesBound("50GiB")) - .withMaxBytesPerSchedule(new DataStorageSpec.LongBytesBound("500GiB")) + .withMaxBytesPerSchedule(new DataStorageSpec.LongBytesBound("100GiB")) .build()); put(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.PREVIEW_REPAIRED) .build()); }}; - public RepairTokenRangeSplitter(AutoRepairConfig.RepairType repairType, Map parameters) { this.repairType = repairType; this.givenParameters.putAll(parameters); - RepairTypeDefaults defaults = DEFAULTS_BY_REPAIR_TYPE.get(repairType); - if (parameters.containsKey(BYTES_PER_ASSIGNMENT)) - { - bytesPerAssignment = new DataStorageSpec.LongBytesBound(parameters.get(BYTES_PER_ASSIGNMENT)); - } - else - { - bytesPerAssignment = defaults.bytesPerAssignment; - } - - if (parameters.containsKey(PARTITIONS_PER_ASSIGNMENT)) - { - partitionsPerAssignment = Long.parseLong(parameters.get(PARTITIONS_PER_ASSIGNMENT)); - } - else - { - partitionsPerAssignment = defaults.partitionsPerAssignment; - } - - if (parameters.containsKey(MAX_TABLES_PER_ASSIGNMENT)) - { - maxTablesPerAssignment = Integer.parseInt(parameters.get(MAX_TABLES_PER_ASSIGNMENT)); - } - else{ - maxTablesPerAssignment = defaults.maxTablesPerAssignment; - } + reinitParameters(); + } - if (parameters.containsKey(MAX_BYTES_PER_SCHEDULE)) - { - maxBytesPerSchedule = new DataStorageSpec.LongBytesBound(parameters.get(MAX_BYTES_PER_SCHEDULE)); - } - else - { - maxBytesPerSchedule = defaults.maxBytesPerSchedule; - } + private void reinitParameters() + { + RepairTypeDefaults defaults = DEFAULTS_BY_REPAIR_TYPE.get(repairType); + DataStorageSpec.LongBytesBound bytesPerAssignmentTmp = getPropertyOrDefault(BYTES_PER_ASSIGNMENT, DataStorageSpec.LongBytesBound::new, defaults.bytesPerAssignment); + DataStorageSpec.LongBytesBound maxBytesPerScheduleTmp = getPropertyOrDefault(MAX_BYTES_PER_SCHEDULE, DataStorageSpec.LongBytesBound::new, defaults.maxBytesPerSchedule); - if (bytesPerAssignment.toBytes() > maxBytesPerSchedule.toBytes()) + // Validate that bytesPerAssignment <= maxBytesPerSchedule + if (bytesPerAssignmentTmp.toBytes() > maxBytesPerScheduleTmp.toBytes()) { throw new IllegalArgumentException(String.format("%s='%s' cannot be greater than %s='%s' for %s", BYTES_PER_ASSIGNMENT, - bytesPerAssignment.toBytes(), + bytesPerAssignmentTmp, MAX_BYTES_PER_SCHEDULE, - maxBytesPerSchedule, + maxBytesPerScheduleTmp, repairType.getConfigName())); } + bytesPerAssignment = bytesPerAssignmentTmp; + maxBytesPerSchedule = maxBytesPerScheduleTmp; + + partitionsPerAssignment = getPropertyOrDefault(PARTITIONS_PER_ASSIGNMENT, Long::parseLong, defaults.partitionsPerAssignment); + maxTablesPerAssignment = getPropertyOrDefault(MAX_TABLES_PER_ASSIGNMENT, Integer::parseInt, defaults.maxTablesPerAssignment); + logger.info("Configured {}[{}] with {}={}, {}={}, {}={}, {}={}", RepairTokenRangeSplitter.class.getName(), repairType.getConfigName(), BYTES_PER_ASSIGNMENT, bytesPerAssignment, @@ -272,12 +254,11 @@ public RepairTokenRangeSplitter(AutoRepairConfig.RepairType repairType, Map T getPropertyOrDefault(String propertyName, Function mapper, T defaultValue) { - return repairType; + return Optional.ofNullable(this.givenParameters.get(propertyName)).map(mapper).orElse(defaultValue); } - @Override public Iterator getRepairAssignments(boolean primaryRangeOnly, List repairPlans) { @@ -300,7 +281,7 @@ private class BytesBasedRepairAssignmentIterator extends RepairAssignmentIterato } @Override - KeyspaceRepairAssignments nextInternal(int priority, KeyspaceRepairPlan repairPlan) + protected KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repairPlan) { // short circuit if we've accumulated too many bytes by returning a KeyspaceRepairAssignments with // no assignments. We do this rather than returning false in hasNext() because we want to signal @@ -334,8 +315,18 @@ List getRepairAssignmentsForKeyspace(AutoRepairConfig.Rep tableNames.sort((t1, t2) -> { ColumnFamilyStore cfs1 = ColumnFamilyStore.getIfExists(keyspaceName, t1); ColumnFamilyStore cfs2 = ColumnFamilyStore.getIfExists(keyspaceName, t2); - if (cfs1 == null || cfs2 == null) - throw new IllegalArgumentException(String.format("Could not resolve ColumnFamilyStore from %s.%s", keyspaceName, t1)); + // If for whatever reason the CFS is not retrievable, we can assume its been deleted, so give the + // other cfs precedence. + if (cfs1 == null) + { + // cfs1 is lesser than because its null + return -1; + } + else if (cfs2 == null) + { + // cfs1 is greather than because cfs2 is null + return 1; + } return Long.compare(cfs1.metric.totalDiskSpaceUsed.getCount(), cfs2.metric.totalDiskSpaceUsed.getCount()); }); } @@ -530,6 +521,11 @@ protected List getRepairAssignmentsForTable(String keyspa return getRepairAssignments(sizeEstimates); } + private static void logSkippingTable(String keyspaceName, String tableName) + { + logger.warn("Could not resolve table data for {}.{} assuming it has since been deleted, skipping", keyspaceName, tableName); + } + @VisibleForTesting protected List getRepairAssignments(List sizeEstimates) { @@ -555,16 +551,25 @@ protected List getRepairAssignments(List si if (estimate.sizeForRepair == 0) { ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(estimate.keyspace, estimate.table); + + if (cfs == null) + { + logSkippingTable(estimate.keyspace, estimate.table); + continue; + } + long memtableSize = cfs.getTracker().getView().getCurrentMemtable().getLiveDataSize(); if (memtableSize > 0L) { logger.debug("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", estimate.keyspace, estimate.table, estimate.tokenRange, memtableSize); - SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "memtable only", memtableSize); + SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "full primary rangee for table with memtable only detected", memtableSize); repairAssignments.add(assignment); } else { - logger.debug("Skipping {}.{} for range {} because it had no unrepaired SSTables and no memtable data", estimate.keyspace, estimate.table, estimate.tokenRange); + logger.debug("Included {}.{} range {}, has no SSTables or memtable data, but adding single repair assignment for entire range in case writes were missed", estimate.keyspace, estimate.table, estimate.tokenRange); + SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "full primary range for table with no data detected", 0L); + repairAssignments.add(assignment); } } else @@ -724,14 +729,13 @@ static SizeEstimate getSizesForRangeOfSSTables(AutoRepairConfig.RepairType repai static Refs getSSTableReaderRefs(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName, Range tokenRange) { final ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); - if (cfs == null) { - throw new IllegalArgumentException(String.format("Could not resolve ColumnFamilyStore from %s.%s", keyspaceName, tableName)); + logSkippingTable(keyspaceName, tableName); + return Refs.ref(Collections.emptyList()); } Refs refs = null; - while (refs == null) { Iterable sstables = cfs.getTracker().getView().select(SSTableSet.CANONICAL); @@ -750,28 +754,14 @@ static Refs getSSTableReaderRefs(AutoRepairConfig.RepairType repa @Override public void setParameter(String key, String value) { - switch (key) + if (!PARAMETERS.contains(key)) { - case BYTES_PER_ASSIGNMENT: - setBytesPerAssignment(new DataStorageSpec.LongBytesBound(value)); - givenParameters.put(key, value); - return; - case PARTITIONS_PER_ASSIGNMENT: - setPartitionsPerAssignment(Long.parseLong(value)); - givenParameters.put(key, value); - return; - case MAX_TABLES_PER_ASSIGNMENT: - setMaxTablesPerAssignment(Integer.parseInt(value)); - givenParameters.put(key, value); - return; - case MAX_BYTES_PER_SCHEDULE: - setMaxBytesPerSchedule(new DataStorageSpec.LongBytesBound(value)); - givenParameters.put(key, value); - return; - default: - throw new IllegalArgumentException("Unexpected parameter '" + key + "', must be one of " + - PARAMETERS); + throw new IllegalArgumentException("Unexpected parameter '" + key + "', must be one of " + PARAMETERS); } + + logger.info("Setting {} to {} for repair type {}", key, value, repairType); + givenParameters.put(key, value); + reinitParameters(); } @Override @@ -780,7 +770,7 @@ public Map getParameters() final Map parameters = new LinkedHashMap<>(); for (String parameter : PARAMETERS) { - // Use the parameter as configured if it exists, user would likely to prefer to see something more readable. + // Use the parameter as provided if present. if (givenParameters.containsKey(parameter)) { parameters.put(parameter, givenParameters.get(parameter)); @@ -790,16 +780,16 @@ public Map getParameters() switch (parameter) { case BYTES_PER_ASSIGNMENT: - parameters.put(parameter, getBytesPerAssignment().toString()); + parameters.put(parameter, bytesPerAssignment.toString()); continue; case PARTITIONS_PER_ASSIGNMENT: - parameters.put(parameter, Long.toString(getPartitionsPerAssignment())); + parameters.put(parameter, Long.toString(partitionsPerAssignment)); continue; case MAX_TABLES_PER_ASSIGNMENT: - parameters.put(parameter, Integer.toString(getMaxTablesPerAssignment())); + parameters.put(parameter, Integer.toString(maxTablesPerAssignment)); continue; case MAX_BYTES_PER_SCHEDULE: - parameters.put(parameter, getMaxBytesPerSchedule().toString()); + parameters.put(parameter, maxBytesPerSchedule.toString()); continue; default: // not expected @@ -809,46 +799,6 @@ public Map getParameters() return Collections.unmodifiableMap(parameters); } - public DataStorageSpec.LongBytesBound getBytesPerAssignment() - { - return bytesPerAssignment; - } - - public void setBytesPerAssignment(DataStorageSpec.LongBytesBound bytesPerAssignment) - { - this.bytesPerAssignment = bytesPerAssignment; - } - - public long getPartitionsPerAssignment() - { - return partitionsPerAssignment; - } - - public void setPartitionsPerAssignment(long partitionsPerAssignment) - { - this.partitionsPerAssignment = partitionsPerAssignment; - } - - public int getMaxTablesPerAssignment() - { - return maxTablesPerAssignment; - } - - public void setMaxTablesPerAssignment(int maxTablesPerAssignment) - { - this.maxTablesPerAssignment = maxTablesPerAssignment; - } - - public DataStorageSpec.LongBytesBound getMaxBytesPerSchedule() - { - return maxBytesPerSchedule; - } - - public void setMaxBytesPerSchedule(DataStorageSpec.LongBytesBound maxBytesPerSchedule) - { - this.maxBytesPerSchedule = maxBytesPerSchedule; - } - /** * Represents a size estimate by both bytes and partition count for a given keyspace and table for a token range. */ diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index e69a45482bc0..14fe9feb7958 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -73,5 +73,5 @@ public interface AutoRepairServiceMBean public Map getAutoRepairTokenRangeSplitterParameters(RepairType repairType); - public void setAutoRepairTokenRangeSplitterParameter(AutoRepairConfig.RepairType repairType, String key, String value); + public void setAutoRepairTokenRangeSplitterParameter(RepairType repairType, String key, String value); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 1d0a35513703..7a09f46a5d79 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -18,12 +18,9 @@ package org.apache.cassandra.repair.autorepair; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.UUID; @@ -34,17 +31,13 @@ import com.google.common.collect.Sets; import org.apache.cassandra.config.DurationSpec; -import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.cql3.statements.schema.TableAttributes; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.LocalStrategy; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.schema.AutoRepairParams; import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableParams; -import org.apache.cassandra.service.StorageService; import org.junit.After; import org.junit.Assert; @@ -84,7 +77,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; @@ -110,10 +102,9 @@ public class AutoRepairParameterizedTest extends CQLTester AutoRepairState autoRepairState; @Mock RepairRunnable repairRunnable; - private static AutoRepairConfig defaultConfig; - // Expected number of tables that should be repaired. - private static int expectedTablesGoingThroughRepair; + // Expected number of repairs to be executed. + private static int expectedRepairAssignments; @Parameterized.Parameter() public AutoRepairConfig.RepairType repairType; @@ -175,7 +166,7 @@ public static void setupClass() throws Exception // Calculate the expected number of tables to be repaired, this should be all system keyspaces that are // distributed, plus 1 for the table we created (ks.tbl), excluding the 'mv' materialized view and // 'tbl_disabled_auto_repair' we created. - expectedTablesGoingThroughRepair = 0; + int expectedTablesGoingThroughRepair = 0; for (Keyspace keyspace : Keyspace.all()) { // skip LocalStrategy keyspaces as these aren't repaired. @@ -191,6 +182,8 @@ public static void setupClass() throws Exception int expectedTables = keyspace.getName().equals("ks") ? 1 : keyspace.getColumnFamilyStores().size(); expectedTablesGoingThroughRepair += expectedTables; } + // Since the splitter will unwrap a full token range, we expect twice as many repairs. + expectedRepairAssignments = expectedTablesGoingThroughRepair * 2; } @Before @@ -241,7 +234,7 @@ private void resetCounters() private void resetConfig() { // prepare a fresh default config - defaultConfig = new AutoRepairConfig(true); + AutoRepairConfig defaultConfig = new AutoRepairConfig(true); for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) { defaultConfig.setAutoRepairEnabled(repairType, true); @@ -254,10 +247,6 @@ private void resetConfig() config.global_settings = defaultConfig.global_settings; config.history_clear_delete_hosts_buffer_interval = defaultConfig.history_clear_delete_hosts_buffer_interval; config.repair_task_min_duration = new DurationSpec.LongSecondsBound("0s"); - // Use fixed splitter so we get a deterministic amount of repairs. - config.setTokenRangeSplitter(repairType, - new ParameterizedClass(FixedSplitTokenRangeSplitter.class.getName(), - ImmutableMap.of(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, "1"))); } private void executeCQL() @@ -601,31 +590,6 @@ public void testDisabledAutoRepairForATableThroughTableLevelConfiguration() disabledTablesRepairCountBefore < disabledTablesRepairCountAfter); } - @Test - public void testTokenRangesNoSplit() - { - Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); - assertEquals(1, tokens.size()); - List> expectedToken = new ArrayList<>(tokens); - - List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE); - - Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.emptyMap()).getRepairAssignments(true, plan); - - // should be only 1 entry for the keyspace. - assertTrue(keyspaceAssignments.hasNext()); - KeyspaceRepairAssignments keyspace = keyspaceAssignments.next(); - assertFalse(keyspaceAssignments.hasNext()); - - List assignments = keyspace.getRepairAssignments(); - assertNotNull(assignments); - - // should be 1 entry for the table which covers the full range. - assertEquals(1, assignments.size()); - assertEquals(expectedToken.get(0).left, assignments.get(0).getTokenRange().left); - assertEquals(expectedToken.get(0).right, assignments.get(0).getTokenRange().right); - } - @Test public void testTableAttribute() { @@ -709,10 +673,10 @@ public void testRepairMaxRetries() AutoRepair.instance.repair(repairType); // Expect configured retries for each table expected to be repaired - assertEquals(config.getRepairMaxRetries()*expectedTablesGoingThroughRepair, sleepCalls.get()); + assertEquals(config.getRepairMaxRetries() * expectedRepairAssignments, sleepCalls.get()); verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); - verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(expectedRepairAssignments); } @Test @@ -738,7 +702,7 @@ public void testRepairSuccessAfterRetry() AutoRepair.instance.repair(repairType); assertEquals(1, sleepCalls.get()); - verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedRepairAssignments); verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); } @@ -810,7 +774,7 @@ public void testSoakAfterImmediateRepair() AutoRepair.instance.repair(repairType); assertEquals(1, sleepCalls.get()); - verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedRepairAssignments); verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); } @@ -830,7 +794,7 @@ public void testNoSoakAfterRepair() AutoRepair.instance.repair(repairType); - verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedRepairAssignments); verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java index 87303cee375f..e743948d1f23 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java @@ -23,7 +23,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -57,10 +56,7 @@ public class FixedSplitTokenRangeSplitterTest extends CQLTester private static final String TABLE2 = "tbl2"; private static final String TABLE3 = "tbl3"; - private final Map splitterParams = new LinkedHashMap() - {{ - put(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(4)); - }}; + private static final Map splitterParams = Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(4)); @Parameterized.Parameter() public AutoRepairConfig.RepairType repairType; @@ -176,4 +172,32 @@ public void testTokenRangesSplitByKeyspace() assertEquals(tables, assignments.get(i).getTableNames()); } } + + @Test + public void testTokenRangesNoSplitByDefault() + { + Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); + int totalTokenRanges = 3; + assertEquals(totalTokenRanges, tokens.size()); + List> expectedToken = new ArrayList<>(tokens); + + List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1); + + Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.emptyMap()).getRepairAssignments(true, plan); + + // should be only 1 entry for the keyspace. + assertTrue(keyspaceAssignments.hasNext()); + KeyspaceRepairAssignments keyspace = keyspaceAssignments.next(); + assertFalse(keyspaceAssignments.hasNext()); + + List assignments = keyspace.getRepairAssignments(); + assertNotNull(assignments); + + // should be 3 entries for the table which covers each token range. + assertEquals(totalTokenRanges, assignments.size()); + for (int i = 0; i < totalTokenRanges; i++) + { + assertEquals(expectedToken.get(i), assignments.get(i).getTokenRange()); + } + } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index 3b7e45af3291..1081a0b9d5cd 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -109,9 +109,10 @@ public void testSizePartitionCountSplit() throws Throwable @Test public void testGetRepairAssignmentsForTable_NoSSTables() { + // Should return 1 assignment if there are no SSTables Collection> ranges = Collections.singleton(new Range<>(Murmur3Partitioner.instance.getMinimumToken(), Murmur3Partitioner.instance.getMaximumToken())); List assignments = repairRangeSplitter.getRepairAssignmentsForTable(CQLTester.KEYSPACE, tableName, ranges); - assertEquals(0, assignments.size()); + assertEquals(1, assignments.size()); } @Test @@ -273,11 +274,10 @@ public void testMergeWithDuplicateTables() @Test public void testGetRepairAssignmentsSplitsBySubrangeSizeAndFilterLimitsByMaxBytesPerSchedule() { - // Ensures that getRepairAssignments splits by SUBRANGE_SIZE and filterRepairAssignments limits by MAX_BYTES_PER_SCHEDULE. - Map parameters = new HashMap<>(); - parameters.put(BYTES_PER_ASSIGNMENT, "50GiB"); - parameters.put(MAX_BYTES_PER_SCHEDULE, "100GiB"); - repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.INCREMENTAL, parameters); + // Ensures that getRepairAssignments splits by BYTES_PER_ASSIGNMENT and filterRepairAssignments limits by MAX_BYTES_PER_SCHEDULE. + repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.INCREMENTAL, Collections.emptyMap()); + repairRangeSplitter.setParameter(BYTES_PER_ASSIGNMENT, "50GiB"); + repairRangeSplitter.setParameter(MAX_BYTES_PER_SCHEDULE, "100GiB"); // Given a size estimate of 1024GiB, we should expect 21 splits (50GiB*21 = 1050GiB < 1024GiB) SizeEstimate sizeEstimate = sizeEstimateByBytes(new LongMebibytesBound("1024GiB")); @@ -300,6 +300,40 @@ public void testGetRepairAssignmentsSplitsBySubrangeSizeAndFilterLimitsByMaxByte assertEquals(expectedBytes*2, filteredRepairAssignments.newBytesSoFar); } + @Test(expected=IllegalArgumentException.class) + public void testSetParameterShouldNotAllowUnknownParameter() + { + repairRangeSplitter.setParameter("unknown", "x"); + } + + @Test(expected=IllegalArgumentException.class) + public void testSetParameterShouldNotAllowSettingBytesPerAssignmentGreaterThanMaxBytesPerSchedule() + { + repairRangeSplitter.setParameter(MAX_BYTES_PER_SCHEDULE, "500GiB"); + repairRangeSplitter.setParameter(BYTES_PER_ASSIGNMENT, "600GiB"); + } + + @Test(expected=IllegalArgumentException.class) + public void testSetParameterShouldNotAllowSettingMaxBytesPerScheduleLessThanBytesPerAssignment() + { + repairRangeSplitter.setParameter(BYTES_PER_ASSIGNMENT, "100MiB"); + repairRangeSplitter.setParameter(MAX_BYTES_PER_SCHEDULE, "50MiB"); + } + + @Test + public void testGetParameters() + { + repairRangeSplitter.setParameter(BYTES_PER_ASSIGNMENT, "100MiB"); + repairRangeSplitter.setParameter(MAX_TABLES_PER_ASSIGNMENT, "5"); + + Map parameters = repairRangeSplitter.getParameters(); + // Each parameter should be present. + assertEquals(RepairTokenRangeSplitter.PARAMETERS.size(), parameters.size()); + // The parameters we explicitly set should be set exactly as we set them. + assertEquals("100MiB", parameters.get(BYTES_PER_ASSIGNMENT)); + assertEquals("5", parameters.get(MAX_TABLES_PER_ASSIGNMENT)); + } + private SizeEstimate sizeEstimateByBytes(LongMebibytesBound totalSize) { return sizeEstimateByBytes(totalSize, totalSize); From 1aa37904098c80785d42c90dab8948be5a68d3c8 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 7 Jan 2025 17:14:36 -0800 Subject: [PATCH 114/199] Add configuration to cassandra.yaml and cassandra_later.yaml --- conf/cassandra.yaml | 101 ++++++++++++++++++ .../repair/autorepair/AutoRepairConfig.java | 98 ++++++++--------- 2 files changed, 147 insertions(+), 52 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 1df0bb587587..fb6e1bd2c278 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1865,3 +1865,104 @@ drop_compact_storage_enabled: false # heartbeat_file: /var/lib/cassandra/data/cassandra-heartbeat # excluded_keyspaces: # comma separated list of keyspaces to exclude from the check # excluded_tables: # comma separated list of keyspace.table pairs to exclude from the check + +# Configuration for AutoRepair Scheduler +#auto_repair: +# # Enable/Disable auto repair scheduler. +# # If it is set to false, then the scheduler thread will not be spun up. +# # If it is set to true, then the repair scheduler thread will be created. The thread will +# # look for secondary config available for each repair type (full, incremental, and preview_repaired), +# # and based on that, it will schedule repairs. +# enabled: false +# # Time interval between successive checks for repair scheduler to check if either the ongoing repair is completed or if +# # none is going, then check if it's time to schedule or wait. +# repair_check_interval: 5m +# # Maximum number of retries for a repair session. +# repair_max_retries: 3 +# # Backoff time in seconds for retrying a repair session. +# repair_retry_backoff: 30s +# # Minimum duration for the execution of a single repair task (i.e.: RepairRunnable). +# # This helps prevent the auto-repair scheduler from overwhelming the node by scheduling a large number of +# # repair tasks in a short period of time. +# repair_task_min_duration: 5s +# # When any nodes leave the ring then the repair schedule needs to adjust the order, etc. +# # The AutoRepair scheduler keeps the deleted hosts information in its persisted metadata for the defined interval in this config. +# # This information is useful so the scheduler is absolutely sure that the node is indeed removed from the ring, and then it can adjust the repair schedule accordingly. +# # So, the duration in this config determinses for how long deleted host's information is kept in the scheduler's metadata. +# history_clear_delete_hosts_buffer_interval: 2h +# # Configuration specific to each repair type. Options: full, incremental, and preview_repaired. +# repair_type_overrides: +# full: +# # Enable/Disable full auto repair +# enabled: false +# # Minimum time in hours between repairing the same node again. This is useful for extremely tiny clusters, say 5 nodes, which finishes +# # repair quicly. The default is 24 hours. This means that if the scheduler finishes one round on all the nodes in < 24 hours. On a given node it won’t start a new repair round +# # until the last repair conducted on a given node is < 24 hours. +# min_repair_interval: 24h +# # Repair table by table if this is set to 'false'. If it is 'true', then repair all the tables in a keyspace in one go. +# repair_by_keyspace: false +# # Number of repair threads to run for a given invoked Repair Job. +# # Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. +# # This is similar to -j for repair options for the nodetool repair command. +# number_of_repair_threads: 1 +# # Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. +# # This is to avoid penalizing good tables (neighbors) with an outlier. +# sstable_upper_threshold: 10000 +# # Max time for repairing one table on a given node, if exceeded, skip the table. The default is 6 hours. +# # Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. +# # Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. +# # Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator and would require a lot of back-and-forth manual interventions, etc. +# # So, the idea here is to penalize the outliers instead of good candidates. This can easily be configured with a higher value if we want to disable the functionality. +# table_max_repair_time: 6h +# # This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. +# # If you want to avoid repairing nodes in one or more data centers, then you can specify the data centers in this list. +# ignore_dcs: [] +# # Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false' +# # It is the same as -pr in nodetool repair options. +# repair_primary_token_range_only: true +# # The number of nodes running repair parallelly. If parallel_repair_percentage is set, it will choose the larger value of the two. The default is 3. +# # This configuration controls how many nodes would run repair in parallel. +# # The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. +# # If one or more node(s) finish repair, then the framework automatically picks up the next candidate(s) based on the least repair time. +# # It will ensure the maximum number of nodes running repair do not exceed “3”. +# parallel_repair_count: 3 +# # The percentage of nodes in the cluster that run repair parallelly. If parallel_repair_count is set, it will choose the larger value of the two. +# # The problem with a fixed number of nodes (parallel_repair_count property) is that in a large-scale environment, +# # the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. +# # The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. +# # So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. +# # Extremely fewer manual interventions as it will rarely violate the repair SLA for customers +# parallel_repair_percentage: 3 +# # If the scheduler should repair MV table or not. +# mv_repair_enabled: false +# # After a node restart, wait for this much delay before scheduler starts running repair; this is to avoid starting repair immediately after a node restart. +# initial_scheduler_delay: 5m +# # The major issue with Repair is sometimes repair session hangs; so this timeout is useful to resume such stuck repair sessions. +# repair_session_timeout: 3h +# # Whether to force immediate repair on new nodes. This is useful if you want to repair newly bootstrapped nodes immediately after they join the ring. +# force_repair_new_node: false +# # Splitter implementation to use for generating repair assignments. +# # The default is {@link RepairTokenRangeSplitter}. The class should implement {@link IAutoRepairTokenRangeSplitter} +# # and have a constructor accepting ({@link RepairType}, {@link java.util.Map}) +# token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter +# # The target and maximum amount of bytes that should be included in a repair +# # assignment. This is meant to scope the amount of work involved in a repair. For incremental repair, this +# # involves the total number of bytes in all SSTables containing unrepaired data involving the ranges being +# # repaired, including data that doesn't cover the range. This is to account for the amount of anticompaction +# # that is expected. For all other repair types, this involves the amount of data covering the range being +# # repaired. +# token_range_splitter.bytes_per_assignment: 200GiB +# # The target number of partitions that should be included in a repair +# # assignment. This configuration exists to reduce excessive overstreaming. +# token_range_splitter.partitions_per_assignment: 1048576 +# # The maximum number of tables that can be included in a repair assignment. +# # This aims to reduce the number of repairs, especially in cases where a large amount of tables exists for +# # a keyspace. Note that the splitter will avoid batching tables together if they exceed the other +# # configuration paramters such as bytes_per_assignment and partitions_per_assignment +# token_range_splitter.max_tables_per_assignment: 64 +# # The maximum number of bytes to cover an individiual schedule. This serves +# # as a mechanism for throttling the amount of work that can be done on each repair cycle. One may opt to +# # reduce this value if the impact of repairs is causing too many load on the cluster, or increase it if +# # writes outpace the amount of data being repaired. Alternatively, one may want to choose tuning down or up +# # the min_repair_interval. +# token_range_splitter.max_bytes_per_schedule: 100000GiB diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index e69f98318499..2199a3e10616 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -39,23 +39,24 @@ public class AutoRepairConfig implements Serializable { - // enable/disable auto repair globally, overrides all other settings. Cannot be modified dynamically. - // if it is set to false, then no repair will be scheduled, including full and incremental repairs by this framework. - // if it is set to true, then this repair scheduler will consult another config available for each RepairType, and based on that config, it will schedule repairs. + // Enable/Disable auto repair scheduler. If it is set to false, then the scheduler thread will not be spun up. + // If it is set to true, then the repair scheduler thread will be created. The thread will + // look for secondary config available for each repair type (full, incremental, and preview_repaired), + // and based on that, it will schedule repairs. public volatile Boolean enabled; - // the interval between successive checks for repair scheduler to check if either the ongoing repair is completed or if - // none is going, then check if it's time to schedule or wait + // Time interval between successive checks for repair scheduler to check if either the ongoing repair is completed or if + // none is going, then check if it's time to schedule or wait. public final DurationSpec.IntSecondsBound repair_check_interval = new DurationSpec.IntSecondsBound("5m"); - // when any nodes leave the ring then the repair schedule needs to adjust the order, etc. - // the repair scheduler keeps the deleted hosts information in its persisted metadata for the defined interval in this config. + // When any nodes leave the ring then the repair schedule needs to adjust the order, etc. + // The AutoRepair scheduler keeps the deleted hosts information in its persisted metadata for the defined interval in this config. // This information is useful so the scheduler is absolutely sure that the node is indeed removed from the ring, and then it can adjust the repair schedule accordingly. // So, the duration in this config determinses for how long deleted host's information is kept in the scheduler's metadata. public volatile DurationSpec.IntSecondsBound history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound("2h"); - // the maximum number of retries for a repair session. + // Maximum number of retries for a repair session. public volatile Integer repair_max_retries = 3; - // the backoff time in seconds for retrying a repair session. + // Backoff time in seconds for retrying a repair session. public volatile DurationSpec.LongSecondsBound repair_retry_backoff = new DurationSpec.LongSecondsBound("30s"); - // the minimum duration for the execution of a single repair task (i.e.: RepairRunnable). + // Minimum duration for the execution of a single repair task (i.e.: RepairRunnable). // This helps prevent the auto-repair scheduler from overwhelming the node by scheduling a large number of // repair tasks in a short period of time. public volatile DurationSpec.LongSecondsBound repair_task_min_duration = new DurationSpec.LongSecondsBound("5s"); @@ -99,7 +100,7 @@ public static AutoRepairState getAutoRepairState(RepairType repairType) case INCREMENTAL: return new IncrementalRepairState(); case PREVIEW_REPAIRED: - return new PreviewRepairedState(); + return new PreviewRepairedState(); } throw new IllegalArgumentException("Invalid repair type: " + repairType); @@ -411,55 +412,49 @@ protected static Options getDefaultOptions() return opts; } - // enable/disable auto repair for the given repair type + // Enable/Disable full auto repair public volatile Boolean enabled; - // auto repair is default repair table by table, if this is enabled, the framework will repair all the tables in a keyspace in one go. + // Repair table by table if this is set to 'false'. If it is 'true', then repair all the tables in a keyspace in one go. public volatile Boolean repair_by_keyspace; - // the number of repair threads to run for a given invoked Repair Job. - // Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. - // This is similar to -j for repair options for the nodetool repair command. + // Number of repair threads to run for a given invoked Repair Job. + // Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. + // This is similar to -j for repair options for the nodetool repair command. public volatile Integer number_of_repair_threads; - // The number of nodes running repair parallelly. If parallel_repair_count is set, it will choose the larger value of the two. The default is 3. - // This configuration controls how many nodes would run repair in parallel. - // The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. - // If one or more node(s) finish repair, then the framework automatically picks up the next candidate and ensures the maximum number of nodes running repair do not exceed “3”. + // The number of nodes running repair parallelly. If parallel_repair_percentage is set, it will choose the larger value of the two. The default is 3. + // This configuration controls how many nodes would run repair in parallel. + // The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. + // If one or more node(s) finish repair, then the framework automatically picks up the next candidate(s) based on the least repair time. + // It will ensure the maximum number of nodes running repair do not exceed “3”. public volatile Integer parallel_repair_count; - // the number of repair nodes that can run in parallel - // of the total number of nodes in the group [0,100] - // The percentage of nodes in the cluster that run repair parallelly. If parallelrepaircount is set, it will choose the larger value of the two. - // The problem with a fixed number of nodes (the above property) is that in a large-scale environment, - // the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. - // The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. - // So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. - // Extremely fewer manual interventions as it will rarely violate the repair SLA for customers + // The percentage of nodes in the cluster that run repair parallelly. If parallel_repair_count is set, it will choose the larger value of the two. + // The problem with a fixed number of nodes (parallel_repair_count property) is that in a large-scale environment, + // the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. + // The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. + // So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. + // Extremely fewer manual interventions as it will rarely violate the repair SLA for customers public volatile Integer parallel_repair_percentage; - // the upper threshold of SSTables allowed to participate in a single repair session - // Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. - // This is to avoid penalizing good tables (neighbors) with an outlier. + // Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. + // This is to avoid penalizing good tables (neighbors) with an outlier. public volatile Integer sstable_upper_threshold; - // the minimum time in hours between repairing the same node again. This is useful for extremely tiny clusters, say 5 nodes, which finishes - // repair quicly. - // The default is 24 hours. This means that if the scheduler finishes one round on all the nodes in < 24 hours. On a given node it won’t start a new repair round - // until the last repair conducted on a given node is < 24 hours. + // Minimum time in hours between repairing the same node again. This is useful for extremely tiny clusters, say 5 nodes, which finishes + // repair quicly. The default is 24 hours. This means that if the scheduler finishes one round on all the nodes in < 24 hours. On a given node it won’t start a new repair round + // until the last repair conducted on a given node is < 24 hours. public volatile DurationSpec.IntSecondsBound min_repair_interval; - // specifies a denylist of datacenters to repair - // This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. + // This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. + // If you want to avoid repairing nodes in one or more data centers, then you can specify the data centers in this list. public volatile Set ignore_dcs; - // Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false' - // It is the same as -pr in nodetool repair options. + // Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false' + // It is the same as -pr in nodetool repair options. public volatile Boolean repair_primary_token_range_only; - // configures whether to force immediate repair on new nodes - // default it is set to 'false'; this is useful if you want to repair new nodes immediately after they join the ring. + // Whether to force immediate repair on new nodes. This is useful if you want to repair newly bootstrapped nodes immediately after they join the ring. public volatile Boolean force_repair_new_node; - // the maximum time that a repair session can run for a single table - // Max time for repairing one table on a given node, if exceeded, skip the table. The default is 6 hours. - // Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. - // Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. - // Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator and would require a lot of back-and-forth manual interventions, etc. - // So, the idea here is to penalize the outliers instead of good candidates. This can easily be configured with a higher value if we want to disable the functionality. + // Max time for repairing one table on a given node, if exceeded, skip the table. The default is 6 hours. + // Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. + // Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. + // Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator and would require a lot of back-and-forth manual interventions, etc. + // So, the idea here is to penalize the outliers instead of good candidates. This can easily be configured with a higher value if we want to disable the functionality. public volatile DurationSpec.IntSecondsBound table_max_repair_time; - // the default is 'true'. - // This flag determines whether the auto-repair framework needs to run anti-entropy, a.k.a, repair on the MV table or not. + // If the scheduler should repair MV table or not. public volatile Boolean mv_repair_enabled; /** * Splitter implementation to use for generating repair assignments. @@ -468,10 +463,9 @@ protected static Options getDefaultOptions() * and have a constructor accepting ({@link RepairType}, {@link java.util.Map}) */ public volatile ParameterizedClass token_range_splitter; - // the minimum delay after a node starts before the scheduler starts running repair + // After a node restart, wait for this much delay before scheduler starts running repair; this is to avoid starting repair immediately after a node restart. public volatile DurationSpec.IntSecondsBound initial_scheduler_delay; - // repair session timeout - this is applicable for each repair session - // the major issue with Repair is a session sometimes hangs; so this timeout is useful to unblock such problems + // The major issue with Repair is sometimes repair session hangs; so this timeout is useful to resume such stuck repair sessions. public volatile DurationSpec.IntSecondsBound repair_session_timeout; public String toString() From 833d525d909ca01eb94d908c3c3a0f99e1bbdaa7 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 7 Jan 2025 18:36:12 -0800 Subject: [PATCH 115/199] Add a doc for AutoRepair feature --- conf/cassandra.yaml | 12 +- .../cassandra/pages/auto-repair/overview.adoc | 125 ++++++++++++++++++ .../repair/autorepair/AutoRepairConfig.java | 66 ++++----- .../autorepair/RepairTokenRangeSplitter.java | 4 +- 4 files changed, 166 insertions(+), 41 deletions(-) create mode 100644 doc/modules/cassandra/pages/auto-repair/overview.adoc diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index fb6e1bd2c278..4a5ca8f43cca 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1908,7 +1908,7 @@ drop_compact_storage_enabled: false # # Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. # # This is to avoid penalizing good tables (neighbors) with an outlier. # sstable_upper_threshold: 10000 -# # Max time for repairing one table on a given node, if exceeded, skip the table. The default is 6 hours. +# # Max time for repairing one table on a given node, if exceeded, skip the table. # # Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. # # Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. # # Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator and would require a lot of back-and-forth manual interventions, etc. @@ -1917,10 +1917,10 @@ drop_compact_storage_enabled: false # # This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. # # If you want to avoid repairing nodes in one or more data centers, then you can specify the data centers in this list. # ignore_dcs: [] -# # Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false' +# # Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false'. # # It is the same as -pr in nodetool repair options. # repair_primary_token_range_only: true -# # The number of nodes running repair parallelly. If parallel_repair_percentage is set, it will choose the larger value of the two. The default is 3. +# # The number of nodes running repair parallelly. If parallel_repair_percentage is set, it will choose the larger value of the two. # # This configuration controls how many nodes would run repair in parallel. # # The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. # # If one or more node(s) finish repair, then the framework automatically picks up the next candidate(s) based on the least repair time. @@ -1931,7 +1931,7 @@ drop_compact_storage_enabled: false # # the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. # # The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. # # So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. -# # Extremely fewer manual interventions as it will rarely violate the repair SLA for customers +# # Extremely fewer manual interventions as it will rarely violate the repair SLA for customers. # parallel_repair_percentage: 3 # # If the scheduler should repair MV table or not. # mv_repair_enabled: false @@ -1958,9 +1958,9 @@ drop_compact_storage_enabled: false # # The maximum number of tables that can be included in a repair assignment. # # This aims to reduce the number of repairs, especially in cases where a large amount of tables exists for # # a keyspace. Note that the splitter will avoid batching tables together if they exceed the other -# # configuration paramters such as bytes_per_assignment and partitions_per_assignment +# # configuration parameters such as bytes_per_assignment and partitions_per_assignment # token_range_splitter.max_tables_per_assignment: 64 -# # The maximum number of bytes to cover an individiual schedule. This serves +# # The maximum number of bytes to cover an individual schedule. This serves # # as a mechanism for throttling the amount of work that can be done on each repair cycle. One may opt to # # reduce this value if the impact of repairs is causing too many load on the cluster, or increase it if # # writes outpace the amount of data being repaired. Alternatively, one may want to choose tuning down or up diff --git a/doc/modules/cassandra/pages/auto-repair/overview.adoc b/doc/modules/cassandra/pages/auto-repair/overview.adoc new file mode 100644 index 000000000000..4305e63a9810 --- /dev/null +++ b/doc/modules/cassandra/pages/auto-repair/overview.adoc @@ -0,0 +1,125 @@ += Overview of Auto Repair +:navtitle: Auto Repair overview +:description: Auto Repair concepts - How it works, how to configure it, and more. +:keywords: CEP-37 + +Repair orchestration inside Apache Cassandra. This fully Automated Repair scheduler inside Cassandra does not depend on the control plane, significantly reducing our operational overhead. +At a high level, a dedicated thread pool is assigned to the repair scheduler. The repair scheduler inside Cassandra maintains a new replicated table under a distributed system_distributed keyspace. This table maintains the repair history for all the nodes, such as when it was repaired the last time, etc. The scheduler will pick the node(s) that run the repair first and continue orchestration to ensure Every table and all of their token ranges are repaired. The algorithm can also run repairs simultaneously on multiple nodes and splits the token range into subranges with the necessary retry to handle transient failures. The automatic repair runs as soon as we start a Cassandra cluster, like Compaction, and does not require human-in-the-loop. +The scheduler supports Full, Incremental, and Preview repair types today with the following features. A new repair type, such as Paxos repair and any future repair types, should be extended with minimal dev work! + +=== Features +- Capability to run on multiple nodes simultaneously +- Default implementation and an interface to override the data set being repaired per repair session +- Ability to extend the token split algorithms with the following two implementations readily available: +- Splits token ranges by putting a max cap on the size of data being repaired as part of one repair session as well as a max cap at a table level - this is crucial for Incremental Repair (Default) +- Split tokens evenly based on the number of splits specified +- A new CQL table property to do the following: +- Disable repair type at a table level if one wants the scheduler to skip one or more tables +- Setting repair priority for certain tables to prioritize those tables over others +- Enabling/Disabling scheduler for each repair type dynamically +- Per Data Center per job configuration +- Rich Configuration for each repair type, e.g., full, incremental, or preview_repair +- Enough observability so an operator can configure alarms depending on their need + +=== Yaml Configuration +[cols=",,",options="header",] +|=== +| Name | Default | Description +| enabled | false | Enable/Disable auto repair scheduler +| min_repair_interval | 24h | Minimum time in hours between repairing the same node again. This is useful for extremely tiny clusters, say 5 nodes, which finishes. repair quicly. The default is 24 hours. This means that if the scheduler finishes one round on all the nodes in < 24 hours. On a given node it won’t start a new repair round until the last repair conducted on a given node is < 24 hours. +| repair_by_keyspace | false | Repair table by table if this is set to 'false'. If it is 'true', then repair all the tables in a keyspace in one go. +| number_of_repair_threads | 1 | Number of repair threads to run for a given invoked Repair Job. Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. This is similar to -j for repair options for the nodetool repair command. +| sstable_upper_threshold | 10000 | Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. This is to avoid penalizing good tables (neighbors) with an outlier. +| table_max_repair_time | 6h | Max time for repairing one table on a given node, if exceeded, skip the table. Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator and would require a lot of back-and-forth manual interventions, etc. So, the idea here is to penalize the outliers instead of good candidates. This can easily be configured with a higher value if we want to disable the functionality. +| ignore_dcs | [] | This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. If you want to avoid repairing nodes in one or more data centers, then you can specify the data centers in this list. +| repair_primary_token_range_only | true | Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false'. It is the same as -pr in nodetool repair options. +| parallel_repair_count | 3 | The number of nodes running repair parallelly. If parallel_repair_percentage is set, it will choose the larger value of the two. This configuration controls how many nodes would run repair in parallel. The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. If one or more node(s) finish repair, then the framework automatically picks up the next candidate(s) based on the least repair time. It will ensure the maximum number of nodes running repair do not exceed “3”. +| parallel_repair_percentage | 3 | The percentage of nodes in the cluster that run repair parallelly. If parallel_repair_count is set, it will choose the larger value of the two. The problem with a fixed number of nodes (parallel_repair_count property) is that in a large-scale environment,the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. Extremely fewer manual interventions as it will rarely violate the repair SLA for customers. +| mv_repair_enabled | false | If the scheduler should repair MV table or not. +| initial_scheduler_delay | 5m | After a node restart, wait for this much delay before scheduler starts running repair; this is to avoid starting repair immediately after a node restart. +| repair_session_timeout | 3h | The major issue with Repair is sometimes repair session hangs; so this timeout is useful to resume such stuck repair sessions. +| force_repair_new_node | false | Whether to force immediate repair on new nodes. This is useful if you want to repair newly bootstrapped nodes immediately after they join the ring. +| token_range_splitter | org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter | Splitter implementation to use for generating repair assignments.The default is {@link RepairTokenRangeSplitter}. The class should implement {@link IAutoRepairTokenRangeSplitter} and have a constructor accepting ({@link RepairType}, {@link java.util.Map}) +| token_range_splitter.bytes_per_assignment | 200GiB |The target and maximum amount of bytes that should be included in a repair assignment. This is meant to scope the amount of work involved in a repair. For incremental repair, this involves the total number of bytes in all SSTables containing unrepaired data involving the ranges being repaired, including data that doesn't cover the range. This is to account for the amount of anticompaction that is expected. For all other repair types, this involves the amount of data covering the range being repaired. +| token_range_splitter.partitions_per_assignment | 1048576 | he target number of partitions that should be included in a repair assignment. This configuration exists to reduce excessive overstreaming. +| token_range_splitter.max_tables_per_assignment | 64 | The maximum number of tables that can be included in a repair assignment. This aims to reduce the number of repairs, especially in cases where a large amount of tables exists for a keyspace. Note that the splitter will avoid batching tables together if they exceed the other configuration parameters such as bytes_per_assignment and partitions_per_assignment +| token_range_splitter.max_bytes_per_schedule | 100000GiB | The maximum number of bytes to cover an individual schedule. This serves as a mechanism for throttling the amount of work that can be done on each repair cycle. One may opt to reduce this value if the impact of repairs is causing too many load on the cluster, or increase it if writes outpace the amount of data being repaired. Alternatively, one may want to choose tuning down or up the min_repair_interval. +|=== + + +=== Nodetool Configuration +==== nodetool getautorepairconfig +``` +$> nodetool getautorepairconfig +repair scheduler configuration: + repair_check_interval: 5m + repair_max_retries: 3 + repair_retry_backoff: 30s + repair_task_min_duration: 5s + history_clear_delete_hosts_buffer_interval: 2h +configuration for repair_type: full + enabled: true + min_repair_interval: 24h + repair_by_keyspace: false + number_of_repair_threads: 1 + sstable_upper_threshold: 10000 + table_max_repair_time: 6h + ignore_dcs: [] + repair_primary_token_range_only: true + parallel_repair_count: 3 + parallel_repair_percentage: 3 + mv_repair_enabled: false + initial_scheduler_delay: 5m + repair_session_timeout: 3h + force_repair_new_node: false + token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter + token_range_splitter.bytes_per_assignment: 200GiB + token_range_splitter.partitions_per_assignment: 1048576 + token_range_splitter.max_tables_per_assignment: 64 + token_range_splitter.max_bytes_per_schedule: 100000GiB +configuration for repair_type: incremental + enabled: true + min_repair_interval: 24h + repair_by_keyspace: false + number_of_repair_threads: 1 + sstable_upper_threshold: 10000 + table_max_repair_time: 6h + ignore_dcs: [] + repair_primary_token_range_only: true + parallel_repair_count: 3 + parallel_repair_percentage: 3 + mv_repair_enabled: false + initial_scheduler_delay: 5m + repair_session_timeout: 3h + force_repair_new_node: false + token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter + token_range_splitter.bytes_per_assignment: 50GiB + token_range_splitter.partitions_per_assignment: 1048576 + token_range_splitter.max_tables_per_assignment: 64 + token_range_splitter.max_bytes_per_schedule: 100GiB +configuration for repair_type: preview_repaired + enabled: false +``` + +==== nodetool autorepairstatus +``` +$> nodetool autorepairstatus -t incremental +Active Repairs +425cea55-09aa-46e0-8911-9f37a4424574 + + +$> nodetool autorepairstatus -t full +Active Repairs +NONE + +``` + +==== nodetool setautorepairconfig +``` +$> nodetool setautorepairconfig -t incremental number_of_repair_threads 2 +``` + + + +==== More details +https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-37+Apache+Cassandra+Automated+Repair+Solution[CEP-37] \ No newline at end of file diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 2199a3e10616..1cdaddafd462 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -414,47 +414,47 @@ protected static Options getDefaultOptions() // Enable/Disable full auto repair public volatile Boolean enabled; - // Repair table by table if this is set to 'false'. If it is 'true', then repair all the tables in a keyspace in one go. + // Repair table by table if this is set to 'false'. If it is 'true', then repair all the tables in a keyspace in one go. public volatile Boolean repair_by_keyspace; - // Number of repair threads to run for a given invoked Repair Job. - // Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. - // This is similar to -j for repair options for the nodetool repair command. + // Number of repair threads to run for a given invoked Repair Job. + // Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. + // This is similar to -j for repair options for the nodetool repair command. public volatile Integer number_of_repair_threads; - // The number of nodes running repair parallelly. If parallel_repair_percentage is set, it will choose the larger value of the two. The default is 3. - // This configuration controls how many nodes would run repair in parallel. - // The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. - // If one or more node(s) finish repair, then the framework automatically picks up the next candidate(s) based on the least repair time. - // It will ensure the maximum number of nodes running repair do not exceed “3”. + // The number of nodes running repair parallelly. If parallel_repair_percentage is set, it will choose the larger value of the two. + // This configuration controls how many nodes would run repair in parallel. + // The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. + // If one or more node(s) finish repair, then the framework automatically picks up the next candidate(s) based on the least repair time. + // It will ensure the maximum number of nodes running repair do not exceed “3”. public volatile Integer parallel_repair_count; - // The percentage of nodes in the cluster that run repair parallelly. If parallel_repair_count is set, it will choose the larger value of the two. - // The problem with a fixed number of nodes (parallel_repair_count property) is that in a large-scale environment, - // the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. - // The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. - // So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. - // Extremely fewer manual interventions as it will rarely violate the repair SLA for customers + // The percentage of nodes in the cluster that run repair parallelly. If parallel_repair_count is set, it will choose the larger value of the two. + // The problem with a fixed number of nodes (parallel_repair_count property) is that in a large-scale environment, + // the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. + // The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. + // So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. + // Extremely fewer manual interventions as it will rarely violate the repair SLA for customers. public volatile Integer parallel_repair_percentage; - // Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. - // This is to avoid penalizing good tables (neighbors) with an outlier. + // Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. + // This is to avoid penalizing good tables (neighbors) with an outlier. public volatile Integer sstable_upper_threshold; - // Minimum time in hours between repairing the same node again. This is useful for extremely tiny clusters, say 5 nodes, which finishes - // repair quicly. The default is 24 hours. This means that if the scheduler finishes one round on all the nodes in < 24 hours. On a given node it won’t start a new repair round - // until the last repair conducted on a given node is < 24 hours. + // Minimum time in hours between repairing the same node again. This is useful for extremely tiny clusters, say 5 nodes, which finishes + // repair quicly. The default is 24 hours. This means that if the scheduler finishes one round on all the nodes in < 24 hours. On a given node it won’t start a new repair round + // until the last repair conducted on a given node is < 24 hours. public volatile DurationSpec.IntSecondsBound min_repair_interval; - // This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. - // If you want to avoid repairing nodes in one or more data centers, then you can specify the data centers in this list. + // This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. + // If you want to avoid repairing nodes in one or more data centers, then you can specify the data centers in this list. public volatile Set ignore_dcs; - // Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false' - // It is the same as -pr in nodetool repair options. + // Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false'. + // It is the same as -pr in nodetool repair options. public volatile Boolean repair_primary_token_range_only; - // Whether to force immediate repair on new nodes. This is useful if you want to repair newly bootstrapped nodes immediately after they join the ring. + // Whether to force immediate repair on new nodes. This is useful if you want to repair newly bootstrapped nodes immediately after they join the ring. public volatile Boolean force_repair_new_node; - // Max time for repairing one table on a given node, if exceeded, skip the table. The default is 6 hours. - // Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. - // Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. - // Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator and would require a lot of back-and-forth manual interventions, etc. - // So, the idea here is to penalize the outliers instead of good candidates. This can easily be configured with a higher value if we want to disable the functionality. + // Max time for repairing one table on a given node, if exceeded, skip the table. + // Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. + // Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. + // Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator and would require a lot of back-and-forth manual interventions, etc. + // So, the idea here is to penalize the outliers instead of good candidates. This can easily be configured with a higher value if we want to disable the functionality. public volatile DurationSpec.IntSecondsBound table_max_repair_time; - // If the scheduler should repair MV table or not. + // If the scheduler should repair MV table or not. public volatile Boolean mv_repair_enabled; /** * Splitter implementation to use for generating repair assignments. @@ -463,9 +463,9 @@ protected static Options getDefaultOptions() * and have a constructor accepting ({@link RepairType}, {@link java.util.Map}) */ public volatile ParameterizedClass token_range_splitter; - // After a node restart, wait for this much delay before scheduler starts running repair; this is to avoid starting repair immediately after a node restart. + // After a node restart, wait for this much delay before scheduler starts running repair; this is to avoid starting repair immediately after a node restart. public volatile DurationSpec.IntSecondsBound initial_scheduler_delay; - // The major issue with Repair is sometimes repair session hangs; so this timeout is useful to resume such stuck repair sessions. + // The major issue with Repair is sometimes repair session hangs; so this timeout is useful to resume such stuck repair sessions. public volatile DurationSpec.IntSecondsBound repair_session_timeout; public String toString() diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index 7a1223626a60..76b6c88a5fdc 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -116,10 +116,10 @@ * max_tables_per_assignment: The maximum number of tables that can be included in a repair assignment. * This aims to reduce the number of repairs, especially in cases where a large amount of tables exists for * a keyspace. Note that the splitter will avoid batching tables together if they exceed the other - * configuration paramters such as bytes_per_assignment and partitions_per_assignment. + * configuration parameters such as bytes_per_assignment and partitions_per_assignment. * *
  • - * max_bytes_per_schedule: The maximum number of bytes to cover an individiual schedule. This serves + * max_bytes_per_schedule: The maximum number of bytes to cover an individual schedule. This serves * as a mechanism for throttling the amount of work that can be done on each repair cycle. One may opt to * reduce this value if the impact of repairs is causing too many load on the cluster, or increase it if * writes outpace the amount of data being repaired. Alternatively, one may want to choose tuning down or up From dd3facadd1c2c44cfd02d8fff85670ec128f9a09 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 15 Jan 2025 20:34:01 -0800 Subject: [PATCH 116/199] cr from Andy --- conf/cassandra.yaml | 207 +++++++++--------- .../cassandra/pages/auto-repair/overview.adoc | 158 +++++++++---- .../repair/autorepair/AutoRepairConfig.java | 134 +++++++----- .../repair/autorepair/AutoRepairUtils.java | 2 +- .../cassandra/service/AutoRepairService.java | 2 +- .../org/apache/cassandra/tools/NodeProbe.java | 2 +- .../tools/nodetool/GetAutoRepairConfig.java | 2 +- .../tools/nodetool/SetAutoRepairConfig.java | 6 +- .../config/YamlConfigurationLoaderTest.java | 2 +- .../autorepair/AutoRepairConfigTest.java | 65 +++++- .../AutoRepairParameterizedTest.java | 26 +-- .../nodetool/SetAutoRepairConfigTest.java | 2 +- 12 files changed, 385 insertions(+), 223 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 4a5ca8f43cca..d1ef50b9e701 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1866,103 +1866,110 @@ drop_compact_storage_enabled: false # excluded_keyspaces: # comma separated list of keyspaces to exclude from the check # excluded_tables: # comma separated list of keyspace.table pairs to exclude from the check -# Configuration for AutoRepair Scheduler -#auto_repair: -# # Enable/Disable auto repair scheduler. -# # If it is set to false, then the scheduler thread will not be spun up. -# # If it is set to true, then the repair scheduler thread will be created. The thread will -# # look for secondary config available for each repair type (full, incremental, and preview_repaired), -# # and based on that, it will schedule repairs. -# enabled: false -# # Time interval between successive checks for repair scheduler to check if either the ongoing repair is completed or if -# # none is going, then check if it's time to schedule or wait. -# repair_check_interval: 5m -# # Maximum number of retries for a repair session. -# repair_max_retries: 3 -# # Backoff time in seconds for retrying a repair session. -# repair_retry_backoff: 30s -# # Minimum duration for the execution of a single repair task (i.e.: RepairRunnable). -# # This helps prevent the auto-repair scheduler from overwhelming the node by scheduling a large number of -# # repair tasks in a short period of time. -# repair_task_min_duration: 5s -# # When any nodes leave the ring then the repair schedule needs to adjust the order, etc. -# # The AutoRepair scheduler keeps the deleted hosts information in its persisted metadata for the defined interval in this config. -# # This information is useful so the scheduler is absolutely sure that the node is indeed removed from the ring, and then it can adjust the repair schedule accordingly. -# # So, the duration in this config determinses for how long deleted host's information is kept in the scheduler's metadata. -# history_clear_delete_hosts_buffer_interval: 2h -# # Configuration specific to each repair type. Options: full, incremental, and preview_repaired. -# repair_type_overrides: -# full: -# # Enable/Disable full auto repair -# enabled: false -# # Minimum time in hours between repairing the same node again. This is useful for extremely tiny clusters, say 5 nodes, which finishes -# # repair quicly. The default is 24 hours. This means that if the scheduler finishes one round on all the nodes in < 24 hours. On a given node it won’t start a new repair round -# # until the last repair conducted on a given node is < 24 hours. -# min_repair_interval: 24h -# # Repair table by table if this is set to 'false'. If it is 'true', then repair all the tables in a keyspace in one go. -# repair_by_keyspace: false -# # Number of repair threads to run for a given invoked Repair Job. -# # Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. -# # This is similar to -j for repair options for the nodetool repair command. -# number_of_repair_threads: 1 -# # Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. -# # This is to avoid penalizing good tables (neighbors) with an outlier. -# sstable_upper_threshold: 10000 -# # Max time for repairing one table on a given node, if exceeded, skip the table. -# # Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. -# # Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. -# # Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator and would require a lot of back-and-forth manual interventions, etc. -# # So, the idea here is to penalize the outliers instead of good candidates. This can easily be configured with a higher value if we want to disable the functionality. -# table_max_repair_time: 6h -# # This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. -# # If you want to avoid repairing nodes in one or more data centers, then you can specify the data centers in this list. -# ignore_dcs: [] -# # Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false'. -# # It is the same as -pr in nodetool repair options. -# repair_primary_token_range_only: true -# # The number of nodes running repair parallelly. If parallel_repair_percentage is set, it will choose the larger value of the two. -# # This configuration controls how many nodes would run repair in parallel. -# # The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. -# # If one or more node(s) finish repair, then the framework automatically picks up the next candidate(s) based on the least repair time. -# # It will ensure the maximum number of nodes running repair do not exceed “3”. -# parallel_repair_count: 3 -# # The percentage of nodes in the cluster that run repair parallelly. If parallel_repair_count is set, it will choose the larger value of the two. -# # The problem with a fixed number of nodes (parallel_repair_count property) is that in a large-scale environment, -# # the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. -# # The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. -# # So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. -# # Extremely fewer manual interventions as it will rarely violate the repair SLA for customers. -# parallel_repair_percentage: 3 -# # If the scheduler should repair MV table or not. -# mv_repair_enabled: false -# # After a node restart, wait for this much delay before scheduler starts running repair; this is to avoid starting repair immediately after a node restart. -# initial_scheduler_delay: 5m -# # The major issue with Repair is sometimes repair session hangs; so this timeout is useful to resume such stuck repair sessions. -# repair_session_timeout: 3h -# # Whether to force immediate repair on new nodes. This is useful if you want to repair newly bootstrapped nodes immediately after they join the ring. -# force_repair_new_node: false -# # Splitter implementation to use for generating repair assignments. -# # The default is {@link RepairTokenRangeSplitter}. The class should implement {@link IAutoRepairTokenRangeSplitter} -# # and have a constructor accepting ({@link RepairType}, {@link java.util.Map}) -# token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter -# # The target and maximum amount of bytes that should be included in a repair -# # assignment. This is meant to scope the amount of work involved in a repair. For incremental repair, this -# # involves the total number of bytes in all SSTables containing unrepaired data involving the ranges being -# # repaired, including data that doesn't cover the range. This is to account for the amount of anticompaction -# # that is expected. For all other repair types, this involves the amount of data covering the range being -# # repaired. -# token_range_splitter.bytes_per_assignment: 200GiB -# # The target number of partitions that should be included in a repair -# # assignment. This configuration exists to reduce excessive overstreaming. -# token_range_splitter.partitions_per_assignment: 1048576 -# # The maximum number of tables that can be included in a repair assignment. -# # This aims to reduce the number of repairs, especially in cases where a large amount of tables exists for -# # a keyspace. Note that the splitter will avoid batching tables together if they exceed the other -# # configuration parameters such as bytes_per_assignment and partitions_per_assignment -# token_range_splitter.max_tables_per_assignment: 64 -# # The maximum number of bytes to cover an individual schedule. This serves -# # as a mechanism for throttling the amount of work that can be done on each repair cycle. One may opt to -# # reduce this value if the impact of repairs is causing too many load on the cluster, or increase it if -# # writes outpace the amount of data being repaired. Alternatively, one may want to choose tuning down or up -# # the min_repair_interval. -# token_range_splitter.max_bytes_per_schedule: 100000GiB +# Configuration for AutoRepair Scheduler. +# auto_repair: +# # Enable/Disable the auto-repair scheduler. +# # If set to false, the scheduler thread will not be started. +# # If set to true, the repair scheduler thread will be created. The thread will +# # check for secondary configuration available for each repair type (full, incremental, +# # and preview_repaired), and based on that, it will schedule repairs. +# enabled: true +# repair_type_overrides: +# full: +# # Enable/Disable full auto-repair +# enabled: true +# # Minimum duration between repairing the same node again. This is useful for tiny clusters, such as +# # clusters with 5 nodes that finish repairs quickly. The default is 24 hours. This means that if the scheduler +# # completes one round on all nodes in less than 24 hours, it will not start a new repair round on a given node +# # until 24 hours have passed since the last repair. +# min_repair_interval: 24h +# token_range_splitter: +# parameters: +# # The target and maximum amount of bytes that should be included in a repair +# # assignment. This scopes the amount of work involved in a repair and includes +# # the data covering the range being repaired. +# bytes_per_assignment: 200GiB +# # The maximum number of bytes to cover in an individual schedule. This serves as +# # a mechanism to throttle the work done in each repair cycle. You may reduce this +# # value if the impact of repairs is causing too much load on the cluster or increase it +# # if writes outpace the amount of data being repaired. Alternatively, adjust the +# # min_repair_interval. +# max_bytes_per_schedule: 100000GiB +# incremental: +# enabled: true +# # Incremental repairs finish quickly, so they can be run more frequently. The default is 1 hour. +# min_repair_interval: 1h +# token_range_splitter: +# parameters: +# # Configured to attempt repairing 50GiB of data per repair. +# # This throttles the amount of incremental repair and anticompaction done per schedule +# # after incremental repairs are turned on. +# bytes_per_assignment: 50GiB +# # The maximum number of bytes to cover in an individual schedule to 100GiB. +# # Consider increasing max_bytes_per_schedule if +# # more data is written than this limit within the min_repair_interval. +# max_bytes_per_schedule: 100GiB +# preview_repaired: +# enabled: true +# min_repair_interval: 24h +# token_range_splitter: +# parameters: +# bytes_per_assignment: 200GiB +# max_bytes_per_schedule: 100000GiB +# # Time interval between successive checks to see if ongoing repairs are complete or if it is time to schedule +# # repairs. +# repair_check_interval: 5m +# # Maximum number of retries for a repair session. +# repair_max_retries: 3 +# # Backoff time before retrying a repair session. +# repair_retry_backoff: 30s +# # Minimum duration for the execution of a single repair task. This prevents the scheduler from overwhelming +# # the node by scheduling too many repair tasks in a short period of time. +# repair_task_min_duration: 5s +# # The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata +# # for a specified duration to ensure they are indeed removed before adjustments are made to the schedule. +# history_clear_delete_hosts_buffer_interval: 2h +# global_settings: +# # If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired +# # individually. +# repair_by_keyspace: false +# # Number of threads to use for each repair job scheduled by the scheduler. Similar to the -j option in nodetool +# # repair. +# number_of_repair_threads: 1 +# # Number of nodes running repair in parallel. If parallel_repair_percentage is set, the larger value is used. +# parallel_repair_count: 3 +# # Percentage of nodes in the cluster running repair in parallel. If parallel_repair_count is set, the larger value +# # is used. Recommendation is that the repair cycle on the cluster should finish within gc_grace_seconds. +# parallel_repair_percentage: 3 +# # Repairs materialized view if true. +# materialized_view_repair_enabled: false +# # Delay before starting repairs after a node restarts to avoid repairs starting immediately after a restart. +# initial_scheduler_delay: 5m +# # Timeout for resuming stuck repair sessions. +# repair_session_timeout: 3h +# # Force immediate repair on new nodes after they join the ring. +# force_repair_new_node: false +# # Threshold to skip repairing tables with too many SSTables. Defaults to 10,000 SSTables to avoid penalizing good +# # tables. +# sstable_upper_threshold: 10000 +# # Maximum time allowed for repairing one table on a given node. If exceeded, the repair proceeds to the +# # next table. +# table_max_repair_time: 6h +# # Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify data +# # centers to exclude in this list. Note that repair sessions will still consider all replicas from excluded +# # data centers. Useful if you have keyspaces that are not replicated in certain data centers, and you want to +# # not run repair schedule in certain data centers. +# ignore_dcs: [] +# # Repair only the primary ranges owned by a node. Equivalent to the -pr option in nodetool repair. Defaults +# # to true. General advice is to keep this true. +# repair_primary_token_range_only: true +# # Splitter implementation to generate repair assignments. Defaults to RepairTokenRangeSplitter. +# token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter +# token_range_splitter: +# parameters: +# # Target number of partitions to include in a repair assignment to reduce excessive overstreaming. +# partitions_per_assignment: 1048576 +# # Maximum number of tables to include in a repair assignment. This reduces the number of repairs, +# # especially in keyspaces with many tables. The splitter avoids batching tables together if they +# # exceed other configuration parameters like bytes_per_assignment or partitions_per_assignment. +# max_tables_per_assignment: 64 diff --git a/doc/modules/cassandra/pages/auto-repair/overview.adoc b/doc/modules/cassandra/pages/auto-repair/overview.adoc index 4305e63a9810..d9961a5e752a 100644 --- a/doc/modules/cassandra/pages/auto-repair/overview.adoc +++ b/doc/modules/cassandra/pages/auto-repair/overview.adoc @@ -3,47 +3,129 @@ :description: Auto Repair concepts - How it works, how to configure it, and more. :keywords: CEP-37 -Repair orchestration inside Apache Cassandra. This fully Automated Repair scheduler inside Cassandra does not depend on the control plane, significantly reducing our operational overhead. -At a high level, a dedicated thread pool is assigned to the repair scheduler. The repair scheduler inside Cassandra maintains a new replicated table under a distributed system_distributed keyspace. This table maintains the repair history for all the nodes, such as when it was repaired the last time, etc. The scheduler will pick the node(s) that run the repair first and continue orchestration to ensure Every table and all of their token ranges are repaired. The algorithm can also run repairs simultaneously on multiple nodes and splits the token range into subranges with the necessary retry to handle transient failures. The automatic repair runs as soon as we start a Cassandra cluster, like Compaction, and does not require human-in-the-loop. -The scheduler supports Full, Incremental, and Preview repair types today with the following features. A new repair type, such as Paxos repair and any future repair types, should be extended with minimal dev work! +Auto Repair is a fully automated scheduler that provides repair orchestration within Apache Cassandra. This +significantly reduces operational overhead by eliminating the need for operators to deploy external tools to submit and +manage repairs. + +At a high level, a dedicated thread pool is assigned to the repair scheduler. The repair scheduler in Cassandra +maintains a new replicated table, system_distributed.auto_repair_history, which stores the repair history for all nodes, +including details such as the last repair time. The scheduler selects the node(s) to begin repairs and orchestrates the +process to ensure that every table and its token ranges are repaired. The algorithm can run repairs simultaneously on +multiple nodes and splits token ranges into subranges, with necessary retries to handle transient failures. Automatic +repair starts as soon as a Cassandra cluster is launched, similar to compaction, and does not require human +intervention. + +The scheduler currently supports Full, Incremental, and Preview repair types with the following features. New repair +types, such as Paxos repair or other future repair mechanisms, can be integrated with minimal development effort! + === Features -- Capability to run on multiple nodes simultaneously -- Default implementation and an interface to override the data set being repaired per repair session -- Ability to extend the token split algorithms with the following two implementations readily available: -- Splits token ranges by putting a max cap on the size of data being repaired as part of one repair session as well as a max cap at a table level - this is crucial for Incremental Repair (Default) -- Split tokens evenly based on the number of splits specified -- A new CQL table property to do the following: -- Disable repair type at a table level if one wants the scheduler to skip one or more tables -- Setting repair priority for certain tables to prioritize those tables over others -- Enabling/Disabling scheduler for each repair type dynamically -- Per Data Center per job configuration -- Rich Configuration for each repair type, e.g., full, incremental, or preview_repair -- Enough observability so an operator can configure alarms depending on their need +- Capability to run repairs on multiple nodes simultaneously. +- A default implementation and an interface to override the dataset being repaired per session. +- Extendable token split algorithms with two implementations readily available: +. Splits token ranges by placing a cap on the size of data repaired in one session and a maximum cap at the schedule +level (default). +. Splits tokens evenly based on the specified number of splits. +- A new CQL table property offering: +. The ability to disable specific repair types at the table level, allowing the scheduler to skip one or more tables. +. Configuring repair priorities for certain tables to prioritize them over others. +- Dynamic enablement or disablement of the scheduler for each repair type. +- Configurable settings tailored to each repair job. +- Rich configuration options for each repair type (e.g., Full, Incremental, or Preview repairs). +- Comprehensive observability features that allow operators to configure alarms as needed. === Yaml Configuration + +==== Top level settings +The following settings are defined at the top level of the configuration file and apply universally across all +repair types. + +[cols=",,",options="header",] +|=== +| Name | Default | Description +| enabled | false | Enable/Disable the auto-repair scheduler. If set to false, the scheduler thread will not be started. +If set to true, the repair scheduler thread will be created. The thread will check for secondary configuration available +for each repair type (full, incremental, and preview_repaired), and based on that, it will schedule repairs. +| repair_check_interval | 5m | Time interval between successive checks to see if ongoing repairs are complete or if it +is time to schedule repairs. +| repair_max_retries | 3 | Maximum number of retries for a repair session. +| repair_retry_backoff | 30s | Backoff time before retrying a repair session. +| repair_task_min_duration | 5s | Minimum duration for the execution of a single repair task. This prevents the +scheduler from overwhelming the node by scheduling too many repair tasks in a short period of time. +| history_clear_delete_hosts_buffer_interval | 2h | The scheduler needs to adjust its order when nodes leave the ring. +Deleted hosts are tracked in metadata. for a specified duration to ensure they are indeed removed before adjustments +are made to the schedule. +|=== + + +==== Repair level settings +The following settings can be tailored individually for each repair type. [cols=",,",options="header",] |=== | Name | Default | Description -| enabled | false | Enable/Disable auto repair scheduler -| min_repair_interval | 24h | Minimum time in hours between repairing the same node again. This is useful for extremely tiny clusters, say 5 nodes, which finishes. repair quicly. The default is 24 hours. This means that if the scheduler finishes one round on all the nodes in < 24 hours. On a given node it won’t start a new repair round until the last repair conducted on a given node is < 24 hours. -| repair_by_keyspace | false | Repair table by table if this is set to 'false'. If it is 'true', then repair all the tables in a keyspace in one go. -| number_of_repair_threads | 1 | Number of repair threads to run for a given invoked Repair Job. Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. This is similar to -j for repair options for the nodetool repair command. -| sstable_upper_threshold | 10000 | Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. This is to avoid penalizing good tables (neighbors) with an outlier. -| table_max_repair_time | 6h | Max time for repairing one table on a given node, if exceeded, skip the table. Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator and would require a lot of back-and-forth manual interventions, etc. So, the idea here is to penalize the outliers instead of good candidates. This can easily be configured with a higher value if we want to disable the functionality. -| ignore_dcs | [] | This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. If you want to avoid repairing nodes in one or more data centers, then you can specify the data centers in this list. -| repair_primary_token_range_only | true | Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false'. It is the same as -pr in nodetool repair options. -| parallel_repair_count | 3 | The number of nodes running repair parallelly. If parallel_repair_percentage is set, it will choose the larger value of the two. This configuration controls how many nodes would run repair in parallel. The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. If one or more node(s) finish repair, then the framework automatically picks up the next candidate(s) based on the least repair time. It will ensure the maximum number of nodes running repair do not exceed “3”. -| parallel_repair_percentage | 3 | The percentage of nodes in the cluster that run repair parallelly. If parallel_repair_count is set, it will choose the larger value of the two. The problem with a fixed number of nodes (parallel_repair_count property) is that in a large-scale environment,the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. Extremely fewer manual interventions as it will rarely violate the repair SLA for customers. -| mv_repair_enabled | false | If the scheduler should repair MV table or not. -| initial_scheduler_delay | 5m | After a node restart, wait for this much delay before scheduler starts running repair; this is to avoid starting repair immediately after a node restart. -| repair_session_timeout | 3h | The major issue with Repair is sometimes repair session hangs; so this timeout is useful to resume such stuck repair sessions. -| force_repair_new_node | false | Whether to force immediate repair on new nodes. This is useful if you want to repair newly bootstrapped nodes immediately after they join the ring. -| token_range_splitter | org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter | Splitter implementation to use for generating repair assignments.The default is {@link RepairTokenRangeSplitter}. The class should implement {@link IAutoRepairTokenRangeSplitter} and have a constructor accepting ({@link RepairType}, {@link java.util.Map}) -| token_range_splitter.bytes_per_assignment | 200GiB |The target and maximum amount of bytes that should be included in a repair assignment. This is meant to scope the amount of work involved in a repair. For incremental repair, this involves the total number of bytes in all SSTables containing unrepaired data involving the ranges being repaired, including data that doesn't cover the range. This is to account for the amount of anticompaction that is expected. For all other repair types, this involves the amount of data covering the range being repaired. -| token_range_splitter.partitions_per_assignment | 1048576 | he target number of partitions that should be included in a repair assignment. This configuration exists to reduce excessive overstreaming. -| token_range_splitter.max_tables_per_assignment | 64 | The maximum number of tables that can be included in a repair assignment. This aims to reduce the number of repairs, especially in cases where a large amount of tables exists for a keyspace. Note that the splitter will avoid batching tables together if they exceed the other configuration parameters such as bytes_per_assignment and partitions_per_assignment -| token_range_splitter.max_bytes_per_schedule | 100000GiB | The maximum number of bytes to cover an individual schedule. This serves as a mechanism for throttling the amount of work that can be done on each repair cycle. One may opt to reduce this value if the impact of repairs is causing too many load on the cluster, or increase it if writes outpace the amount of data being repaired. Alternatively, one may want to choose tuning down or up the min_repair_interval. +| enabled | false | Enable/Disable full/incremental/preview_repaired auto-repair +| repair_by_keyspace | false | If true, attempts to group tables in the same keyspace into one repair; otherwise, +each table is repaired individually. +| number_of_repair_threads | 1 | Number of threads to use for each repair job scheduled by the scheduler. Similar to +the -j option in nodetool repair. +| parallel_repair_count | 3 | Number of nodes running repair in parallel. If parallel_repair_percentage is set, the +larger value is used. +| parallel_repair_percentage | 3 | Percentage of nodes in the cluster running repair in parallel. If +parallel_repair_count is set, the larger value is used. Recommendation is that the repair cycle on the cluster should +finish within gc_grace_seconds. +| materialized_view_repair_enabled | false | Repairs materialized view if true. +| initial_scheduler_delay | 5m | Delay before starting repairs after a node restarts to avoid repairs starting +immediately after a restart. +| repair_session_timeout | 3h | Timeout for resuming stuck repair sessions. +| force_repair_new_node | false | Force immediate repair on new nodes after they join the ring. +| sstable_upper_threshold | 10000 | Threshold to skip repairing tables with too many SSTables. Defaults to 10,000 +SSTables to avoid penalizing good tables. +| table_max_repair_time | 6h | Maximum time allowed for repairing one table on a given node. If exceeded, the repair +proceeds to the next table. +| ignore_dcs | [] | Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify +data centers to exclude in this list. Note that repair sessions will still consider all replicas from excluded data +centers. Useful if you have keyspaces that are not replicated in certain data centers, and you want to not run repair +schedule in certain data centers. +| repair_primary_token_range_only | true | Repair only the primary ranges owned by a node. Equivalent to the -pr option +in nodetool repair. Defaults to true. General advice is to keep this true. +| token_range_splitter | org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter | Splitter implementation to +generate repair assignments. Defaults to RepairTokenRangeSplitter. +| token_range_splitter.partitions_per_assignment | 1048576 | Target number of partitions to include in a repair +assignment to reduce excessive overstreaming. +| token_range_splitter.max_tables_per_assignment | 64 | Maximum number of tables to include in a repair assignment. +This reduces the number of repairs, especially in keyspaces with many tables. The splitter avoids batching tables +together if they exceed other configuration parameters like bytes_per_assignment or partitions_per_assignment. +|=== + +==== Full & Preview_Repaired repair level settings +The following settings can be tailored individually for each repair type. +[cols=",,",options="header",] +|=== +| min_repair_interval | 24h | Minimum duration between repairing the same node again. This is useful for tiny clusters, +such as clusters with 5 nodes that finish repairs quickly. The default is 24 hours. This means that if the scheduler +completes one round on all nodes in less than 24 hours, it will not start a new repair round on a given node until 24 +hours have passed since the last repair. +| token_range_splitter.bytes_per_assignment | 200GiB | The target and maximum amount of bytes that should be included +in a repair assignment. This scopes the amount of work involved in a repair and includes the data covering the range +being repaired. +| token_range_splitter.max_bytes_per_schedule | 100000GiB | The maximum number of bytes to cover in an individual +schedule. This serves as a mechanism to throttle the work done in each repair cycle. You may reduce this value if the +impact of repairs is causing too much load on the cluster or increase it if writes outpace the amount of data being +repaired. Alternatively, adjust the min_repair_interval. +|=== + +==== Incremental repair level settings +The following settings can be customized for each repair type. +[cols=",,",options="header",] +|=== +| min_repair_interval | 1h | Incremental repairs finish quickly, so they can be run more frequently. The default is +1 hour. +| token_range_splitter.bytes_per_assignment | 50GiB | Configured to attempt repairing 50GiB of data per repair. +This throttles the amount of incremental repair and anticompaction done per schedule after incremental repairs are +turned on. +| token_range_splitter.max_bytes_per_schedule | 100GiB | The maximum number of bytes to cover in an individual schedule +to 100GiB. Consider increasing max_bytes_per_schedule if more data is written than this limit within +the min_repair_interval. |=== @@ -68,7 +150,7 @@ configuration for repair_type: full repair_primary_token_range_only: true parallel_repair_count: 3 parallel_repair_percentage: 3 - mv_repair_enabled: false + materialized_view_repair_enabled: false initial_scheduler_delay: 5m repair_session_timeout: 3h force_repair_new_node: false @@ -79,7 +161,7 @@ configuration for repair_type: full token_range_splitter.max_bytes_per_schedule: 100000GiB configuration for repair_type: incremental enabled: true - min_repair_interval: 24h + min_repair_interval: 1h repair_by_keyspace: false number_of_repair_threads: 1 sstable_upper_threshold: 10000 @@ -88,7 +170,7 @@ configuration for repair_type: incremental repair_primary_token_range_only: true parallel_repair_count: 3 parallel_repair_percentage: 3 - mv_repair_enabled: false + materialized_view_repair_enabled: false initial_scheduler_delay: 5m repair_session_timeout: 3h force_repair_new_node: false @@ -122,4 +204,4 @@ $> nodetool setautorepairconfig -t incremental number_of_repair_threads 2 ==== More details -https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-37+Apache+Cassandra+Automated+Repair+Solution[CEP-37] \ No newline at end of file +https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-37+Apache+Cassandra+Unified+Repair+Solution[CEP-37] \ No newline at end of file diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 1cdaddafd462..59a9f0b35840 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -39,26 +39,24 @@ public class AutoRepairConfig implements Serializable { - // Enable/Disable auto repair scheduler. If it is set to false, then the scheduler thread will not be spun up. - // If it is set to true, then the repair scheduler thread will be created. The thread will - // look for secondary config available for each repair type (full, incremental, and preview_repaired), - // and based on that, it will schedule repairs. + // Enable/Disable the auto-repair scheduler. + // If set to false, the scheduler thread will not be started. + // If set to true, the repair scheduler thread will be created. The thread will + // check for secondary configuration available for each repair type (full, incremental, + // and preview_repaired), and based on that, it will schedule repairs. public volatile Boolean enabled; - // Time interval between successive checks for repair scheduler to check if either the ongoing repair is completed or if - // none is going, then check if it's time to schedule or wait. + // Time interval between successive checks to see if ongoing repairs are complete or if it is time to schedule + // repairs. public final DurationSpec.IntSecondsBound repair_check_interval = new DurationSpec.IntSecondsBound("5m"); - // When any nodes leave the ring then the repair schedule needs to adjust the order, etc. - // The AutoRepair scheduler keeps the deleted hosts information in its persisted metadata for the defined interval in this config. - // This information is useful so the scheduler is absolutely sure that the node is indeed removed from the ring, and then it can adjust the repair schedule accordingly. - // So, the duration in this config determinses for how long deleted host's information is kept in the scheduler's metadata. + // The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata + // for a specified duration to ensure they are indeed removed before adjustments are made to the schedule. public volatile DurationSpec.IntSecondsBound history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound("2h"); // Maximum number of retries for a repair session. public volatile Integer repair_max_retries = 3; - // Backoff time in seconds for retrying a repair session. + // Backoff time before retrying a repair session. public volatile DurationSpec.LongSecondsBound repair_retry_backoff = new DurationSpec.LongSecondsBound("30s"); - // Minimum duration for the execution of a single repair task (i.e.: RepairRunnable). - // This helps prevent the auto-repair scheduler from overwhelming the node by scheduling a large number of - // repair tasks in a short period of time. + // Minimum duration for the execution of a single repair task. This prevents the scheduler from overwhelming + // the node by scheduling too many repair tasks in a short period of time. public volatile DurationSpec.LongSecondsBound repair_task_min_duration = new DurationSpec.LongSecondsBound("5s"); // global_settings overides Options.defaultOptions for all repair types @@ -278,14 +276,14 @@ public void setParallelRepairCount(RepairType repairType, int count) getOptions(repairType).parallel_repair_count = count; } - public boolean getMVRepairEnabled(RepairType repairType) + public boolean getMaterializedViewRepairEnabled(RepairType repairType) { - return applyOverrides(repairType, opt -> opt.mv_repair_enabled); + return applyOverrides(repairType, opt -> opt.materialized_view_repair_enabled); } - public void setMVRepairEnabled(RepairType repairType, boolean enabled) + public void setMaterializedViewRepairEnabled(RepairType repairType, boolean enabled) { - getOptions(repairType).mv_repair_enabled = enabled; + getOptions(repairType).materialized_view_repair_enabled = enabled; } public void setForceRepairNewNode(RepairType repairType, boolean forceRepairNewNode) @@ -382,7 +380,38 @@ public static class Options implements Serializable { // defaultOptions defines the default auto-repair behavior when no overrides are defined @VisibleForTesting - protected static final Options defaultOptions = getDefaultOptions(); + private static Map defaultOptions; + + private static Map initializeDefaultOptions() + { + Map options = new EnumMap<>(AutoRepairConfig.RepairType.class); + options.put(AutoRepairConfig.RepairType.FULL, getDefaultOptions()); + options.put(RepairType.INCREMENTAL, getDefaultOptions()); + options.put(RepairType.PREVIEW_REPAIRED, getDefaultOptions()); + + + options.get(RepairType.FULL).min_repair_interval = new DurationSpec.IntSecondsBound("24h"); + // Incremental repairs finish quickly, so they can be run more frequently. Hence, the default is 1 hour. + options.get(RepairType.INCREMENTAL).min_repair_interval = new DurationSpec.IntSecondsBound("1h"); + options.get(RepairType.PREVIEW_REPAIRED).min_repair_interval = new DurationSpec.IntSecondsBound("24h"); + + return options; + } + + public static Map getDefaultOptionsMap() + { + if (defaultOptions == null) + { + synchronized (AutoRepairConfig.class) + { + if (defaultOptions == null) + { + defaultOptions = initializeDefaultOptions(); + } + } + } + return defaultOptions; + } public Options() { @@ -399,12 +428,11 @@ protected static Options getDefaultOptions() opts.parallel_repair_count = 3; opts.parallel_repair_percentage = 3; opts.sstable_upper_threshold = 10000; - opts.min_repair_interval = new DurationSpec.IntSecondsBound("24h"); opts.ignore_dcs = new HashSet<>(); opts.repair_primary_token_range_only = true; opts.force_repair_new_node = false; opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h"); - opts.mv_repair_enabled = false; + opts.materialized_view_repair_enabled = false; opts.token_range_splitter = new ParameterizedClass(DEFAULT_SPLITTER.getName(), Collections.emptyMap()); opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("5m"); // 5 minutes opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h"); // 3 hours @@ -412,50 +440,42 @@ protected static Options getDefaultOptions() return opts; } - // Enable/Disable full auto repair + // Enable/Disable full or incremental or previewed_repair auto repair public volatile Boolean enabled; - // Repair table by table if this is set to 'false'. If it is 'true', then repair all the tables in a keyspace in one go. + // If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired + // individually. public volatile Boolean repair_by_keyspace; - // Number of repair threads to run for a given invoked Repair Job. - // Once the scheduler schedules one repair session, then howmany threads to use inside that job will be controlled through this parameter. - // This is similar to -j for repair options for the nodetool repair command. + // Number of threads to use for each repair job scheduled by the scheduler. Similar to the -j option in nodetool + // repair. public volatile Integer number_of_repair_threads; - // The number of nodes running repair parallelly. If parallel_repair_percentage is set, it will choose the larger value of the two. - // This configuration controls how many nodes would run repair in parallel. - // The value “3” means, at any given point in time, at most 3 nodes would be running repair in parallel. These selected nodes can be from any datacenters. - // If one or more node(s) finish repair, then the framework automatically picks up the next candidate(s) based on the least repair time. - // It will ensure the maximum number of nodes running repair do not exceed “3”. + // Number of nodes running repair in parallel. If parallel_repair_percentage is set, the larger value is used. public volatile Integer parallel_repair_count; - // The percentage of nodes in the cluster that run repair parallelly. If parallel_repair_count is set, it will choose the larger value of the two. - // The problem with a fixed number of nodes (parallel_repair_count property) is that in a large-scale environment, - // the nodes keep getting added/removed due to elasticity, so if we have a fixed number, then manual interventions would increase because, on a continuous basis,operators would have to adjust to meet the SLA. - // The default is 3%, which means that 3% of the nodes in the Cassandra cluster would be repaired in parallel. - // So now, if a fleet, an operator won't have to worry about changing the repair frequency, etc., as overall repair time will continue to remain the same even if nodes are added or removed due to elasticity. - // Extremely fewer manual interventions as it will rarely violate the repair SLA for customers. + // Percentage of nodes in the cluster running repair in parallel. If parallel_repair_count is set, the larger value + // is used. Recommendation is that the repair cycle on the cluster should finish within gc_grace_seconds. public volatile Integer parallel_repair_percentage; - // Threshold to skip a table if it has too many sstables. The default is 10000. This means, if a table on a node has 10000 or more SSTables, then that table will be skipped. - // This is to avoid penalizing good tables (neighbors) with an outlier. + // Threshold to skip repairing tables with too many SSTables. Defaults to 10,000 SSTables to avoid penalizing good + // tables. public volatile Integer sstable_upper_threshold; - // Minimum time in hours between repairing the same node again. This is useful for extremely tiny clusters, say 5 nodes, which finishes - // repair quicly. The default is 24 hours. This means that if the scheduler finishes one round on all the nodes in < 24 hours. On a given node it won’t start a new repair round - // until the last repair conducted on a given node is < 24 hours. + // Minimum duration between repairing the same node again. This is useful for tiny clusters, such as + // clusters with 5 nodes that finish repairs quickly. The default is 24 hours. This means that if the scheduler + // completes one round on all nodes in less than 24 hours, it will not start a new repair round on a given node + // until 24 hours have passed since the last repair. public volatile DurationSpec.IntSecondsBound min_repair_interval; - // This is useful if you want to completely avoid running repairs in one or more data centers. By default, it is empty, i.e., the framework will repair nodes in all the datacenters. - // If you want to avoid repairing nodes in one or more data centers, then you can specify the data centers in this list. + // Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify data + // centers to exclude in this list. Note that repair sessions will still consider all replicas from excluded + // data centers. Useful if you have keyspaces that are not replicated in certain data centers, and you want to + // not run repair schedule in certain data centers. public volatile Set ignore_dcs; - // Set this 'true' if AutoRepair should repair only the primary ranges owned by this node; else, 'false'. - // It is the same as -pr in nodetool repair options. + // Repair only the primary ranges owned by a node. Equivalent to the -pr option in nodetool repair. Defaults + // to true. General advice is to keep this true. public volatile Boolean repair_primary_token_range_only; - // Whether to force immediate repair on new nodes. This is useful if you want to repair newly bootstrapped nodes immediately after they join the ring. + // Force immediate repair on new nodes after they join the ring. public volatile Boolean force_repair_new_node; - // Max time for repairing one table on a given node, if exceeded, skip the table. - // Let's say there is a Cassandra cluster in that there are 10 tables belonging to 10 different customers. - // Out of these 10 tables, 1 table is humongous. Repairing this 1 table, say, takes 5 days, in the worst case, but others could finish in just 1 hour. - // Then we would penalize 9 customers just because of one bad actor, and those 9 customers would ping an operator and would require a lot of back-and-forth manual interventions, etc. - // So, the idea here is to penalize the outliers instead of good candidates. This can easily be configured with a higher value if we want to disable the functionality. + // Maximum time allowed for repairing one table on a given node. If exceeded, the repair proceeds to the + // next table. public volatile DurationSpec.IntSecondsBound table_max_repair_time; - // If the scheduler should repair MV table or not. - public volatile Boolean mv_repair_enabled; + // Repairs materialized view if true. + public volatile Boolean materialized_view_repair_enabled; /** * Splitter implementation to use for generating repair assignments. *

    @@ -465,7 +485,7 @@ protected static Options getDefaultOptions() public volatile ParameterizedClass token_range_splitter; // After a node restart, wait for this much delay before scheduler starts running repair; this is to avoid starting repair immediately after a node restart. public volatile DurationSpec.IntSecondsBound initial_scheduler_delay; - // The major issue with Repair is sometimes repair session hangs; so this timeout is useful to resume such stuck repair sessions. + // Timeout for resuming stuck repair sessions. public volatile DurationSpec.IntSecondsBound repair_session_timeout; public String toString() @@ -482,7 +502,7 @@ public String toString() ", repair_primary_token_range_only=" + repair_primary_token_range_only + ", force_repair_new_node=" + force_repair_new_node + ", table_max_repair_time=" + table_max_repair_time + - ", mv_repair_enabled=" + mv_repair_enabled + + ", materialized_view_repair_enabled=" + materialized_view_repair_enabled + ", token_range_splitter=" + token_range_splitter + ", intial_scheduler_delay=" + initial_scheduler_delay + ", repair_session_timeout=" + repair_session_timeout + @@ -521,6 +541,6 @@ protected T applyOverrides(RepairType repairType, Function optio } // Otherwise check defaults - return getOverride(Options.defaultOptions, optionSupplier); + return getOverride(Options.getDefaultOptionsMap().get(repairType), optionSupplier); } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index 6057e8254587..b8cd8e0a940f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -830,7 +830,7 @@ public static boolean keyspaceMaxRepairTimeExceeded(RepairType repairType, long public static List getAllMVs(RepairType repairType, Keyspace keyspace, TableMetadata tableMetadata) { List allMvs = new ArrayList<>(); - if (AutoRepairService.instance.getAutoRepairConfig().getMVRepairEnabled(repairType) && keyspace.getMetadata().views != null) + if (AutoRepairService.instance.getAutoRepairConfig().getMaterializedViewRepairEnabled(repairType) && keyspace.getMetadata().views != null) { Iterator views = keyspace.getMetadata().views.forTable(tableMetadata.id).iterator(); while (views.hasNext()) diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 4056e6e6a930..74a8259a4a3c 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -182,7 +182,7 @@ public void setParallelRepairCount(RepairType repairType, int count) public void setMVRepairEnabled(RepairType repairType, boolean enabled) { - config.setMVRepairEnabled(repairType, enabled); + config.setMaterializedViewRepairEnabled(repairType, enabled); } public void setRepairSessionTimeout(RepairType repairType, String timeout) diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 02e77bf00b9b..296bb862f1d3 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2246,7 +2246,7 @@ public void setPrimaryTokenRangeOnly(AutoRepairConfig.RepairType repairType, boo autoRepairProxy.setPrimaryTokenRangeOnly(repairType, primaryTokenRangeOnly); } - public void setMVRepairEnabled(AutoRepairConfig.RepairType repairType, boolean enabled) + public void setMaterializedViewRepairEnabled(AutoRepairConfig.RepairType repairType, boolean enabled) { autoRepairProxy.setMVRepairEnabled(repairType, enabled); } diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java index 296049f78408..de234893cb49 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java @@ -85,7 +85,7 @@ private String formatRepairTypeConfig(NodeProbe probe, RepairType repairType, Au appendConfig(sb , "repair_primary_token_range_only", config.getRepairPrimaryTokenRangeOnly(repairType)); appendConfig(sb , "parallel_repair_count", config.getParallelRepairCount(repairType)); appendConfig(sb , "parallel_repair_percentage", config.getParallelRepairPercentage(repairType)); - appendConfig(sb , "mv_repair_enabled", config.getMVRepairEnabled(repairType)); + appendConfig(sb , "materialized_view_repair_enabled", config.getMaterializedViewRepairEnabled(repairType)); appendConfig(sb , "initial_scheduler_delay", config.getInitialSchedulerDelay(repairType)); appendConfig(sb , "repair_session_timeout", config.getRepairSessionTimeout(repairType)); appendConfig(sb , "force_repair_new_node", config.getForceRepairNewNode(repairType)); diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index d62b5b1d95db..b71ae85195e1 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -46,7 +46,7 @@ public class SetAutoRepairConfig extends NodeToolCmd "[start_scheduler|number_of_repair_threads|min_repair_interval|sstable_upper_threshold" + "|enabled|table_max_repair_time|priority_hosts|forcerepair_hosts|ignore_dcs" + "|history_clear_delete_hosts_buffer_interval|repair_primary_token_range_only" + - "|parallel_repair_count|parallel_repair_percentage|mv_repair_enabled|repair_max_retries" + + "|parallel_repair_count|parallel_repair_percentage|materialized_view_repair_enabled|repair_max_retries" + "|repair_retry_backoff|repair_session_timeout|min_repair_task_duration|token_range_splitter.]", required = true) protected List args = new ArrayList<>(); @@ -159,8 +159,8 @@ public void execute(NodeProbe probe) case "parallel_repair_percentage": probe.setParallelRepairPercentage(repairType, Integer.parseInt(paramVal)); break; - case "mv_repair_enabled": - probe.setMVRepairEnabled(repairType, Boolean.parseBoolean(paramVal)); + case "materialized_view_repair_enabled": + probe.setMaterializedViewRepairEnabled(repairType, Boolean.parseBoolean(paramVal)); break; case "repair_session_timeout": probe.setRepairSessionTimeout(repairType, paramVal); diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java index ad914c261a78..5e21228b6976 100644 --- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java +++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java @@ -254,7 +254,7 @@ public void fromMapTest() assertEquals(new DataStorageSpec.IntBytesBound("5B"), config.internode_socket_receive_buffer_size); // Check names backward compatibility (CASSANDRA-17141 and CASSANDRA-15234) assertEquals(true, config.auto_repair.enabled); assertEquals(new DurationSpec.IntSecondsBound("6h"), config.auto_repair.getAutoRepairTableMaxRepairTime(AutoRepairConfig.RepairType.INCREMENTAL)); - config.auto_repair.setMVRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, false); + config.auto_repair.setMaterializedViewRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, false); } @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 8f02baa52f57..b95b8dd84b21 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -18,6 +18,8 @@ package org.apache.cassandra.repair.autorepair; +import java.util.HashSet; +import java.util.Map; import java.util.Objects; import java.util.Collections; import java.util.Set; @@ -39,6 +41,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @RunWith(Parameterized.class) @@ -305,11 +308,11 @@ public void testSetParallelRepairCountInGroup() } @Test - public void testGetMVRepairEnabled() + public void testGetMaterializedViewRepairEnabled() { - config.global_settings.mv_repair_enabled = true; + config.global_settings.materialized_view_repair_enabled = true; - boolean result = config.getMVRepairEnabled(repairType); + boolean result = config.getMaterializedViewRepairEnabled(repairType); assertTrue(result); } @@ -317,9 +320,9 @@ public void testGetMVRepairEnabled() @Test public void testSetMVRepairEnabled() { - config.setMVRepairEnabled(repairType, true); + config.setMaterializedViewRepairEnabled(repairType, true); - assertTrue(config.getOptions(repairType).mv_repair_enabled); + assertTrue(config.getOptions(repairType).materialized_view_repair_enabled); } @Test @@ -363,7 +366,7 @@ public void testGetDefaultOptionsMVRepairIsEnabledByDefault() { Options defaultOptions = Options.getDefaultOptions(); - assertFalse(defaultOptions.mv_repair_enabled); + assertFalse(defaultOptions.materialized_view_repair_enabled); } @Test @@ -409,4 +412,54 @@ public void testSetRepairSessionTimeout() assert config.getOptions(repairType).repair_session_timeout.toSeconds() == 3600; } + @Test + public void testDefaultOptions() + { + Map defaultOptions = Options.getDefaultOptionsMap(); + Options options = defaultOptions.get(repairType); + assertFalse(options.enabled); + assertFalse(options.repair_by_keyspace); + assertEquals(Integer.valueOf(1), options.number_of_repair_threads); + assertEquals(Integer.valueOf(3), options.parallel_repair_count); + assertEquals(Integer.valueOf(3), options.parallel_repair_percentage); + assertEquals(Integer.valueOf(10000), options.sstable_upper_threshold); + assertEquals(new HashSet<>(), options.ignore_dcs); + assertTrue(options.repair_primary_token_range_only); + assertFalse(options.force_repair_new_node); + assertEquals(new DurationSpec.IntSecondsBound("6h"), options.table_max_repair_time); + assertFalse(options.materialized_view_repair_enabled); + assertEquals(new ParameterizedClass(RepairTokenRangeSplitter.class.getName(), Collections.emptyMap()), options.token_range_splitter); + assertEquals(new DurationSpec.IntSecondsBound("5m"), options.initial_scheduler_delay); + assertEquals(new DurationSpec.IntSecondsBound("3h"), options.repair_session_timeout); + + if (repairType == AutoRepairConfig.RepairType.INCREMENTAL) + { + assertEquals(new DurationSpec.IntSecondsBound("1h"), options.min_repair_interval); + } + else + { + assertEquals(new DurationSpec.IntSecondsBound("24h"), options.min_repair_interval); + } + } + + @Test + public void testGlobalOptions() + { + AutoRepairConfig config = new AutoRepairConfig(); + assertFalse(config.global_settings.enabled); + assertFalse(config.global_settings.repair_by_keyspace); + assertEquals(Integer.valueOf(1), config.global_settings.number_of_repair_threads); + assertEquals(Integer.valueOf(3), config.global_settings.parallel_repair_count); + assertEquals(Integer.valueOf(3), config.global_settings.parallel_repair_percentage); + assertEquals(Integer.valueOf(10000), config.global_settings.sstable_upper_threshold); + assertEquals(new HashSet<>(), config.global_settings.ignore_dcs); + assertTrue(config.global_settings.repair_primary_token_range_only); + assertFalse(config.global_settings.force_repair_new_node); + assertEquals(new DurationSpec.IntSecondsBound("6h"), config.global_settings.table_max_repair_time); + assertFalse(config.global_settings.materialized_view_repair_enabled); + assertEquals(new ParameterizedClass(RepairTokenRangeSplitter.class.getName(), Collections.emptyMap()), config.global_settings.token_range_splitter); + assertEquals(new DurationSpec.IntSecondsBound("5m"), config.global_settings.initial_scheduler_delay); + assertEquals(new DurationSpec.IntSecondsBound("3h"), config.global_settings.repair_session_timeout); + assertNull(config.global_settings.min_repair_interval); + } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 7a09f46a5d79..0ecba02fe77c 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -238,7 +238,7 @@ private void resetConfig() for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) { defaultConfig.setAutoRepairEnabled(repairType, true); - defaultConfig.setMVRepairEnabled(repairType, false); + defaultConfig.setMaterializedViewRepairEnabled(repairType, false); } // reset the AutoRepairService config to default @@ -398,15 +398,15 @@ public void testRepairPrimaryRangesByDefault() public void testGetAllMVs() { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - config.setMVRepairEnabled(repairType, false); - assertFalse(config.getMVRepairEnabled(repairType)); + config.setMaterializedViewRepairEnabled(repairType, false); + assertFalse(config.getMaterializedViewRepairEnabled(repairType)); assertEquals(0, AutoRepairUtils.getAllMVs(repairType, keyspace, cfm).size()); - config.setMVRepairEnabled(repairType, true); + config.setMaterializedViewRepairEnabled(repairType, true); - assertTrue(config.getMVRepairEnabled(repairType)); + assertTrue(config.getMaterializedViewRepairEnabled(repairType)); assertEquals(Arrays.asList(MV), AutoRepairUtils.getAllMVs(repairType, keyspace, cfm)); - config.setMVRepairEnabled(repairType, false); + config.setMaterializedViewRepairEnabled(repairType, false); } @@ -414,20 +414,20 @@ public void testGetAllMVs() public void testMVRepair() { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - config.setMVRepairEnabled(repairType, true); + config.setMaterializedViewRepairEnabled(repairType, true); config.setRepairMinInterval(repairType, "0s"); AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); AutoRepair.instance.repair(repairType); assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); - config.setMVRepairEnabled(repairType, false); + config.setMaterializedViewRepairEnabled(repairType, false); AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); AutoRepair.instance.repair(repairType); assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); - config.setMVRepairEnabled(repairType, true); + config.setMaterializedViewRepairEnabled(repairType, true); AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); AutoRepair.instance.repair(repairType); assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); @@ -463,7 +463,7 @@ public void testSkipRepairSSTableCountHigherThreshold() assert diffMVTable.size() == 10; int beforeCount = config.getRepairSSTableCountHigherThreshold(repairType); - config.setMVRepairEnabled(repairType, true); + config.setMaterializedViewRepairEnabled(repairType, true); config.setRepairSSTableCountHigherThreshold(repairType, 9); assertEquals(0, state.getSkippedTokenRangesCount()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); @@ -506,7 +506,7 @@ public void testGetRepairState() public void testMetrics() { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - config.setMVRepairEnabled(repairType, true); + config.setMaterializedViewRepairEnabled(repairType, true); config.setRepairMinInterval(repairType, "0s"); config.setRepairRetryBackoff("0s"); config.setAutoRepairTableMaxRepairTime(repairType, "0s"); @@ -544,7 +544,7 @@ public void testMetrics() public void testRepairWaitsForRepairToFinishBeforeSchedullingNewSession() throws Exception { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - config.setMVRepairEnabled(repairType, false); + config.setMaterializedViewRepairEnabled(repairType, false); config.setRepairRetryBackoff("0s"); when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())) .thenReturn(repairRunnable); @@ -641,7 +641,7 @@ public void testRepairShufflesKeyspacesAndTables() public void testRepairTakesLastRepairTimeFromDB() { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - config.setMVRepairEnabled(repairType, true); + config.setMaterializedViewRepairEnabled(repairType, true); long lastRepairTime = System.currentTimeMillis() - 1000; AutoRepairUtils.insertNewRepairHistory(repairType, 0, lastRepairTime); AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(0); diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index 6a068dc55f15..5f3be97cbbac 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -284,7 +284,7 @@ public static Collection testCases() forEachRepairType("repair_primary_token_range_only", "true", (type) -> verify(probe, times(1)).setPrimaryTokenRangeOnly(type, true)), forEachRepairType("parallel_repair_count", "6", (type) -> verify(probe, times(1)).setParallelRepairCount(type, 6)), forEachRepairType("parallel_repair_percentage", "7", (type) -> verify(probe, times(1)).setParallelRepairPercentage(type, 7)), - forEachRepairType("mv_repair_enabled", "true", (type) -> verify(probe, times(1)).setMVRepairEnabled(type, true)), + forEachRepairType("materialized_view_repair_enabled", "true", (type) -> verify(probe, times(1)).setMaterializedViewRepairEnabled(type, true)), forEachRepairType("ignore_dcs", "dc1,dc2", (type) -> verify(probe, times(1)).setAutoRepairIgnoreDCs(type, ImmutableSet.of("dc1", "dc2"))), forEachRepairType("token_range_splitter.max_bytes_per_schedule", "500GiB", (type) -> verify(probe, times(1)).setAutoRepairTokenRangeSplitterParameter(type, "max_bytes_per_schedule", "500GiB")) ).flatMap(Function.identity()).collect(Collectors.toList()); From 8f2d83cf56a04e0078dae4f600229dc5f8ad1579 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 20 Jan 2025 20:07:43 -0800 Subject: [PATCH 117/199] address cr from Andy --- conf/cassandra.yaml | 14 +++++++--- .../cassandra/pages/auto-repair/overview.adoc | 12 ++++++--- .../repair/autorepair/AutoRepairConfig.java | 6 +++-- .../autorepair/RepairTokenRangeSplitter.java | 2 ++ .../cassandra/service/AutoRepairService.java | 4 +-- .../AutoRepairParameterizedTest.java | 26 +++++++++++++++++++ .../repair/autorepair/AutoRepairTest.java | 13 ++++++++++ .../RepairTokenRangeSplitterTest.java | 1 - .../service/AutoRepairServiceTest.java | 4 ++- 9 files changed, 68 insertions(+), 14 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index d1ef50b9e701..ee3715cf38d8 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1894,10 +1894,15 @@ drop_compact_storage_enabled: false # # value if the impact of repairs is causing too much load on the cluster or increase it # # if writes outpace the amount of data being repaired. Alternatively, adjust the # # min_repair_interval. +# # This is set to a large value for full repair to attempt to repair all data per repair schedule. # max_bytes_per_schedule: 100000GiB # incremental: # enabled: true -# # Incremental repairs finish quickly, so they can be run more frequently. The default is 1 hour. +# # Incremental repairs operate over unrepaired data and should finish quickly. Running them more frequently keeps +# # the unrepaired set smaller and thus causes repairs to operate over a smaller set of data, so a more frequent +# # schedule such as 1h is recommended. +# # When turning on incremental repair for the first time with a decent amount of data it may be advisable to +# # increase this interval to 24h or longer to reduce the impact of anticompaction caused by incremental repair. # min_repair_interval: 1h # token_range_splitter: # parameters: @@ -1929,6 +1934,7 @@ drop_compact_storage_enabled: false # # The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata # # for a specified duration to ensure they are indeed removed before adjustments are made to the schedule. # history_clear_delete_hosts_buffer_interval: 2h +# # NOTE: Each of the below settings can be overridden per repair type under repair_type_overrides # global_settings: # # If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired # # individually. @@ -1941,7 +1947,7 @@ drop_compact_storage_enabled: false # # Percentage of nodes in the cluster running repair in parallel. If parallel_repair_count is set, the larger value # # is used. Recommendation is that the repair cycle on the cluster should finish within gc_grace_seconds. # parallel_repair_percentage: 3 -# # Repairs materialized view if true. +# # Repairs materialized views if true. # materialized_view_repair_enabled: false # # Delay before starting repairs after a node restarts to avoid repairs starting immediately after a restart. # initial_scheduler_delay: 5m @@ -1963,9 +1969,9 @@ drop_compact_storage_enabled: false # # Repair only the primary ranges owned by a node. Equivalent to the -pr option in nodetool repair. Defaults # # to true. General advice is to keep this true. # repair_primary_token_range_only: true -# # Splitter implementation to generate repair assignments. Defaults to RepairTokenRangeSplitter. -# token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter # token_range_splitter: +# # Splitter implementation to generate repair assignments. Defaults to RepairTokenRangeSplitter. +# class_name: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter # parameters: # # Target number of partitions to include in a repair assignment to reduce excessive overstreaming. # partitions_per_assignment: 1048576 diff --git a/doc/modules/cassandra/pages/auto-repair/overview.adoc b/doc/modules/cassandra/pages/auto-repair/overview.adoc index d9961a5e752a..1eb54a595ccd 100644 --- a/doc/modules/cassandra/pages/auto-repair/overview.adoc +++ b/doc/modules/cassandra/pages/auto-repair/overview.adoc @@ -73,7 +73,7 @@ larger value is used. | parallel_repair_percentage | 3 | Percentage of nodes in the cluster running repair in parallel. If parallel_repair_count is set, the larger value is used. Recommendation is that the repair cycle on the cluster should finish within gc_grace_seconds. -| materialized_view_repair_enabled | false | Repairs materialized view if true. +| materialized_view_repair_enabled | false | Repairs materialized views if true. | initial_scheduler_delay | 5m | Delay before starting repairs after a node restarts to avoid repairs starting immediately after a restart. | repair_session_timeout | 3h | Timeout for resuming stuck repair sessions. @@ -111,15 +111,19 @@ being repaired. | token_range_splitter.max_bytes_per_schedule | 100000GiB | The maximum number of bytes to cover in an individual schedule. This serves as a mechanism to throttle the work done in each repair cycle. You may reduce this value if the impact of repairs is causing too much load on the cluster or increase it if writes outpace the amount of data being -repaired. Alternatively, adjust the min_repair_interval. +repaired. Alternatively, adjust the min_repair_interval. This is set to a large value for full repair to attempt to +repair all data per repair schedule. |=== ==== Incremental repair level settings The following settings can be customized for each repair type. [cols=",,",options="header",] |=== -| min_repair_interval | 1h | Incremental repairs finish quickly, so they can be run more frequently. The default is -1 hour. +| min_repair_interval | 1h | Incremental repairs operate over unrepaired data and should finish quickly. Running them +more frequently keeps the unrepaired set smaller and thus causes repairs to operate over a smaller set of data, +so a more frequent schedule such as 1h is recommended. When turning on incremental repair for the first time with a +decent amount of data it may be advisable to increase this interval to 24h or longer to reduce the impact of +anticompaction caused by incremental repair. | token_range_splitter.bytes_per_assignment | 50GiB | Configured to attempt repairing 50GiB of data per repair. This throttles the amount of incremental repair and anticompaction done per schedule after incremental repairs are turned on. diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 59a9f0b35840..af287a14e71e 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -391,7 +391,9 @@ private static Map initializeDefaultOption options.get(RepairType.FULL).min_repair_interval = new DurationSpec.IntSecondsBound("24h"); - // Incremental repairs finish quickly, so they can be run more frequently. Hence, the default is 1 hour. + // Incremental repairs operate over unrepaired data and should finish quickly. Running them more frequently keeps + // the unrepaired set smaller and thus causes repairs to operate over a smaller set of data, so a more frequent + // increase this interval to 24h or longer to reduce the impact of anticompaction caused by incremental repair. options.get(RepairType.INCREMENTAL).min_repair_interval = new DurationSpec.IntSecondsBound("1h"); options.get(RepairType.PREVIEW_REPAIRED).min_repair_interval = new DurationSpec.IntSecondsBound("24h"); @@ -474,7 +476,7 @@ protected static Options getDefaultOptions() // Maximum time allowed for repairing one table on a given node. If exceeded, the repair proceeds to the // next table. public volatile DurationSpec.IntSecondsBound table_max_repair_time; - // Repairs materialized view if true. + // Repairs materialized views if true. public volatile Boolean materialized_view_repair_enabled; /** * Splitter implementation to use for generating repair assignments. diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index 76b6c88a5fdc..424454cf6304 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -131,6 +131,8 @@ *

  • * full: Configured in a way that attempts to accomplish repairing all data in a schedule, with * individual repairs targeting at most 200GiB of data and 1048576 partitions. + * max_bytes_per_schedule is set to a large value for full repair to attempt to repair all data per + * repair schedule. *
      *
    • bytes_per_assignment: 200GiB
      • *
    • partitions_per_assignment: 2^repair_session_max_tree_depth (2^20 == 1048576 by default)
      • diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 74a8259a4a3c..9e3b56d18a27 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -65,10 +65,10 @@ public void checkCanRun(RepairType repairType) if (repairType != RepairType.INCREMENTAL) return; - if (DatabaseDescriptor.isMaterializedViewsOnRepairEnabled()) + if (config.getMaterializedViewRepairEnabled(repairType) && DatabaseDescriptor.isMaterializedViewsOnRepairEnabled()) throw new ConfigurationException("Cannot run incremental repair while materialized view replay is enabled. Set materialized_views_on_repair_enabled to false."); - if (DatabaseDescriptor.isCDCOnRepairEnabled()) + if (DatabaseDescriptor.isCDCEnabled() && DatabaseDescriptor.isCDCOnRepairEnabled()) throw new ConfigurationException("Cannot run incremental repair while CDC replay is enabled. Set cdc_on_repair_enabled to false."); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 0ecba02fe77c..1f169ef80685 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -707,11 +707,36 @@ public void testRepairSuccessAfterRetry() verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); } + @Test + public void testRepairDoesNotThrowsForIRWithMVReplayButMVRepairDisabled() + { + AutoRepair.instance.setup(); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); + AutoRepairService.instance.getAutoRepairConfig().setMaterializedViewRepairEnabled(repairType, false); + + if (repairType == AutoRepairConfig.RepairType.INCREMENTAL) + { + try + { + AutoRepair.instance.repair(repairType); + } + catch (ConfigurationException ignored) + { + fail("ConfigurationException not expected"); + } + } + else + { + AutoRepair.instance.repair(repairType); + } + } + @Test public void testRepairThrowsForIRWithMVReplay() { AutoRepair.instance.setup(); DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); + AutoRepairService.instance.getAutoRepairConfig().setMaterializedViewRepairEnabled(repairType, true); if (repairType == AutoRepairConfig.RepairType.INCREMENTAL) { @@ -735,6 +760,7 @@ public void testRepairThrowsForIRWithMVReplay() public void testRepairThrowsForIRWithCDCReplay() { AutoRepair.instance.setup(); + DatabaseDescriptor.setCDCEnabled(true); DatabaseDescriptor.setCDCOnRepairEnabled(true); if (repairType == AutoRepairConfig.RepairType.INCREMENTAL) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index a7d5eb2f9e48..348e5d89a565 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -107,10 +107,23 @@ public void testSetupFailsWhenIREnabledWithCDCReplay() instance.setup(); } + @Test + public void testNoFailureIfMVRepairOnButConfigIsOff() + { + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.INCREMENTAL, true); + DatabaseDescriptor.getAutoRepairConfig().setMaterializedViewRepairEnabled(RepairType.INCREMENTAL, false); + DatabaseDescriptor.setCDCOnRepairEnabled(false); + DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); + AutoRepair instance = new AutoRepair(); + instance.setup(); + } + @Test(expected = ConfigurationException.class) public void testSetupFailsWhenIREnabledWithMVReplay() { DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(RepairType.INCREMENTAL, true); + DatabaseDescriptor.getAutoRepairConfig().setMaterializedViewRepairEnabled(RepairType.INCREMENTAL, true); + DatabaseDescriptor.setCDCOnRepairEnabled(false); DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); AutoRepair instance = new AutoRepair(); instance.setup(); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index 1081a0b9d5cd..48536efd82bd 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -22,7 +22,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index 432c09556d88..80ce2cfca8b9 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -126,6 +126,7 @@ public void testSetAutoRepairEnabledThrowsWithSchedulerDisabled() public void testSetAutoRepairEnabledThrowsForIRWithMVReplay() { autoRepairService.config = new AutoRepairConfig(true); + autoRepairService.config.setMaterializedViewRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); @@ -146,6 +147,7 @@ public void testSetAutoRepairEnabledDoesNotThrowForIRWithMVReplayDisabled() public void testSetAutoRepairEnabledThrowsForIRWithCDCReplay() { autoRepairService.config = new AutoRepairConfig(true); + DatabaseDescriptor.setCDCEnabled(true); DatabaseDescriptor.setCDCOnRepairEnabled(true); autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); @@ -236,7 +238,7 @@ public static Collection testCases() forEachRepairType(true, AutoRepairService.instance::setPrimaryTokenRangeOnly, config::getRepairPrimaryTokenRangeOnly), forEachRepairType(600, AutoRepairService.instance::setParallelRepairPercentage, config::getParallelRepairPercentage), forEachRepairType(700, AutoRepairService.instance::setParallelRepairCount, config::getParallelRepairCount), - forEachRepairType(true, AutoRepairService.instance::setMVRepairEnabled, config::getMVRepairEnabled), + forEachRepairType(true, AutoRepairService.instance::setMVRepairEnabled, config::getMaterializedViewRepairEnabled), forEachRepairType(ImmutableSet.of(InetAddressAndPort.getLocalHost()), AutoRepairService.instance::setRepairPriorityForHosts, AutoRepairUtils::getPriorityHosts), forEachRepairType(ImmutableSet.of(InetAddressAndPort.getLocalHost()), AutoRepairService.instance::setForceRepairForHosts, SetterTests::isLocalHostForceRepair) ).flatMap(Function.identity()).collect(Collectors.toList()); From 189bc3c6883802c4334a3402997c08fb9d787f00 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Wed, 22 Jan 2025 22:56:16 -0600 Subject: [PATCH 118/199] Make AutoRepair's repair_by_keyspace default to true instead of false Creating repair sessions by table can create an overwhelming amount of repairs, especially with vnodes. If a repair assignment is too big (> 64 tables by default, or > 200GB for full/50GB for incremental) RepairTokenRangeSplitter will already split into multiple repair assignments. Adjusts repair_by_keyspace to default to true. patch by Andy Tolbert; reviewed by Jaydeep Chovatia for CASSANDRA-20232 --- conf/cassandra.yaml | 2 +- doc/modules/cassandra/pages/auto-repair/overview.adoc | 6 +++--- .../cassandra/repair/autorepair/AutoRepairConfig.java | 2 +- .../repair/autorepair/AutoRepairConfigTest.java | 11 +++++++---- .../autorepair/RepairTokenRangeSplitterTest.java | 1 - 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index ee3715cf38d8..928e87f45c1f 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1938,7 +1938,7 @@ drop_compact_storage_enabled: false # global_settings: # # If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired # # individually. -# repair_by_keyspace: false +# repair_by_keyspace: true # # Number of threads to use for each repair job scheduled by the scheduler. Similar to the -j option in nodetool # # repair. # number_of_repair_threads: 1 diff --git a/doc/modules/cassandra/pages/auto-repair/overview.adoc b/doc/modules/cassandra/pages/auto-repair/overview.adoc index 1eb54a595ccd..bd8f10fb0912 100644 --- a/doc/modules/cassandra/pages/auto-repair/overview.adoc +++ b/doc/modules/cassandra/pages/auto-repair/overview.adoc @@ -64,7 +64,7 @@ The following settings can be tailored individually for each repair type. |=== | Name | Default | Description | enabled | false | Enable/Disable full/incremental/preview_repaired auto-repair -| repair_by_keyspace | false | If true, attempts to group tables in the same keyspace into one repair; otherwise, +| repair_by_keyspace | true | If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired individually. | number_of_repair_threads | 1 | Number of threads to use for each repair job scheduled by the scheduler. Similar to the -j option in nodetool repair. @@ -146,7 +146,7 @@ repair scheduler configuration: configuration for repair_type: full enabled: true min_repair_interval: 24h - repair_by_keyspace: false + repair_by_keyspace: true number_of_repair_threads: 1 sstable_upper_threshold: 10000 table_max_repair_time: 6h @@ -166,7 +166,7 @@ configuration for repair_type: full configuration for repair_type: incremental enabled: true min_repair_interval: 1h - repair_by_keyspace: false + repair_by_keyspace: true number_of_repair_threads: 1 sstable_upper_threshold: 10000 table_max_repair_time: 6h diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index af287a14e71e..73bf19d59d74 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -425,7 +425,7 @@ protected static Options getDefaultOptions() Options opts = new Options(); opts.enabled = false; - opts.repair_by_keyspace = false; + opts.repair_by_keyspace = true; opts.number_of_repair_threads = 1; opts.parallel_repair_count = 3; opts.parallel_repair_percentage = 3; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index b95b8dd84b21..3bef45b8bd53 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -130,9 +130,12 @@ public void testSetAutoRepairEnabledNoMVOrCDC() @Test public void testSetRepairByKeyspace() { - config.setRepairByKeyspace(repairType, true); + // Should default to true. + assertTrue(config.getRepairByKeyspace(repairType)); - assertTrue(config.getOptions(repairType).repair_by_keyspace); + config.setRepairByKeyspace(repairType, false); + + assertFalse(config.getOptions(repairType).repair_by_keyspace); } @Test @@ -418,7 +421,7 @@ public void testDefaultOptions() Map defaultOptions = Options.getDefaultOptionsMap(); Options options = defaultOptions.get(repairType); assertFalse(options.enabled); - assertFalse(options.repair_by_keyspace); + assertTrue(options.repair_by_keyspace); assertEquals(Integer.valueOf(1), options.number_of_repair_threads); assertEquals(Integer.valueOf(3), options.parallel_repair_count); assertEquals(Integer.valueOf(3), options.parallel_repair_percentage); @@ -447,7 +450,7 @@ public void testGlobalOptions() { AutoRepairConfig config = new AutoRepairConfig(); assertFalse(config.global_settings.enabled); - assertFalse(config.global_settings.repair_by_keyspace); + assertTrue(config.global_settings.repair_by_keyspace); assertEquals(Integer.valueOf(1), config.global_settings.number_of_repair_threads); assertEquals(Integer.valueOf(3), config.global_settings.parallel_repair_count); assertEquals(Integer.valueOf(3), config.global_settings.parallel_repair_percentage); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index 48536efd82bd..433e0d3ef5e5 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -64,7 +64,6 @@ public static void setUpClass() { CQLTester.setUpClass(); AutoRepairService.setup(); - AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(RepairType.FULL, true); FULL_RANGE = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken()); } From c799a0275964bf28e8d6bd0fda0e34e0176b4897 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 22 Jan 2025 23:36:45 -0600 Subject: [PATCH 119/199] partitions_per_assignment to not use repair_session_max_tree_depth Updates partitions_per_assignment to not be based on repair_session_max_tree_depth which is deprecated. Instead, just use 2^20. Also updates documentation around partitions_per_assignment and cleans up some warnings in RepairTokeRangeSplitter. patch by Andy Tolbert; reviewed by Jaydeep Chovatia for CASSANDRA-20231 --- conf/cassandra.yaml | 3 ++- .../cassandra/pages/auto-repair/overview.adoc | 4 ++-- .../autorepair/RepairTokenRangeSplitter.java | 21 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 928e87f45c1f..9897ddb3ff1c 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1973,7 +1973,8 @@ drop_compact_storage_enabled: false # # Splitter implementation to generate repair assignments. Defaults to RepairTokenRangeSplitter. # class_name: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter # parameters: -# # Target number of partitions to include in a repair assignment to reduce excessive overstreaming. +# # Maximum number of partitions to include in a repair assignment. Used to reduce number of partitions +# # present in merkle tree leaf nodes to avoid overstreaming. # partitions_per_assignment: 1048576 # # Maximum number of tables to include in a repair assignment. This reduces the number of repairs, # # especially in keyspaces with many tables. The splitter avoids batching tables together if they diff --git a/doc/modules/cassandra/pages/auto-repair/overview.adoc b/doc/modules/cassandra/pages/auto-repair/overview.adoc index bd8f10fb0912..698e53c44677 100644 --- a/doc/modules/cassandra/pages/auto-repair/overview.adoc +++ b/doc/modules/cassandra/pages/auto-repair/overview.adoc @@ -90,8 +90,8 @@ schedule in certain data centers. in nodetool repair. Defaults to true. General advice is to keep this true. | token_range_splitter | org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter | Splitter implementation to generate repair assignments. Defaults to RepairTokenRangeSplitter. -| token_range_splitter.partitions_per_assignment | 1048576 | Target number of partitions to include in a repair -assignment to reduce excessive overstreaming. +| token_range_splitter.partitions_per_assignment | 1048576 | Maximum number of partitions to include in a repair +assignment. Used to reduce number of partitions present in merkle tree leaf nodes to avoid overstreaming. | token_range_splitter.max_tables_per_assignment | 64 | Maximum number of tables to include in a repair assignment. This reduces the number of repairs, especially in keyspaces with many tables. The splitter avoids batching tables together if they exceed other configuration parameters like bytes_per_assignment or partitions_per_assignment. diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index 424454cf6304..fcf9b9742eaa 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -44,7 +44,6 @@ import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; import com.clearspring.analytics.stream.cardinality.ICardinality; import org.apache.cassandra.config.DataStorageSpec; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.lifecycle.SSTableIntervalTree; @@ -109,8 +108,9 @@ * repaired. * *
    • - * partitions_per_assignment: The target number of partitions that should be included in a repair - * assignment. This configuration exists to reduce excessive overstreaming. + * partitions_per_assignment: The maximum number of partitions that should be included in a repair + * assignment. This configuration exists to reduce excessive overstreaming by attempting to limit the number + * of partitions present in a merkle tree leaf node. *
      • *
    • * max_tables_per_assignment: The maximum number of tables that can be included in a repair assignment. @@ -135,7 +135,7 @@ * repair schedule. *
        *
      • bytes_per_assignment: 200GiB
        • - *
      • partitions_per_assignment: 2^repair_session_max_tree_depth (2^20 == 1048576 by default)
        • + *
      • partitions_per_assignment: 1048576
        • *
      • max_tables_per_assignment: 64
        • *
      • max_bytes_per_schedule: 100TiB
        • *
      @@ -147,7 +147,7 @@ * more than this much data is written per min_repair_interval. *
        *
      • bytes_per_assignment: 50GiB
        • - *
      • partitions_per_assignment: 2^repair_session_max_tree_depth (2^20 == 1048576 by default)
        • + *
      • partitions_per_assignment: 1048576
        • *
      • max_tables_per_assignment: 64
        • *
      • max_bytes_per_schedule: 100GiB
        • *
      @@ -157,7 +157,7 @@ * with previews targeting at most 200GiB of data and 1048576 partitions. *
        *
      • bytes_per_assignment: 200GiB
        • - *
      • partitions_per_assignment: 2^repair_session_max_tree_depth (2^20 == 1048576 by default)
        • + *
      • partitions_per_assignment: 1048576
        • *
      • max_tables_per_assignment: 64
        • *
      • max_bytes_per_schedule: 100TiB
        • *
      @@ -204,7 +204,7 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter * move the entire repaired set from unrepaired to repaired at steady state, assuming not more the 100GiB of * data is written to a node per min_repair_interval. */ - private static final Map DEFAULTS_BY_REPAIR_TYPE = new EnumMap(AutoRepairConfig.RepairType.class) {{ + private static final Map DEFAULTS_BY_REPAIR_TYPE = new EnumMap<>(AutoRepairConfig.RepairType.class) {{ put(AutoRepairConfig.RepairType.FULL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.FULL) .build()); // Restrict incremental repair to 50GB bytes per assignment to confine the amount of possible autocompaction. @@ -317,7 +317,7 @@ List getRepairAssignmentsForKeyspace(AutoRepairConfig.Rep tableNames.sort((t1, t2) -> { ColumnFamilyStore cfs1 = ColumnFamilyStore.getIfExists(keyspaceName, t1); ColumnFamilyStore cfs2 = ColumnFamilyStore.getIfExists(keyspaceName, t2); - // If for whatever reason the CFS is not retrievable, we can assume its been deleted, so give the + // If for whatever reason the CFS is not retrievable, we can assume it has been deleted, so give the // other cfs precedence. if (cfs1 == null) { @@ -687,8 +687,6 @@ static SizeEstimate getSizesForRangeOfSSTables(AutoRepairConfig.RepairType repai long sstableSize = reader.bytesOnDisk(); totalBytes += sstableSize; - // TODO since reading the index file anyway may be able to get more accurate ratio for partition count, - // still better to use the cardinality estimator then the index since it wont count duplicates. // get the bounds of the sstable for this range using the index file but do not actually read it. List> bounds = BigTableScanner.makeBounds(reader, Collections.singleton(tokenRange)); @@ -954,7 +952,8 @@ static class RepairTypeDefaultsBuilder { private final AutoRepairConfig.RepairType repairType; private DataStorageSpec.LongBytesBound bytesPerAssignment = new DataStorageSpec.LongBytesBound("200GiB"); - private long partitionsPerAssignment = (long) Math.pow(2, DatabaseDescriptor.getRepairSessionMaxTreeDepth()); + // Aims to target at most 1 partitons per leaf assuming a merkle tree of depth 20 (2^20 = 1,048,576) + private long partitionsPerAssignment = 1_048_576; private int maxTablesPerAssignment = 64; private DataStorageSpec.LongBytesBound maxBytesPerSchedule = MAX_BYTES; From 51eee9d976c60b5473bccd1278398cd9ef2c4472 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 23 Jan 2025 15:48:50 -0600 Subject: [PATCH 120/199] Fix inconsistent spacing in javadoc --- .../autorepair/RepairTokenRangeSplitter.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index fcf9b9742eaa..c8456d4cfcb6 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -91,9 +91,9 @@ *
      • * * To manage these issues, the strategy involves estimating the size and number of partitions within a range and - * splitting it accordingly to bound the size of the range splits. This is established by iterating over SSTable + * splitting it accordingly to bound the size of the range splits. This is established by iterating over SSTable * index files to estimate the amount of bytes and partitions involved in the ranges being repaired and by what - * repair type is beinb invoked. + * repair type is being invoked. *

      * While this splitter has a lot of tuning parameters, the expectation is that the established default configuration * shall be sensible for all {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType}'s. The following @@ -101,28 +101,28 @@ *

        *
      • * bytes_per_assigment: The target and maximum amount of bytes that should be included in a repair - * assignment. This is meant to scope the amount of work involved in a repair. For incremental repair, this + * assignment. This is meant to scope the amount of work involved in a repair. For incremental repair, this * involves the total number of bytes in all SSTables containing unrepaired data involving the ranges being - * repaired, including data that doesn't cover the range. This is to account for the amount of anticompaction + * repaired, including data that doesn't cover the range. This is to account for the amount of anticompaction * that is expected. For all other repair types, this involves the amount of data covering the range being * repaired. *
        • *
      • * partitions_per_assignment: The maximum number of partitions that should be included in a repair - * assignment. This configuration exists to reduce excessive overstreaming by attempting to limit the number + * assignment. This configuration exists to reduce excessive overstreaming by attempting to limit the number * of partitions present in a merkle tree leaf node. *
        • *
      • * max_tables_per_assignment: The maximum number of tables that can be included in a repair assignment. * This aims to reduce the number of repairs, especially in cases where a large amount of tables exists for - * a keyspace. Note that the splitter will avoid batching tables together if they exceed the other + * a keyspace. Note that the splitter will avoid batching tables together if they exceed the other * configuration parameters such as bytes_per_assignment and partitions_per_assignment. *
        • *
      • - * max_bytes_per_schedule: The maximum number of bytes to cover an individual schedule. This serves - * as a mechanism for throttling the amount of work that can be done on each repair cycle. One may opt to + * max_bytes_per_schedule: The maximum number of bytes to cover an individual schedule. This serves + * as a mechanism for throttling the amount of work that can be done on each repair cycle. One may opt to * reduce this value if the impact of repairs is causing too many load on the cluster, or increase it if - * writes outpace the amount of data being repaired. Alternatively, one may want to choose tuning down or up + * writes outpace the amount of data being repaired. Alternatively, one may want to choose tuning down or up * the min_repair_interval. *
        • *
      @@ -142,8 +142,8 @@ * *
    • * incremental: Configured in a way that attempts to repair 50GiB of data per repair, and 100GiB per - * schedule. This attempts to throttle the amount of IR and anticompaction done per schedule after turning - * incremental on for the first time. You may want to consider increasing max_bytes_per_schedule + * schedule. This attempts to throttle the amount of IR and anticompaction done per schedule after turning + * incremental on for the first time. You may want to consider increasing max_bytes_per_schedule * more than this much data is written per min_repair_interval. *
        *
      • bytes_per_assignment: 50GiB
        • @@ -199,12 +199,12 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter *
          max_bytes_per_schedule: 1000GiB
        * * It's expected that these defaults should work well for everything except incremental, so we confine - * bytes_per_assignment to 50GiB and max_bytes_per_schedule to 100GiB. This should strike a good balance + * bytes_per_assignment to 50GiB and max_bytes_per_schedule to 100GiB. This should strike a good balance * between the amount of data that will be repaired during an initial migration to incremental repair and should * move the entire repaired set from unrepaired to repaired at steady state, assuming not more the 100GiB of * data is written to a node per min_repair_interval. */ - private static final Map DEFAULTS_BY_REPAIR_TYPE = new EnumMap<>(AutoRepairConfig.RepairType.class) {{ + private static final Map DEFAULTS_BY_REPAIR_TYPE = new EnumMap(AutoRepairConfig.RepairType.class) {{ put(AutoRepairConfig.RepairType.FULL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.FULL) .build()); // Restrict incremental repair to 50GB bytes per assignment to confine the amount of possible autocompaction. @@ -286,7 +286,7 @@ private class BytesBasedRepairAssignmentIterator extends RepairAssignmentIterato protected KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repairPlan) { // short circuit if we've accumulated too many bytes by returning a KeyspaceRepairAssignments with - // no assignments. We do this rather than returning false in hasNext() because we want to signal + // no assignments. We do this rather than returning false in hasNext() because we want to signal // to AutoRepair that a keyspace generated no assignments. if (bytesSoFar >= maxBytesPerSchedule.toBytes()) { @@ -813,7 +813,7 @@ protected static class SizeEstimate public final long sizeInRange; public final long totalSize; /** - * Size to consider in the repair. For incremental repair, we want to consider the total size + * Size to consider in the repair. For incremental repair, we want to consider the total size * of the estimate as we have to factor in anticompacting the entire SSTable. * For full repair, just use the size containing the range. */ @@ -882,7 +882,7 @@ public String getDescription() { } /** - * Estimated bytes involved in the assignment. Typically Derived from {@link SizeEstimate#sizeForRepair}. + * Estimated bytes involved in the assignment. Typically Derived from {@link SizeEstimate#sizeForRepair}. * @return estimated bytes involved in the assignment. */ public long getEstimatedBytes() From c540360c44ac264cf2743bde91736dd5756a9ed8 Mon Sep 17 00:00:00 2001 From: Francisco Guerrero Date: Fri, 24 Jan 2025 16:46:40 -0800 Subject: [PATCH 121/199] AutoRepairServiceMBean should not use custom types for JMX methods Patch by Francisco Guerrero; reviewed by TBD for CASSANDRA-20185 --- .../cassandra/locator/InetAddressAndPort.java | 55 ++++++ .../repair/autorepair/AutoRepairConfig.java | 13 ++ .../FixedSplitTokenRangeSplitter.java | 2 +- .../cassandra/service/AutoRepairService.java | 160 +++++++++++++----- .../service/AutoRepairServiceMBean.java | 40 ++--- .../cassandra/service/StorageService.java | 17 +- .../service/StorageServiceMBean.java | 3 +- .../org/apache/cassandra/tools/NodeProbe.java | 53 +++--- .../tools/nodetool/AutoRepairStatus.java | 6 +- .../tools/nodetool/GetAutoRepairConfig.java | 83 +-------- .../tools/nodetool/SetAutoRepairConfig.java | 63 ++----- .../AutoRepairConfigRepairTypeTest.java | 61 +++++++ .../service/AutoRepairServiceTest.java | 18 +- .../tools/nodetool/AutoRepairStatusTest.java | 8 +- .../nodetool/SetAutoRepairConfigTest.java | 51 +++--- 15 files changed, 359 insertions(+), 274 deletions(-) create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigRepairTypeTest.java diff --git a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java index e6a920b68c2a..3e9fd92da4f0 100644 --- a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java +++ b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java @@ -25,11 +25,19 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.regex.Pattern; import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; import com.google.common.net.HostAndPort; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; @@ -55,6 +63,7 @@ public final class InetAddressAndPort extends InetSocketAddress implements Comparable, Serializable { private static final long serialVersionUID = 0; + private static final Logger logger = LoggerFactory.getLogger(InetAddressAndPort.class); //Store these here to avoid requiring DatabaseDescriptor to be loaded. DatabaseDescriptor will set //these when it loads the config. A lot of unit tests won't end up loading DatabaseDescriptor. @@ -282,6 +291,52 @@ public static void initializeDefaultPort(int port) defaultPort = port; } + public static List stringify(Iterable endpoints) + { + return stringify(endpoints, true); + } + + public static List stringify(Iterable endpoints, boolean withPort) + { + List stringEndpoints = new ArrayList<>(); + for (InetAddressAndPort ep : endpoints) + { + stringEndpoints.add(ep.getHostAddress(withPort)); + } + return stringEndpoints; + } + + /** + * Parses a comma-separated list of hosts to a set of {@link InetAddressAndPort} + * + * @param value the comma-separated list of hosts to parse + * @param failOnError whether to fail when encountering an invalid hostname + * @return the set of parsed {@link InetAddressAndPort} + */ + public static Set parseHosts(String value, boolean failOnError) + { + Set hosts = new HashSet<>(); + for (String host : Splitter.on(',').split(value)) + { + try + { + hosts.add(InetAddressAndPort.getByName(host)); + } + catch (UnknownHostException e) + { + if (failOnError) + { + throw new IllegalArgumentException(e.getMessage()); + } + else + { + logger.warn("Invalid ip address {} from input={}", host, value); + } + } + } + return hosts; + } + static int getDefaultPort() { return defaultPort; diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 73bf19d59d74..07d894c4f1ff 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -23,6 +23,7 @@ import java.util.EnumMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.function.Function; @@ -103,6 +104,18 @@ public static AutoRepairState getAutoRepairState(RepairType repairType) throw new IllegalArgumentException("Invalid repair type: " + repairType); } + + /** + * Case-insensitive parsing of the repair type string into {@link RepairType} + * + * @param repairTypeStr the repair type string + * @return the {@link RepairType} represented by the {@code repairTypeStr} string + * @throws IllegalArgumentException when the repair type string does not match any repair type + */ + public static RepairType parse(String repairTypeStr) + { + return RepairType.valueOf(Objects.requireNonNull(repairTypeStr, "repairTypeStr cannot be null").toUpperCase()); + } } // repair_type_overrides overrides the global_settings for a specific repair type. String used as key instead diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index b6cdd04498cf..d20475984f6f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -142,4 +142,4 @@ public Map getParameters() { return Collections.singletonMap(NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubranges)); } -} \ No newline at end of file +} diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 9e3b56d18a27..af850f6c2dbf 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -18,6 +18,7 @@ package org.apache.cassandra.service; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; @@ -25,6 +26,7 @@ import org.apache.cassandra.repair.autorepair.AutoRepairUtils; import org.apache.cassandra.utils.MBeanWrapper; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -32,6 +34,7 @@ import java.util.UUID; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; public class AutoRepairService implements AutoRepairServiceMBean { @@ -57,6 +60,11 @@ public static void setup() MBeanWrapper.instance.registerMBean(instance, MBEAN_NAME); } + public void checkCanRun(String repairType) + { + checkCanRun(RepairType.parse(repairType)); + } + public void checkCanRun(RepairType repairType) { if (!config.isAutoRepairSchedulingEnabled()) @@ -72,47 +80,71 @@ public void checkCanRun(RepairType repairType) throw new ConfigurationException("Cannot run incremental repair while CDC replay is enabled. Set cdc_on_repair_enabled to false."); } - @Override public AutoRepairConfig getAutoRepairConfig() { return config; } @Override - public void setAutoRepairEnabled(RepairType repairType, boolean enabled) + public boolean isAutoRepairDisabled() { - checkCanRun(repairType); - config.setAutoRepairEnabled(repairType, enabled); + return config == null || !config.isAutoRepairSchedulingEnabled(); + } + + @Override + public String autoRepairConfiguration() + { + StringBuilder sb = new StringBuilder(); + sb.append("repair scheduler configuration:"); + appendConfig(sb, "repair_check_interval", config.getRepairCheckInterval()); + appendConfig(sb, "repair_max_retries", config.getRepairMaxRetries()); + appendConfig(sb, "repair_retry_backoff", config.getRepairRetryBackoff()); + appendConfig(sb, "repair_task_min_duration", config.getRepairTaskMinDuration()); + appendConfig(sb, "history_clear_delete_hosts_buffer_interval", config.getAutoRepairHistoryClearDeleteHostsBufferInterval()); + for (RepairType repairType : RepairType.values()) + { + sb.append(formatRepairTypeConfig(repairType, config)); + } + return sb.toString(); } @Override - public void setRepairThreads(RepairType repairType, int repairThreads) + public void setAutoRepairEnabled(String repairType, boolean enabled) { - config.setRepairThreads(repairType, repairThreads); + checkCanRun(repairType); + config.setAutoRepairEnabled(RepairType.parse(repairType), enabled); } @Override - public void setRepairPriorityForHosts(RepairType repairType, Set hosts) + public void setRepairThreads(String repairType, int repairThreads) { - AutoRepairUtils.addPriorityHosts(repairType, hosts); + config.setRepairThreads(RepairType.parse(repairType), repairThreads); } @Override - public Set getRepairHostPriority(RepairType repairType) + public void setRepairPriorityForHosts(String repairType, String commaSeparatedHostSet) { - return AutoRepairUtils.getPriorityHosts(repairType); + Set hosts = InetAddressAndPort.parseHosts(commaSeparatedHostSet, false); + if (!hosts.isEmpty()) + { + AutoRepairUtils.addPriorityHosts(RepairType.parse(repairType), hosts); + } } @Override - public void setForceRepairForHosts(RepairType repairType, Set hosts) + public void setForceRepairForHosts(String repairType, String commaSeparatedHostSet) { - AutoRepairUtils.setForceRepair(repairType, hosts); + Set hosts = InetAddressAndPort.parseHosts(commaSeparatedHostSet, false); + if (!hosts.isEmpty()) + { + AutoRepairUtils.setForceRepair(RepairType.parse(repairType), hosts); + } } @Override - public void setRepairMinInterval(RepairType repairType, String minRepairInterval) + public void setRepairMinInterval(String repairType, String minRepairInterval) { - config.setRepairMinInterval(repairType, minRepairInterval); + config.setRepairMinInterval(RepairType.parse(repairType), minRepairInterval); } @Override @@ -121,6 +153,7 @@ public void startScheduler() config.startScheduler(); } + @Override public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration) { config.setAutoRepairHistoryClearDeleteHostsBufferInterval(duration); @@ -145,61 +178,63 @@ public void setAutoRepairMinRepairTaskDuration(String duration) } @Override - public void setRepairSSTableCountHigherThreshold(RepairType repairType, int sstableHigherThreshold) + public void setRepairSSTableCountHigherThreshold(String repairType, int sstableHigherThreshold) { - config.setRepairSSTableCountHigherThreshold(repairType, sstableHigherThreshold); + config.setRepairSSTableCountHigherThreshold(RepairType.parse(repairType), sstableHigherThreshold); } @Override - public void setAutoRepairTableMaxRepairTime(RepairType repairType, String autoRepairTableMaxRepairTime) + public void setAutoRepairTableMaxRepairTime(String repairType, String autoRepairTableMaxRepairTime) { - config.setAutoRepairTableMaxRepairTime(repairType, autoRepairTableMaxRepairTime); + config.setAutoRepairTableMaxRepairTime(RepairType.parse(repairType), autoRepairTableMaxRepairTime); } @Override - public void setIgnoreDCs(RepairType repairType, Set ignoreDCs) + public void setIgnoreDCs(String repairType, Set ignoreDCs) { - config.setIgnoreDCs(repairType, ignoreDCs); + config.setIgnoreDCs(RepairType.parse(repairType), ignoreDCs); } @Override - public void setPrimaryTokenRangeOnly(RepairType repairType, boolean primaryTokenRangeOnly) + public void setPrimaryTokenRangeOnly(String repairType, boolean primaryTokenRangeOnly) { - config.setRepairPrimaryTokenRangeOnly(repairType, primaryTokenRangeOnly); + config.setRepairPrimaryTokenRangeOnly(RepairType.parse(repairType), primaryTokenRangeOnly); } @Override - public void setParallelRepairPercentage(RepairType repairType, int percentage) + public void setParallelRepairPercentage(String repairType, int percentage) { - config.setParallelRepairPercentage(repairType, percentage); + config.setParallelRepairPercentage(RepairType.parse(repairType), percentage); } @Override - public void setParallelRepairCount(RepairType repairType, int count) + public void setParallelRepairCount(String repairType, int count) { - config.setParallelRepairCount(repairType, count); + config.setParallelRepairCount(RepairType.parse(repairType), count); } - public void setMVRepairEnabled(RepairType repairType, boolean enabled) + @Override + public void setMVRepairEnabled(String repairType, boolean enabled) { - config.setMaterializedViewRepairEnabled(repairType, enabled); + config.setMaterializedViewRepairEnabled(RepairType.parse(repairType), enabled); } - public void setRepairSessionTimeout(RepairType repairType, String timeout) + @Override + public void setRepairSessionTimeout(String repairType, String timeout) { - config.setRepairSessionTimeout(repairType, timeout); + config.setRepairSessionTimeout(RepairType.parse(repairType), timeout); } @Override - public Set getOnGoingRepairHostIds(RepairType rType) + public Set getOnGoingRepairHostIds(String repairType) { - Set hostIds = new HashSet<>(); - List histories = AutoRepairUtils.getAutoRepairHistory(rType); + List histories = AutoRepairUtils.getAutoRepairHistory(RepairType.parse(repairType)); if (histories == null) { - return hostIds; + return Collections.emptySet(); } - AutoRepairUtils.CurrentRepairStatus currentRepairStatus = new AutoRepairUtils.CurrentRepairStatus(histories, AutoRepairUtils.getPriorityHostIds(rType)); + Set hostIds = new HashSet<>(); + AutoRepairUtils.CurrentRepairStatus currentRepairStatus = new AutoRepairUtils.CurrentRepairStatus(histories, AutoRepairUtils.getPriorityHostIds(RepairType.parse(repairType))); for (UUID id : currentRepairStatus.hostIdsWithOnGoingRepair) { hostIds.add(id.toString()); @@ -208,18 +243,61 @@ public Set getOnGoingRepairHostIds(RepairType rType) { hostIds.add(id.toString()); } - return hostIds; + return Collections.unmodifiableSet(hostIds); } @Override - public Map getAutoRepairTokenRangeSplitterParameters(RepairType repairType) + public void setAutoRepairTokenRangeSplitterParameter(String repairType, String key, String value) { - return config.getTokenRangeSplitterInstance(repairType).getParameters(); + config.getTokenRangeSplitterInstance(RepairType.parse(repairType)).setParameter(key, value); } - @Override - public void setAutoRepairTokenRangeSplitterParameter(RepairType repairType, String key, String value) + private String formatRepairTypeConfig(RepairType repairType, AutoRepairConfig config) + { + StringBuilder sb = new StringBuilder(); + sb.append("\nconfiguration for repair_type: ").append(repairType.getConfigName()); + sb.append("\n\tenabled: ").append(config.isAutoRepairEnabled(repairType)); + // Only show configuration if enabled + if (config.isAutoRepairEnabled(repairType)) + { + Set priorityHosts = AutoRepairUtils.getPriorityHosts(repairType); + if (!priorityHosts.isEmpty()) + { + appendConfig(sb, "priority_hosts", Joiner.on(',').skipNulls().join(priorityHosts)); + } + + appendConfig(sb , "min_repair_interval", config.getRepairMinInterval(repairType)); + appendConfig(sb , "repair_by_keyspace", config.getRepairByKeyspace(repairType)); + appendConfig(sb , "number_of_repair_threads", config.getRepairThreads(repairType)); + appendConfig(sb , "sstable_upper_threshold", config.getRepairSSTableCountHigherThreshold(repairType)); + appendConfig(sb , "table_max_repair_time", config.getAutoRepairTableMaxRepairTime(repairType)); + appendConfig(sb , "ignore_dcs", config.getIgnoreDCs(repairType)); + appendConfig(sb , "repair_primary_token_range_only", config.getRepairPrimaryTokenRangeOnly(repairType)); + appendConfig(sb , "parallel_repair_count", config.getParallelRepairCount(repairType)); + appendConfig(sb , "parallel_repair_percentage", config.getParallelRepairPercentage(repairType)); + appendConfig(sb , "materialized_view_repair_enabled", config.getMaterializedViewRepairEnabled(repairType)); + appendConfig(sb , "initial_scheduler_delay", config.getInitialSchedulerDelay(repairType)); + appendConfig(sb , "repair_session_timeout", config.getRepairSessionTimeout(repairType)); + appendConfig(sb , "force_repair_new_node", config.getForceRepairNewNode(repairType)); + + final ParameterizedClass splitterClass = config.getTokenRangeSplitter(repairType); + final String splitterClassName = splitterClass.class_name != null ? splitterClass.class_name : AutoRepairConfig.DEFAULT_SPLITTER.getName(); + appendConfig(sb, "token_range_splitter", splitterClassName); + Map tokenRangeSplitterParameters = config.getTokenRangeSplitterInstance(repairType).getParameters(); + if (!tokenRangeSplitterParameters.isEmpty()) + { + for (Map.Entry param : tokenRangeSplitterParameters.entrySet()) + { + appendConfig(sb, String.format("token_range_splitter.%s", param.getKey()), param.getValue()); + } + } + } + + return sb.toString(); + } + + private void appendConfig(StringBuilder sb, String config, T value) { - config.getTokenRangeSplitterInstance(repairType).setParameter(key, value); + sb.append(String.format("%s%s: %s", "\n\t", config, value)); } } diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index 14fe9feb7958..bad7667c6acc 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -17,11 +17,7 @@ */ package org.apache.cassandra.service; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; -import java.util.Map; import java.util.Set; public interface AutoRepairServiceMBean @@ -29,17 +25,15 @@ public interface AutoRepairServiceMBean /** * Enable or disable auto-repair for a given repair type */ - public void setAutoRepairEnabled(RepairType repairType, boolean enabled); + public void setAutoRepairEnabled(String repairType, boolean enabled); - public void setRepairThreads(RepairType repairType, int repairThreads); + public void setRepairThreads(String repairType, int repairThreads); - public void setRepairPriorityForHosts(RepairType repairType, Set host); + public void setRepairPriorityForHosts(String repairType, String commaSeparatedHostSet); - public void setForceRepairForHosts(RepairType repairType, Set host); + public void setForceRepairForHosts(String repairType, String commaSeparatedHostSet); - public Set getRepairHostPriority(RepairType repairType); - - public void setRepairMinInterval(RepairType repairType, String minRepairInterval); + public void setRepairMinInterval(String repairType, String minRepairInterval); void startScheduler(); @@ -51,27 +45,27 @@ public interface AutoRepairServiceMBean public void setAutoRepairMinRepairTaskDuration(String duration); - public void setRepairSSTableCountHigherThreshold(RepairType repairType, int ssTableHigherThreshold); + public void setRepairSSTableCountHigherThreshold(String repairType, int ssTableHigherThreshold); - public void setAutoRepairTableMaxRepairTime(RepairType repairType, String autoRepairTableMaxRepairTime); + public void setAutoRepairTableMaxRepairTime(String repairType, String autoRepairTableMaxRepairTime); - public void setIgnoreDCs(RepairType repairType, Set ignorDCs); + public void setIgnoreDCs(String repairType, Set ignorDCs); - public void setPrimaryTokenRangeOnly(RepairType repairType, boolean primaryTokenRangeOnly); + public void setPrimaryTokenRangeOnly(String repairType, boolean primaryTokenRangeOnly); - public void setParallelRepairPercentage(RepairType repairType, int percentage); + public void setParallelRepairPercentage(String repairType, int percentage); - public void setParallelRepairCount(RepairType repairType, int count); + public void setParallelRepairCount(String repairType, int count); - public void setMVRepairEnabled(RepairType repairType, boolean enabled); + public void setMVRepairEnabled(String repairType, boolean enabled); - public AutoRepairConfig getAutoRepairConfig(); + public boolean isAutoRepairDisabled(); - public void setRepairSessionTimeout(RepairType repairType, String timeout); + public String autoRepairConfiguration(); - public Set getOnGoingRepairHostIds(RepairType rType); + public void setRepairSessionTimeout(String repairType, String timeout); - public Map getAutoRepairTokenRangeSplitterParameters(RepairType repairType); + public Set getOnGoingRepairHostIds(String repairType); - public void setAutoRepairTokenRangeSplitterParameter(RepairType repairType, String key, String value); + public void setAutoRepairTokenRangeSplitterParameter(String repairType, String key, String value); } diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 37e7a547e906..491d6b0f27a5 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -243,6 +243,7 @@ import static org.apache.cassandra.index.SecondaryIndexManager.isIndexColumnFamily; import static org.apache.cassandra.net.NoPayload.noPayload; import static org.apache.cassandra.net.Verb.REPLICATION_DONE_REQ; +import static org.apache.cassandra.locator.InetAddressAndPort.stringify; import static org.apache.cassandra.service.ActiveRepairService.ParentRepairStatus; import static org.apache.cassandra.service.ActiveRepairService.repairCommandExecutor; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; @@ -3906,16 +3907,6 @@ public String getSavedCachesLocation() return FileUtils.getCanonicalPath(DatabaseDescriptor.getSavedCachesLocation()); } - private List stringify(Iterable endpoints, boolean withPort) - { - List stringEndpoints = new ArrayList<>(); - for (InetAddressAndPort ep : endpoints) - { - stringEndpoints.add(ep.getHostAddress(withPort)); - } - return stringEndpoints; - } - public int getCurrentGenerationNumber() { return Gossiper.instance.getCurrentGenerationNumber(FBUtilities.getBroadcastAddressAndPort()); @@ -7072,19 +7063,21 @@ public void setEnforceNativeDeadlineForHints(boolean value) DatabaseDescriptor.setEnforceNativeDeadlineForHints(value); } + @Override public List getTablesForKeyspace(String keyspace) { return Keyspace.open(keyspace).getColumnFamilyStores().stream().map(cfs -> cfs.name).collect(Collectors.toList()); } - public List mutateSSTableRepairedState(boolean repaired, boolean preview, String keyspace, List tableNames) throws InvalidRequestException + @Override + public List mutateSSTableRepairedState(boolean repaired, boolean preview, String keyspace, List tableNames) { Map tables = Keyspace.open(keyspace).getColumnFamilyStores() .stream().collect(Collectors.toMap(c -> c.name, c -> c)); for (String tableName : tableNames) { if (!tables.containsKey(tableName)) - throw new InvalidRequestException("Table " + tableName + " does not exist in keyspace " + keyspace); + throw new RuntimeException("Table " + tableName + " does not exist in keyspace " + keyspace); } // only select SSTables that are unrepaired when repaired is true and vice versa diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 475ed2281e10..e26febc3d0c6 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -36,7 +36,6 @@ import org.apache.cassandra.db.ColumnFamilyStoreMBean; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.utils.BreaksJMX; -import org.apache.cassandra.exceptions.InvalidRequestException; public interface StorageServiceMBean extends NotificationEmitter { @@ -1075,5 +1074,5 @@ public void enableAuditLog(String loggerName, String includedKeyspaces, String e public List getTablesForKeyspace(String keyspace); /** Mutates the repaired state of all SSTables for the given SSTables */ - public List mutateSSTableRepairedState(boolean repaired, boolean preview, String keyspace, List tables) throws InvalidRequestException; + public List mutateSSTableRepairedState(boolean repaired, boolean preview, String keyspace, List tables); } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 296bb862f1d3..ee20387fa7f4 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -75,7 +75,6 @@ import org.apache.cassandra.db.ColumnFamilyStoreMBean; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.CompactionManagerMBean; -import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.fql.FullQueryLoggerOptions; import org.apache.cassandra.fql.FullQueryLoggerOptionsCompositeData; import org.apache.cassandra.gms.FailureDetector; @@ -86,14 +85,12 @@ import org.apache.cassandra.hints.HintsServiceMBean; import org.apache.cassandra.locator.DynamicEndpointSnitchMBean; import org.apache.cassandra.locator.EndpointSnitchInfoMBean; -import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.metrics.StorageMetrics; import org.apache.cassandra.metrics.TableMetrics; import org.apache.cassandra.metrics.ThreadPoolMetrics; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.MessagingServiceMBean; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.service.ActiveRepairServiceMBean; import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.service.AutoRepairServiceMBean; @@ -2147,46 +2144,42 @@ public int getDefaultKeyspaceReplicationFactor() return ssProxy.getDefaultKeyspaceReplicationFactor(); } - public AutoRepairConfig getAutoRepairConfig() + public boolean isAutoRepairDisabled() { - return autoRepairProxy.getAutoRepairConfig(); + return autoRepairProxy.isAutoRepairDisabled(); } - public Map getAutoRepairTokenRangeSplitterParameters(AutoRepairConfig.RepairType repairType) + public String autoRepairConfiguration() { - return autoRepairProxy.getAutoRepairTokenRangeSplitterParameters(repairType); + return autoRepairProxy.autoRepairConfiguration(); } - public void setAutoRepairTokenRangeSplitterParameter(AutoRepairConfig.RepairType repairType, String key, String value) + public void setAutoRepairTokenRangeSplitterParameter(String repairType, String key, String value) { autoRepairProxy.setAutoRepairTokenRangeSplitterParameter(repairType, key, value); } - public void setAutoRepairEnabled(AutoRepairConfig.RepairType repairType, boolean enabled) + public void setAutoRepairEnabled(String repairType, boolean enabled) { autoRepairProxy.setAutoRepairEnabled(repairType, enabled); } - public void setRepairThreads(AutoRepairConfig.RepairType repairType, int repairThreads) + public void setRepairThreads(String repairType, int repairThreads) { autoRepairProxy.setRepairThreads(repairType, repairThreads); } - public void setRepairPriorityForHosts(AutoRepairConfig.RepairType repairType, Set hosts) + public void setRepairPriorityForHosts(String repairType, String commaSeparatedHostSet) { - autoRepairProxy.setRepairPriorityForHosts(repairType, hosts); + autoRepairProxy.setRepairPriorityForHosts(repairType, commaSeparatedHostSet); } - public Set getRepairPriorityForHosts(AutoRepairConfig.RepairType repairType) + public void setForceRepairForHosts(String repairType, String commaSeparatedHostSet) { - return autoRepairProxy.getRepairHostPriority(repairType); + autoRepairProxy.setForceRepairForHosts(repairType, commaSeparatedHostSet); } - public void setForceRepairForHosts(AutoRepairConfig.RepairType repairType, Set hosts){ - autoRepairProxy.setForceRepairForHosts(repairType, hosts); - } - - public void setRepairMinInterval(AutoRepairConfig.RepairType repairType, String minRepairInterval) + public void setRepairMinInterval(String repairType, String minRepairInterval) { autoRepairProxy.setRepairMinInterval(repairType, minRepairInterval); } @@ -2216,42 +2209,42 @@ public void setAutoRepairMinRepairTaskDuration(String duration) autoRepairProxy.setAutoRepairMinRepairTaskDuration(duration); } - public void setRepairSSTableCountHigherThreshold(AutoRepairConfig.RepairType repairType, int ssTableHigherThreshold) + public void setRepairSSTableCountHigherThreshold(String repairType, int ssTableHigherThreshold) { autoRepairProxy.setRepairSSTableCountHigherThreshold(repairType, ssTableHigherThreshold); } - public void setAutoRepairTableMaxRepairTime(AutoRepairConfig.RepairType repairType, String autoRepairTableMaxRepairTime) + public void setAutoRepairTableMaxRepairTime(String repairType, String autoRepairTableMaxRepairTime) { autoRepairProxy.setAutoRepairTableMaxRepairTime(repairType, autoRepairTableMaxRepairTime); } - public void setAutoRepairIgnoreDCs(AutoRepairConfig.RepairType repairType, Set ignoreDCs) + public void setAutoRepairIgnoreDCs(String repairType, Set ignoreDCs) { autoRepairProxy.setIgnoreDCs(repairType, ignoreDCs); } - public void setParallelRepairPercentage(AutoRepairConfig.RepairType repairType, int percentage) + public void setParallelRepairPercentage(String repairType, int percentage) { autoRepairProxy.setParallelRepairPercentage(repairType, percentage); } - public void setParallelRepairCount(AutoRepairConfig.RepairType repairType, int count) + public void setParallelRepairCount(String repairType, int count) { autoRepairProxy.setParallelRepairCount(repairType, count); } - public void setPrimaryTokenRangeOnly(AutoRepairConfig.RepairType repairType, boolean primaryTokenRangeOnly) + public void setPrimaryTokenRangeOnly(String repairType, boolean primaryTokenRangeOnly) { autoRepairProxy.setPrimaryTokenRangeOnly(repairType, primaryTokenRangeOnly); } - public void setMaterializedViewRepairEnabled(AutoRepairConfig.RepairType repairType, boolean enabled) + public void setMaterializedViewRepairEnabled(String repairType, boolean enabled) { autoRepairProxy.setMVRepairEnabled(repairType, enabled); } - public List mutateSSTableRepairedState(boolean repair, boolean preview, String keyspace, List tables) throws InvalidRequestException + public List mutateSSTableRepairedState(boolean repair, boolean preview, String keyspace, List tables) { return ssProxy.mutateSSTableRepairedState(repair, preview, keyspace, tables); } @@ -2261,14 +2254,14 @@ public List getTablesForKeyspace(String keyspace) return ssProxy.getTablesForKeyspace(keyspace); } - public void setRepairSessionTimeout(AutoRepairConfig.RepairType repairType, String timeout) + public void setRepairSessionTimeout(String repairType, String timeout) { autoRepairProxy.setRepairSessionTimeout(repairType, timeout); } - public Set getOnGoingRepairHostIds(AutoRepairConfig.RepairType type) + public Set getOnGoingRepairHostIds(String repairType) { - return autoRepairProxy.getOnGoingRepairHostIds(type); + return autoRepairProxy.getOnGoingRepairHostIds(repairType); } } diff --git a/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java index 7b96102c6698..22f9afc43fe1 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java +++ b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java @@ -25,7 +25,6 @@ import io.airlift.airline.Command; import io.airlift.airline.Option; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.NodeTool; import org.apache.cassandra.tools.nodetool.formatter.TableBuilder; @@ -37,7 +36,7 @@ public class AutoRepairStatus extends NodeTool.NodeToolCmd { @VisibleForTesting @Option(title = "repair type", name = { "-t", "--repair-type" }, description = "Repair type") - protected AutoRepairConfig.RepairType repairType; + protected String repairType; @Override public void execute(NodeProbe probe) @@ -45,8 +44,7 @@ public void execute(NodeProbe probe) checkArgument(repairType != null, "--repair-type is required."); PrintStream out = probe.output().out; - AutoRepairConfig config = probe.getAutoRepairConfig(); - if (config == null || !config.isAutoRepairSchedulingEnabled()) + if (probe.isAutoRepairDisabled()) { out.println("Auto-repair is not enabled"); return; diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java index de234893cb49..961fb71fb4e4 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java @@ -17,20 +17,14 @@ */ package org.apache.cassandra.tools.nodetool; +import java.io.PrintStream; + import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; + import io.airlift.airline.Command; -import org.apache.cassandra.config.ParameterizedClass; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.NodeTool.NodeToolCmd; -import java.io.PrintStream; -import java.util.Map; -import java.util.Set; - @Command(name = "getautorepairconfig", description = "Print autorepair configurations") public class GetAutoRepairConfig extends NodeToolCmd { @@ -40,74 +34,9 @@ public class GetAutoRepairConfig extends NodeToolCmd @Override public void execute(NodeProbe probe) { - AutoRepairConfig config = probe.getAutoRepairConfig(); - if (config == null || !config.isAutoRepairSchedulingEnabled()) - { + if (probe.isAutoRepairDisabled()) out.println("Auto-repair is not enabled"); - return; - } - - StringBuilder sb = new StringBuilder(); - sb.append("repair scheduler configuration:"); - appendConfig(sb, "repair_check_interval", config.getRepairCheckInterval()); - appendConfig(sb, "repair_max_retries", config.getRepairMaxRetries()); - appendConfig(sb, "repair_retry_backoff", config.getRepairRetryBackoff()); - appendConfig(sb, "repair_task_min_duration", config.getRepairTaskMinDuration()); - appendConfig(sb, "history_clear_delete_hosts_buffer_interval", config.getAutoRepairHistoryClearDeleteHostsBufferInterval()); - for (RepairType repairType : RepairType.values()) - { - sb.append(formatRepairTypeConfig(probe, repairType, config)); - } - - out.println(sb); - } - - private String formatRepairTypeConfig(NodeProbe probe, RepairType repairType, AutoRepairConfig config) - { - StringBuilder sb = new StringBuilder(); - sb.append("\nconfiguration for repair_type: ").append(repairType.getConfigName()); - sb.append("\n\tenabled: ").append(config.isAutoRepairEnabled(repairType)); - // Only show configuration if enabled - if (config.isAutoRepairEnabled(repairType)) - { - Set priorityHosts = probe.getRepairPriorityForHosts(repairType); - if (!priorityHosts.isEmpty()) - { - appendConfig(sb, "priority_hosts", Joiner.on(',').skipNulls().join(priorityHosts)); - } - - appendConfig(sb , "min_repair_interval", config.getRepairMinInterval(repairType)); - appendConfig(sb , "repair_by_keyspace", config.getRepairByKeyspace(repairType)); - appendConfig(sb , "number_of_repair_threads", config.getRepairThreads(repairType)); - appendConfig(sb , "sstable_upper_threshold", config.getRepairSSTableCountHigherThreshold(repairType)); - appendConfig(sb , "table_max_repair_time", config.getAutoRepairTableMaxRepairTime(repairType)); - appendConfig(sb , "ignore_dcs", config.getIgnoreDCs(repairType)); - appendConfig(sb , "repair_primary_token_range_only", config.getRepairPrimaryTokenRangeOnly(repairType)); - appendConfig(sb , "parallel_repair_count", config.getParallelRepairCount(repairType)); - appendConfig(sb , "parallel_repair_percentage", config.getParallelRepairPercentage(repairType)); - appendConfig(sb , "materialized_view_repair_enabled", config.getMaterializedViewRepairEnabled(repairType)); - appendConfig(sb , "initial_scheduler_delay", config.getInitialSchedulerDelay(repairType)); - appendConfig(sb , "repair_session_timeout", config.getRepairSessionTimeout(repairType)); - appendConfig(sb , "force_repair_new_node", config.getForceRepairNewNode(repairType)); - - final ParameterizedClass splitterClass = config.getTokenRangeSplitter(repairType); - final String splitterClassName = splitterClass.class_name != null ? splitterClass.class_name : AutoRepairConfig.DEFAULT_SPLITTER.getName(); - appendConfig(sb, "token_range_splitter", splitterClassName); - Map tokenRangeSplitterParameters = probe.getAutoRepairTokenRangeSplitterParameters(repairType); - if (!tokenRangeSplitterParameters.isEmpty()) - { - for (Map.Entry param : tokenRangeSplitterParameters.entrySet()) - { - appendConfig(sb, String.format("token_range_splitter.%s", param.getKey()), param.getValue()); - } - } - } - - return sb.toString(); - } - - private void appendConfig(StringBuilder sb, String config, T value) - { - sb.append(String.format("%s%s: %s", "\n\t", config, value)); + else + out.println(probe.autoRepairConfiguration()); } } diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index b71ae85195e1..d55685e72426 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -23,16 +23,14 @@ import io.airlift.airline.Arguments; import io.airlift.airline.Command; import io.airlift.airline.Option; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.NodeTool.NodeToolCmd; import java.io.PrintStream; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; @@ -67,7 +65,7 @@ public void execute(NodeProbe probe) String paramType = args.get(0); String paramVal = args.get(1); - if (!probe.getAutoRepairConfig().isAutoRepairSchedulingEnabled() && !paramType.equalsIgnoreCase("start_scheduler")) + if (probe.isAutoRepairDisabled() && !paramType.equalsIgnoreCase("start_scheduler")) { out.println("Auto-repair is not enabled"); return; @@ -100,47 +98,40 @@ public void execute(NodeProbe probe) } // options below require --repair-type option - checkArgument(repairTypeStr != null, "--repair-type is required for this parameter."); - final RepairType repairType = RepairType.valueOf(repairTypeStr.toUpperCase()); + Objects.requireNonNull(repairTypeStr, "--repair-type is required for this parameter."); if(paramType.startsWith(TOKEN_RANGE_SPLITTER_PROPERTY_PREFIX)) { final String key = paramType.replace(TOKEN_RANGE_SPLITTER_PROPERTY_PREFIX, ""); - probe.setAutoRepairTokenRangeSplitterParameter(repairType, key, paramVal); + probe.setAutoRepairTokenRangeSplitterParameter(repairTypeStr, key, paramVal); return; } - Set hosts; switch (paramType) { case "enabled": - probe.setAutoRepairEnabled(repairType, Boolean.parseBoolean(paramVal)); + probe.setAutoRepairEnabled(repairTypeStr, Boolean.parseBoolean(paramVal)); break; case "number_of_repair_threads": - probe.setRepairThreads(repairType, Integer.parseInt(paramVal)); + probe.setRepairThreads(repairTypeStr, Integer.parseInt(paramVal)); break; case "min_repair_interval": - probe.setRepairMinInterval(repairType, paramVal); + probe.setRepairMinInterval(repairTypeStr, paramVal); break; case "sstable_upper_threshold": - probe.setRepairSSTableCountHigherThreshold(repairType, Integer.parseInt(paramVal)); + probe.setRepairSSTableCountHigherThreshold(repairTypeStr, Integer.parseInt(paramVal)); break; case "table_max_repair_time": - probe.setAutoRepairTableMaxRepairTime(repairType, paramVal); + probe.setAutoRepairTableMaxRepairTime(repairTypeStr, paramVal); break; case "priority_hosts": - hosts = retrieveHosts(paramVal); - if (!hosts.isEmpty()) + if (paramVal!= null && !paramVal.isEmpty()) { - probe.setRepairPriorityForHosts(repairType, hosts); + probe.setRepairPriorityForHosts(repairTypeStr, paramVal); } break; case "forcerepair_hosts": - hosts = retrieveHosts(paramVal); - if (!hosts.isEmpty()) - { - probe.setForceRepairForHosts(repairType, hosts); - } + probe.setForceRepairForHosts(repairTypeStr, paramVal); break; case "ignore_dcs": Set ignoreDCs = new HashSet<>(); @@ -148,43 +139,25 @@ public void execute(NodeProbe probe) { ignoreDCs.add(dc); } - probe.setAutoRepairIgnoreDCs(repairType, ignoreDCs); + probe.setAutoRepairIgnoreDCs(repairTypeStr, ignoreDCs); break; case "repair_primary_token_range_only": - probe.setPrimaryTokenRangeOnly(repairType, Boolean.parseBoolean(paramVal)); + probe.setPrimaryTokenRangeOnly(repairTypeStr, Boolean.parseBoolean(paramVal)); break; case "parallel_repair_count": - probe.setParallelRepairCount(repairType, Integer.parseInt(paramVal)); + probe.setParallelRepairCount(repairTypeStr, Integer.parseInt(paramVal)); break; case "parallel_repair_percentage": - probe.setParallelRepairPercentage(repairType, Integer.parseInt(paramVal)); + probe.setParallelRepairPercentage(repairTypeStr, Integer.parseInt(paramVal)); break; case "materialized_view_repair_enabled": - probe.setMaterializedViewRepairEnabled(repairType, Boolean.parseBoolean(paramVal)); + probe.setMaterializedViewRepairEnabled(repairTypeStr, Boolean.parseBoolean(paramVal)); break; case "repair_session_timeout": - probe.setRepairSessionTimeout(repairType, paramVal); + probe.setRepairSessionTimeout(repairTypeStr, paramVal); break; default: throw new IllegalArgumentException("Unknown parameter: " + paramType); } } - - private Set retrieveHosts(String paramVal) - { - Set hosts = new HashSet<>(); - for (String host : Splitter.on(',').split(paramVal)) - { - try - { - hosts.add(InetAddressAndPort.getByName(host)); - } - catch (UnknownHostException e) - { - out.println("invalid ip address: " + host); - } - } - - return hosts; - } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigRepairTypeTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigRepairTypeTest.java new file mode 100644 index 000000000000..be14b8b912ea --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigRepairTypeTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + + +import org.junit.Assert; +import org.junit.Test; + +/** + * Unit tests for {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType} + */ +public class AutoRepairConfigRepairTypeTest +{ + @Test + public void testRepairTypeParsing() + { + Assert.assertEquals(AutoRepairConfig.RepairType.FULL, AutoRepairConfig.RepairType.parse("FULL")); + Assert.assertEquals(AutoRepairConfig.RepairType.FULL, AutoRepairConfig.RepairType.parse("FuLl")); + Assert.assertEquals(AutoRepairConfig.RepairType.FULL, AutoRepairConfig.RepairType.parse("full")); + Assert.assertEquals(AutoRepairConfig.RepairType.INCREMENTAL, AutoRepairConfig.RepairType.parse("INCREMENTAL")); + Assert.assertEquals(AutoRepairConfig.RepairType.INCREMENTAL, AutoRepairConfig.RepairType.parse("incremental")); + Assert.assertEquals(AutoRepairConfig.RepairType.INCREMENTAL, AutoRepairConfig.RepairType.parse("inCRemenTal")); + Assert.assertEquals(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, AutoRepairConfig.RepairType.parse("PREVIEW_REPAIRED")); + Assert.assertEquals(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, AutoRepairConfig.RepairType.parse("preview_repaired")); + Assert.assertEquals(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, AutoRepairConfig.RepairType.parse("Preview_Repaired")); + } + + @Test(expected = NullPointerException.class) + public void testNullRepairTypeParsing() + { + AutoRepairConfig.RepairType.parse(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testEmptyRepairTypeParsing() + { + AutoRepairConfig.RepairType.parse(""); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidRepairTypeParsing() + { + AutoRepairConfig.RepairType.parse("very_FULL"); + } +} diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index 80ce2cfca8b9..c9defcf54bf2 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -119,7 +119,7 @@ public void testSetAutoRepairEnabledThrowsWithSchedulerDisabled() { autoRepairService.config = new AutoRepairConfig(false); - autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL.name(), true); } @Test(expected = ConfigurationException.class) @@ -129,7 +129,7 @@ public void testSetAutoRepairEnabledThrowsForIRWithMVReplay() autoRepairService.config.setMaterializedViewRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); - autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL.name(), true); } @Test @@ -140,7 +140,7 @@ public void testSetAutoRepairEnabledDoesNotThrowForIRWithMVReplayDisabled() DatabaseDescriptor.setCDCOnRepairEnabled(false); DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); - autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL.name(), true); } @Test(expected = ConfigurationException.class) @@ -150,7 +150,7 @@ public void testSetAutoRepairEnabledThrowsForIRWithCDCReplay() DatabaseDescriptor.setCDCEnabled(true); DatabaseDescriptor.setCDCOnRepairEnabled(true); - autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL.name(), true); } @Test @@ -160,7 +160,7 @@ public void testSetAutoRepairEnabledDoesNotThrowForIRWithCDCReplayDisabled() DatabaseDescriptor.setCDCEnabled(true); DatabaseDescriptor.setCDCOnRepairEnabled(false); - autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); + autoRepairService.setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL.name(), true); } } @@ -204,7 +204,7 @@ public void testGetOnGoingRepairHostIds() AutoRepairUtils.insertNewRepairHistory(repairType, host1, now, now - 1000000); AutoRepairUtils.insertNewRepairHistory(repairType, host2, now, now - 1000000); - Set hosts = instance.getOnGoingRepairHostIds(repairType); + Set hosts = instance.getOnGoingRepairHostIds(repairType.name()); assertEquals(ImmutableSet.of(host1.toString(), host2.toString()), hosts); } @@ -239,8 +239,8 @@ public static Collection testCases() forEachRepairType(600, AutoRepairService.instance::setParallelRepairPercentage, config::getParallelRepairPercentage), forEachRepairType(700, AutoRepairService.instance::setParallelRepairCount, config::getParallelRepairCount), forEachRepairType(true, AutoRepairService.instance::setMVRepairEnabled, config::getMaterializedViewRepairEnabled), - forEachRepairType(ImmutableSet.of(InetAddressAndPort.getLocalHost()), AutoRepairService.instance::setRepairPriorityForHosts, AutoRepairUtils::getPriorityHosts), - forEachRepairType(ImmutableSet.of(InetAddressAndPort.getLocalHost()), AutoRepairService.instance::setForceRepairForHosts, SetterTests::isLocalHostForceRepair) + forEachRepairType(InetAddressAndPort.getLocalHost().getHostAddressAndPort(), (repairType, commaSeparatedHostSet) -> AutoRepairService.instance.setRepairPriorityForHosts(repairType, (String) commaSeparatedHostSet), AutoRepairUtils::getPriorityHosts), + forEachRepairType(InetAddressAndPort.getLocalHost().getHostAddressAndPort(), (repairType, commaSeparatedHostSet) -> AutoRepairService.instance.setForceRepairForHosts(repairType, (String) commaSeparatedHostSet), SetterTests::isLocalHostForceRepair) ).flatMap(Function.identity()).collect(Collectors.toList()); } @@ -258,7 +258,7 @@ private static Set isLocalHostForceRepair(AutoRepairConfig.R return ImmutableSet.of(); } - private static Stream forEachRepairType(T arg, BiConsumer setter, Function getter) + private static Stream forEachRepairType(T arg, BiConsumer setter, Function getter) { Object[][] testCases = new Object[AutoRepairConfig.RepairType.values().length][4]; for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) diff --git a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java index f1faabc0fc03..4e5f86e7233d 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java @@ -72,7 +72,7 @@ public void setUp() throws Exception setAutoRepairEnabled(true); DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(AutoRepairConfig.RepairType.FULL, true); DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); - when(probe.getAutoRepairConfig()).thenReturn(DatabaseDescriptor.getAutoRepairConfig()); +// when(probe.getAutoRepairConfig()).thenReturn(DatabaseDescriptor.getAutoRepairConfig()); } @Test(expected = IllegalArgumentException.class) @@ -85,7 +85,7 @@ public void testExecuteWithoutRepairType() @Test public void testExecuteWithNoNodes() { - cmd.repairType = repairType; + cmd.repairType = repairType.name(); cmd.execute(probe); assertEquals("Active Repairs\n" + @@ -95,8 +95,8 @@ public void testExecuteWithNoNodes() @Test public void testExecute() { - when(probe.getOnGoingRepairHostIds(repairType)).thenReturn(ImmutableSet.of("host1", "host2", "host3", "host4")); - cmd.repairType = repairType; + when(probe.getOnGoingRepairHostIds(repairType.name())).thenReturn(ImmutableSet.of("host1", "host2", "host3", "host4")); + cmd.repairType = repairType.name(); cmd.execute(probe); diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index 5f3be97cbbac..593cb7c190aa 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -50,14 +50,11 @@ SetAutoRepairConfigTest.RepairTypeAndArgsParamsTests.class }) public class SetAutoRepairConfigTest { - protected static AutoRepairConfig config; - protected static SetAutoRepairConfig cmd; public static void before(NodeProbe probeMock, PrintStream outMock) { - config = new AutoRepairConfig(true); - when(probeMock.getAutoRepairConfig()).thenReturn(config); + when(probeMock.isAutoRepairDisabled()).thenReturn(false); cmd = new SetAutoRepairConfig(); cmd.out = outMock; } @@ -87,7 +84,7 @@ public void testHistoryDeleteHostsClearBufferInSec() verify(probe, times(1)).setAutoRepairHistoryClearDeleteHostsBufferDuration("1s"); // test scenario when auto repair is disabled - when(probe.getAutoRepairConfig()).thenReturn(new AutoRepairConfig(false)); + when(probe.isAutoRepairDisabled()).thenReturn(true); cmd.execute(probe); @@ -182,26 +179,26 @@ public void testNoArgs() @Test public void testRepairSchedulingDisabled() { - when(probe.getAutoRepairConfig()).thenReturn(new AutoRepairConfig(false)); + when(probe.isAutoRepairDisabled()).thenReturn(true); cmd.repairTypeStr = repairType.name(); cmd.args = ImmutableList.of("threads", "1"); cmd.execute(probe); verify(out, times(1)).println("Auto-repair is not enabled"); - verify(probe, times(0)).setRepairThreads(repairType, 1); + verify(probe, times(0)).setRepairThreads(repairType.name(), 1); } @Test public void testRepairTypeDisabled() { - config.setAutoRepairEnabled(repairType, false); +// config.setAutoRepairEnabled(repairType, false); cmd.repairTypeStr = repairType.name(); cmd.args = ImmutableList.of("number_of_repair_threads", "1"); cmd.execute(probe); - verify(probe, times(1)).setRepairThreads(repairType, 1); + verify(probe, times(1)).setRepairThreads(repairType.name(), 1); } @@ -222,7 +219,7 @@ public void testV2FlagMissing() // expected } - verify(probe, times(0)).setRepairThreads(repairType, 0); + verify(probe, times(0)).setRepairThreads(repairType.name(), 0); } @Test(expected = IllegalArgumentException.class) @@ -237,23 +234,25 @@ public void testInvalidParamType() @Test public void testPriorityHosts() { + String commaSeparatedHostSet = String.join(",", localEndpoint.toString().substring(1), otherEndpoint.toString().substring(1)); cmd.repairTypeStr = repairType.name(); - cmd.args = ImmutableList.of("priority_hosts", String.join(",", localEndpoint.toString().substring(1), otherEndpoint.toString().substring(1))); + cmd.args = ImmutableList.of("priority_hosts", commaSeparatedHostSet); cmd.execute(probe); - verify(probe, times(1)).setRepairPriorityForHosts(repairType, ImmutableSet.of(localEndpoint, otherEndpoint)); + verify(probe, times(1)).setRepairPriorityForHosts(repairType.name(), commaSeparatedHostSet); } @Test public void testForceRepairHosts() { + String commaSeparatedHostSet = String.join(",", localEndpoint.toString().substring(1), otherEndpoint.toString().substring(1)); cmd.repairTypeStr = repairType.name(); - cmd.args = ImmutableList.of("forcerepair_hosts", String.join(",", localEndpoint.toString().substring(1), otherEndpoint.toString().substring(1))); + cmd.args = ImmutableList.of("forcerepair_hosts", commaSeparatedHostSet); cmd.execute(probe); - verify(probe, times(1)).setForceRepairForHosts(repairType, ImmutableSet.of(localEndpoint, otherEndpoint)); + verify(probe, times(1)).setForceRepairForHosts(repairType.name(), commaSeparatedHostSet); } } @@ -276,17 +275,17 @@ public static class RepairTypeAndArgsParamsTests public static Collection testCases() { return Stream.of( - forEachRepairType("enabled", "true", (type) -> verify(probe, times(1)).setAutoRepairEnabled(type, true)), - forEachRepairType("number_of_repair_threads", "1", (type) -> verify(probe, times(1)).setRepairThreads(type, 1)), - forEachRepairType("min_repair_interval", "3h", (type) -> verify(probe, times(1)).setRepairMinInterval(type, "3h")), - forEachRepairType("sstable_upper_threshold", "4", (type) -> verify(probe, times(1)).setRepairSSTableCountHigherThreshold(type, 4)), - forEachRepairType("table_max_repair_time", "5s", (type) -> verify(probe, times(1)).setAutoRepairTableMaxRepairTime(type, "5s")), - forEachRepairType("repair_primary_token_range_only", "true", (type) -> verify(probe, times(1)).setPrimaryTokenRangeOnly(type, true)), - forEachRepairType("parallel_repair_count", "6", (type) -> verify(probe, times(1)).setParallelRepairCount(type, 6)), - forEachRepairType("parallel_repair_percentage", "7", (type) -> verify(probe, times(1)).setParallelRepairPercentage(type, 7)), - forEachRepairType("materialized_view_repair_enabled", "true", (type) -> verify(probe, times(1)).setMaterializedViewRepairEnabled(type, true)), - forEachRepairType("ignore_dcs", "dc1,dc2", (type) -> verify(probe, times(1)).setAutoRepairIgnoreDCs(type, ImmutableSet.of("dc1", "dc2"))), - forEachRepairType("token_range_splitter.max_bytes_per_schedule", "500GiB", (type) -> verify(probe, times(1)).setAutoRepairTokenRangeSplitterParameter(type, "max_bytes_per_schedule", "500GiB")) + forEachRepairType("enabled", "true", (type) -> verify(probe, times(1)).setAutoRepairEnabled(type.name(), true)), + forEachRepairType("number_of_repair_threads", "1", (type) -> verify(probe, times(1)).setRepairThreads(type.name(), 1)), + forEachRepairType("min_repair_interval", "3h", (type) -> verify(probe, times(1)).setRepairMinInterval(type.name(), "3h")), + forEachRepairType("sstable_upper_threshold", "4", (type) -> verify(probe, times(1)).setRepairSSTableCountHigherThreshold(type.name(), 4)), + forEachRepairType("table_max_repair_time", "5s", (type) -> verify(probe, times(1)).setAutoRepairTableMaxRepairTime(type.name(), "5s")), + forEachRepairType("repair_primary_token_range_only", "true", (type) -> verify(probe, times(1)).setPrimaryTokenRangeOnly(type.name(), true)), + forEachRepairType("parallel_repair_count", "6", (type) -> verify(probe, times(1)).setParallelRepairCount(type.name(), 6)), + forEachRepairType("parallel_repair_percentage", "7", (type) -> verify(probe, times(1)).setParallelRepairPercentage(type.name(), 7)), + forEachRepairType("materialized_view_repair_enabled", "true", (type) -> verify(probe, times(1)).setMaterializedViewRepairEnabled(type.name(), true)), + forEachRepairType("ignore_dcs", "dc1,dc2", (type) -> verify(probe, times(1)).setAutoRepairIgnoreDCs(type.name(), ImmutableSet.of("dc1", "dc2"))), + forEachRepairType("token_range_splitter.max_bytes_per_schedule", "500GiB", (type) -> verify(probe, times(1)).setAutoRepairTokenRangeSplitterParameter(type.name(), "max_bytes_per_schedule", "500GiB")) ).flatMap(Function.identity()).collect(Collectors.toList()); } @@ -325,7 +324,7 @@ public void test() verifyFunc.accept(repairType); // test scenario when auto repair is disabled - when(probe.getAutoRepairConfig()).thenReturn(new AutoRepairConfig(false)); + when(probe.isAutoRepairDisabled()).thenReturn(true); cmd.execute(probe); From 15685b7bef97dabe4c17b437d499f3008bc25e22 Mon Sep 17 00:00:00 2001 From: Francisco Guerrero Date: Sat, 25 Jan 2025 09:01:42 -0800 Subject: [PATCH 122/199] Address comments --- src/java/org/apache/cassandra/locator/InetAddressAndPort.java | 2 +- .../apache/cassandra/tools/nodetool/AutoRepairStatusTest.java | 1 - .../cassandra/tools/nodetool/SetAutoRepairConfigTest.java | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java index 3e9fd92da4f0..53d35d40b040 100644 --- a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java +++ b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java @@ -326,7 +326,7 @@ public static Set parseHosts(String value, boolean failOnErr { if (failOnError) { - throw new IllegalArgumentException(e.getMessage()); + throw new IllegalArgumentException("Failed to parse host: " + host, e); } else { diff --git a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java index 4e5f86e7233d..1c7b04d50bcb 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java @@ -72,7 +72,6 @@ public void setUp() throws Exception setAutoRepairEnabled(true); DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(AutoRepairConfig.RepairType.FULL, true); DatabaseDescriptor.getAutoRepairConfig().setAutoRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, true); -// when(probe.getAutoRepairConfig()).thenReturn(DatabaseDescriptor.getAutoRepairConfig()); } @Test(expected = IllegalArgumentException.class) diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index 593cb7c190aa..b0b499f67a08 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -192,7 +192,6 @@ public void testRepairSchedulingDisabled() @Test public void testRepairTypeDisabled() { -// config.setAutoRepairEnabled(repairType, false); cmd.repairTypeStr = repairType.name(); cmd.args = ImmutableList.of("number_of_repair_threads", "1"); From 920bfec069c8158bd7b879692930c3f469a40c67 Mon Sep 17 00:00:00 2001 From: Francisco Guerrero Date: Mon, 27 Jan 2025 19:48:50 -0800 Subject: [PATCH 123/199] Fix tests and rename autoRepairConfiguration -> getAutoRepairConfiguration in the AutoRepairServiceMBean definition --- .../cassandra/service/AutoRepairService.java | 2 +- .../cassandra/service/AutoRepairServiceMBean.java | 2 +- .../org/apache/cassandra/tools/NodeProbe.java | 2 +- .../cassandra/service/AutoRepairServiceTest.java | 15 ++++++++++++--- .../service/StorageServiceServerTest.java | 4 ++-- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index af850f6c2dbf..1f7004c31d48 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -92,7 +92,7 @@ public boolean isAutoRepairDisabled() } @Override - public String autoRepairConfiguration() + public String getAutoRepairConfiguration() { StringBuilder sb = new StringBuilder(); sb.append("repair scheduler configuration:"); diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index bad7667c6acc..be6cedba9329 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -61,7 +61,7 @@ public interface AutoRepairServiceMBean public boolean isAutoRepairDisabled(); - public String autoRepairConfiguration(); + public String getAutoRepairConfiguration(); public void setRepairSessionTimeout(String repairType, String timeout); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index ee20387fa7f4..d966b0d802d6 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2151,7 +2151,7 @@ public boolean isAutoRepairDisabled() public String autoRepairConfiguration() { - return autoRepairProxy.autoRepairConfiguration(); + return autoRepairProxy.getAutoRepairConfiguration(); } public void setAutoRepairTokenRangeSplitterParameter(String repairType, String key, String value) diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index c9defcf54bf2..ca9cf945c18e 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -50,6 +50,7 @@ import static org.apache.cassandra.Util.setAutoRepairEnabled; import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; import static org.junit.Assert.assertEquals; +import static org.assertj.core.api.Assertions.assertThat; @RunWith(Suite.class) @Suite.SuiteClasses({ AutoRepairServiceTest.BasicTests.class, AutoRepairServiceTest.SetterTests.class }) @@ -222,7 +223,7 @@ public static class SetterTests extends CQLTester public T arg; @Parameterized.Parameter(2) - public BiConsumer setter; + public BiConsumer setter; @Parameterized.Parameter(3) public Function getter; @@ -296,8 +297,16 @@ public void testSetters() { DatabaseDescriptor.setCDCOnRepairEnabled(false); DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); - setter.accept(repairType, arg); - assertEquals(arg, getter.apply(repairType)); + setter.accept(repairType.name(), arg); + T actualConfig = getter.apply(repairType); + if (actualConfig instanceof Set) + // When performing a setRepairPriorityForHosts or setForceRepairForHosts, a comma-separated list of + // ip addresses is provided as input. The configuration is expected to return a Set of Strings that + // represent the configured IP addresses. This especial handling allows verification of this special + // case where one of the entries in the Set must match the configured input. + assertThat(actualConfig).satisfiesAnyOf(entry -> assertThat(entry.toString()).contains(arg.toString())); + else + assertThat(actualConfig).isEqualTo(arg); } } } diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java index d4f6d6a7e475..74d952425b02 100644 --- a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java +++ b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java @@ -47,7 +47,6 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.gms.ApplicationState; import org.apache.cassandra.gms.Gossiper; @@ -721,8 +720,9 @@ public void testMutateSSTableRepairedStateTableNotFound() StorageService.instance.mutateSSTableRepairedState(true, false, keyspace, Arrays.asList("MISSING_TABLE")); fail("Expected an InvalidRequestException to be thrown"); } - catch (InvalidRequestException e) + catch (RuntimeException e) { + assertEquals("Table MISSING_TABLE does not exist in keyspace " + keyspace, e.getMessage()); // Test passed } } From be5e9e145ab051e00345bb3d81d39095fd551b70 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 13 Feb 2025 20:11:51 -0800 Subject: [PATCH 124/199] Leak fix in RepairTokenRangeSplitterTest.java --- .../RepairTokenRangeSplitterTest.java | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index 433e0d3ef5e5..e15ce7a0b2aa 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -78,16 +78,19 @@ public void setUp() public void testSizePartitionCount() throws Throwable { insertAndFlushTable(tableName, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10); - Refs sstables = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE); - assertEquals(10, sstables.iterator().next().getEstimatedPartitionSize().count()); - SizeEstimate sizes = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE, sstables); - assertEquals(10, sizes.partitions); + try (Refs sstables = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE)) + { + assertEquals(10, sstables.iterator().next().getEstimatedPartitionSize().count()); + SizeEstimate sizes = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE, sstables); + assertEquals(10, sizes.partitions); + } } @Test public void testSizePartitionCountSplit() throws Throwable { - int[] values = new int[10000]; + int partitionCount = 100_000; + int[] values = new int[partitionCount]; for (int i = 0; i < values.length; i++) values[i] = i + 1; insertAndFlushTable(tableName, values); @@ -96,12 +99,17 @@ public void testSizePartitionCountSplit() throws Throwable Range tokenRange2 = range.next(); Assert.assertFalse(range.hasNext()); - Refs sstables1 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange1); - Refs sstables2 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange2); - SizeEstimate sizes1 = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange1, sstables1); - SizeEstimate sizes2 = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange2, sstables2); - // +-5% because HLL merge and the applying of range size approx ratio causes estimation errors - assertTrue(Math.abs(10000 - (sizes1.partitions + sizes2.partitions)) <= 60); + try(Refs sstables1 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange1); + Refs sstables2 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange2)) + { + SizeEstimate sizes1 = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange1, sstables1); + SizeEstimate sizes2 = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange2, sstables2); + + // +-5% because including entire compression blocks covering token range, HLL merge and the applying of range size approx ratio causes estimation errors + long allowableDelta = (long) (partitionCount * .05); + long estimatedPartitionDelta = Math.abs(partitionCount - (sizes1.partitions + sizes2.partitions)); + assertTrue("Partition count delta was +/-" + estimatedPartitionDelta + " but expected +/- " + allowableDelta, estimatedPartitionDelta <= allowableDelta); + } } @Test From 45e13e0c48afc6bb07d92cc3b391af5849a9b0bd Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 10 Feb 2025 15:59:30 -0800 Subject: [PATCH 125/199] Fix review comments from Andy --- conf/cassandra.yaml | 5 ++ .../org/apache/cassandra/config/Config.java | 2 +- .../apache/cassandra/config/DurationSpec.java | 3 +- .../cassandra/config/ParameterizedClass.java | 2 +- .../db/streaming/CassandraStreamReceiver.java | 11 ++-- .../FixedSplitTokenRangeSplitter.java | 3 +- .../service/ActiveRepairService.java | 5 +- .../service/ActiveRepairServiceMBean.java | 4 ++ .../cassandra/service/StorageService.java | 2 +- .../apache/cassandra/utils/FBUtilities.java | 8 +-- .../config/YamlConfigurationLoaderTest.java | 2 +- .../service/ActiveRepairServiceTest.java | 51 ++++++++++--------- .../service/AutoRepairServiceTest.java | 3 -- 13 files changed, 55 insertions(+), 46 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 9897ddb3ff1c..fab08ea2987c 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1866,6 +1866,11 @@ drop_compact_storage_enabled: false # excluded_keyspaces: # comma separated list of keyspaces to exclude from the check # excluded_tables: # comma separated list of keyspace.table pairs to exclude from the check +# at least 20% of disk must be unused to run incremental repair +# if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason) +# then set the ratio to 0.0 +# incremental_repair_disk_headroom_reject_ratio = 0.2; + # Configuration for AutoRepair Scheduler. # auto_repair: # # Enable/Disable the auto-repair scheduler. diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 965318466726..841493e7a5c7 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -34,13 +34,13 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.audit.AuditLogOptions; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.fql.FullQueryLoggerOptions; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.service.StartupChecks.StartupCheckType; /** diff --git a/src/java/org/apache/cassandra/config/DurationSpec.java b/src/java/org/apache/cassandra/config/DurationSpec.java index 0c2b32626051..10d56c23eb71 100644 --- a/src/java/org/apache/cassandra/config/DurationSpec.java +++ b/src/java/org/apache/cassandra/config/DurationSpec.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.config; -import java.io.Serializable; import java.util.Arrays; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -40,7 +39,7 @@ * users the opportunity to be able to provide config with a unit of their choice in cassandra.yaml as per the available * options. (CASSANDRA-15234) */ -public abstract class DurationSpec implements Serializable +public abstract class DurationSpec { /** * The Regexp used to parse the duration provided as String. diff --git a/src/java/org/apache/cassandra/config/ParameterizedClass.java b/src/java/org/apache/cassandra/config/ParameterizedClass.java index a101caa92ca5..f14ee1e3ed28 100644 --- a/src/java/org/apache/cassandra/config/ParameterizedClass.java +++ b/src/java/org/apache/cassandra/config/ParameterizedClass.java @@ -28,7 +28,7 @@ import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; @Shared(scope = SIMULATION) -public class ParameterizedClass implements Serializable +public class ParameterizedClass { public static final String CLASS_NAME = "class_name"; public static final String PARAMETERS = "parameters"; diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index 731e9eae528b..fd6b806d3925 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -26,18 +26,17 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; -import org.apache.cassandra.io.sstable.SSTable; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.github.jbellis.jvector.annotations.VisibleForTesting; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.ThrottledUnfilteredIterator; @@ -46,6 +45,7 @@ import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableMultiWriter; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.streaming.IncomingStream; @@ -188,7 +188,8 @@ private boolean cdcRequiresWriteCommitLog(ColumnFamilyStore cfs) * For CDC-enabled tables, we want to ensure that the mutations are run through the CommitLog so they * can be archived by the CDC process on discard. */ - public boolean requiresWritePath(ColumnFamilyStore cfs) + @VisibleForTesting + boolean requiresWritePath(ColumnFamilyStore cfs) { return cdcRequiresWriteCommitLog(cfs) || cfs.streamToMemtable() diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index d20475984f6f..e65f8767687c 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -70,7 +70,8 @@ public FixedSplitTokenRangeSplitter(AutoRepairConfig.RepairType repairType, Map< @Override public Iterator getRepairAssignments(boolean primaryRangeOnly, List repairPlans) { - return new RepairAssignmentIterator(repairPlans) { + return new RepairAssignmentIterator(repairPlans) + { @Override protected KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repairPlan) diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index 4d33e0493bd3..451487f3b254 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -41,6 +41,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DurationSpec; @@ -61,8 +64,6 @@ import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.CountDownLatch; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.DatabaseDescriptor; diff --git a/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java b/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java index 532b17c6f5ff..fdda254f1459 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java @@ -61,4 +61,8 @@ public interface ActiveRepairServiceMBean int parentRepairSessionsCount(); public int getPaxosRepairParallelism(); public void setPaxosRepairParallelism(int v); + + public double getIncrementalRepairDiskHeadroomRejectRatio(); + + public void setIncrementalRepairDiskHeadroomRejectRatio(double value); } diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 491d6b0f27a5..a684a9adac01 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -1352,7 +1352,7 @@ private void doAutoRepairSetup() AutoRepairService.setup(); if (DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) { - logger.info("Enable auto-repair scheduling"); + logger.info("Enabling auto-repair scheduling"); AutoRepair.instance.setup(); } logger.info("AutoRepair setup complete!"); diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index 3d881ddf3f59..151ceef02ca3 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -50,10 +50,6 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.io.util.FileInputStreamPlus; -import org.apache.cassandra.io.util.FileOutputStreamPlus; -import org.apache.cassandra.utils.concurrent.*; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,9 +78,13 @@ import org.apache.cassandra.io.sstable.metadata.ValidationMetadata; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.DataOutputBufferFixed; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileInputStreamPlus; +import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.security.ISslContextFactory; +import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR; diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java index 5e21228b6976..ead42a9f039b 100644 --- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java +++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java @@ -33,6 +33,7 @@ import org.apache.cassandra.distributed.shared.WithProperties; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.yaml.snakeyaml.error.YAMLException; import static org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_ALLOW_SYSTEM_PROPERTIES; @@ -40,7 +41,6 @@ import static org.apache.cassandra.config.YamlConfigurationLoader.SYSTEM_PROPERTY_PREFIX; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java index 2f1b3c3a2288..5a158b61b455 100644 --- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java @@ -1,21 +1,21 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.cassandra.service; import java.util.ArrayList; @@ -39,6 +39,7 @@ import org.apache.cassandra.service.disk.usage.DiskUsageMonitor; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Condition; + import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -233,7 +234,7 @@ public void testGetNeighborsTimesTwoInSpecifiedHosts() throws Throwable } expected.remove(FBUtilities.getBroadcastAddressAndPort()); - Collection hosts = Arrays.asList(FBUtilities.getBroadcastAddressAndPort().getHostAddressAndPort(),expected.get(0).getHostAddressAndPort()); + Collection hosts = Arrays.asList(FBUtilities.getBroadcastAddressAndPort().getHostAddressAndPort(), expected.get(0).getHostAddressAndPort()); Iterable> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges(); assertEquals(expected.get(0), ActiveRepairService.getNeighbors(KEYSPACE5, ranges, @@ -269,7 +270,6 @@ public void testParentRepairStatus() throws Throwable List failed = StorageService.instance.getParentRepairStatus(3); assertNotNull(failed); assertEquals(ActiveRepairService.ParentRepairStatus.FAILED, ActiveRepairService.ParentRepairStatus.valueOf(failed.get(0))); - } Set addTokens(int max) throws Throwable @@ -341,10 +341,10 @@ private static RepairOption opts(String... params) { assert params.length % 2 == 0 : "unbalanced key value pairs"; Map opt = new HashMap<>(); - for (int i=0; i<(params.length >> 1); i++) + for (int i = 0; i < (params.length >> 1); i++) { int idx = i << 1; - opt.put(params[idx], params[idx+1]); + opt.put(params[idx], params[idx + 1]); } return RepairOption.parse(opt, DatabaseDescriptor.getPartitioner()); } @@ -376,7 +376,7 @@ public void repairedAt() throws Exception Assert.assertNotEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), FORCE_REPAIR_KEY, b2s(true)), false)); Assert.assertEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), - FORCE_REPAIR_KEY, b2s(true)), true)); + FORCE_REPAIR_KEY, b2s(true)), true)); // full repair Assert.assertEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(false)), false)); @@ -422,7 +422,8 @@ public void testRejectWhenPoolFullStrategy() throws InterruptedException // Submission is unblocked Thread.sleep(250); - validationExecutor.submit(() -> {}); + validationExecutor.submit(() -> { + }); } finally { @@ -459,8 +460,8 @@ public void testQueueWhenPoolFullStrategy() throws InterruptedException allSubmitted.await(TASK_SECONDS + 1, TimeUnit.SECONDS); // Give the tasks we expect to execute immediately chance to be scheduled - Util.spinAssertEquals(2 , ((ExecutorPlus) validationExecutor)::getActiveTaskCount, 1); - Util.spinAssertEquals(3 , ((ExecutorPlus) validationExecutor)::getPendingTaskCount, 1); + Util.spinAssertEquals(2, ((ExecutorPlus) validationExecutor)::getActiveTaskCount, 1); + Util.spinAssertEquals(3, ((ExecutorPlus) validationExecutor)::getPendingTaskCount, 1); // verify that we've reached a steady state with 2 threads actively processing and 3 queued tasks Assert.assertEquals(2, ((ExecutorPlus) validationExecutor).getActiveTaskCount()); diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index ca9cf945c18e..35239d3b0bb0 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -97,7 +97,6 @@ public void testsetAutoRepairHistoryClearDeleteHostsBufferInSecV2() assertEquals(100, config.getAutoRepairHistoryClearDeleteHostsBufferInterval().toSeconds()); } - @Test public void testsetAutoRepairMaxRetriesCount() { @@ -106,7 +105,6 @@ public void testsetAutoRepairMaxRetriesCount() assertEquals(101, config.getRepairMaxRetries()); } - @Test public void testsetAutoRepairRetryBackoffInSec() { @@ -182,7 +180,6 @@ public static Collection repairTypes() return Arrays.asList(AutoRepairConfig.RepairType.values()); } - @BeforeClass public static void setupClass() throws Exception { From e7351a7674b170dee6b95cfd141a01b4237ba26b Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 10 Feb 2025 20:46:18 -0800 Subject: [PATCH 126/199] Fix review comments from Jaydeepkumar --- conf/cassandra.yaml | 3 +- .../db/streaming/CassandraStreamReceiver.java | 2 +- .../cassandra/metrics/AutoRepairMetrics.java | 1 - .../metrics/AutoRepairMetricsManager.java | 3 ++ .../repair/autorepair/AutoRepair.java | 3 ++ .../repair/autorepair/AutoRepairConfig.java | 3 ++ .../repair/autorepair/AutoRepairState.java | 4 ++- .../FixedSplitTokenRangeSplitter.java | 3 ++ .../cassandra/schema/AutoRepairParams.java | 3 ++ .../apache/cassandra/schema/TableParams.java | 3 +- .../cassandra/service/AutoRepairService.java | 3 ++ .../service/AutoRepairServiceMBean.java | 3 ++ .../org/apache/cassandra/tools/NodeProbe.java | 26 ++++++++-------- .../tools/nodetool/AutoRepairStatus.java | 5 ++- .../tools/nodetool/GetAutoRepairConfig.java | 3 ++ .../tools/nodetool/SSTableRepairedSet.java | 5 ++- .../tools/nodetool/SetAutoRepairConfig.java | 25 ++++++++------- .../test/repair/AutoRepairSchedulerTest.java | 3 ++ .../CassandraStreamReceiverTest.java | 3 ++ .../AutoRepairConfigRepairTypeTest.java | 1 - .../autorepair/AutoRepairConfigTest.java | 3 ++ .../autorepair/AutoRepairKeyspaceTest.java | 3 ++ .../AutoRepairParameterizedTest.java | 3 ++ .../AutoRepairStateFactoryTest.java | 3 ++ .../autorepair/AutoRepairStateTest.java | 3 ++ .../repair/autorepair/AutoRepairTest.java | 3 ++ .../autorepair/AutoRepairUtilsTest.java | 3 ++ .../FixedSplitTokenRangeSplitterTest.java | 3 ++ .../autorepair/PrioritizedRepairPlanTest.java | 3 ++ .../RepairTokenRangeSplitterTest.java | 6 ++++ .../service/AutoRepairServiceTest.java | 3 ++ .../tools/nodetool/AutoRepairStatusTest.java | 5 ++- .../nodetool/SSTableRepairedSetTest.java | 7 +++-- .../nodetool/SetAutoRepairConfigTest.java | 31 ++++++++++--------- 34 files changed, 134 insertions(+), 50 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index fab08ea2987c..89aa58714fd5 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1866,7 +1866,8 @@ drop_compact_storage_enabled: false # excluded_keyspaces: # comma separated list of keyspaces to exclude from the check # excluded_tables: # comma separated list of keyspace.table pairs to exclude from the check -# at least 20% of disk must be unused to run incremental repair +# At least 20% of disk must be unused to run incremental repair. It is useful to avoid disks filling up during +# incremental repair as anti-compaction during incremental repair may contribute to additional space temporarily. # if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason) # then set the ratio to 0.0 # incremental_repair_disk_headroom_reject_ratio = 0.2; diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index fd6b806d3925..62aa326a1ea2 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -23,13 +23,13 @@ import java.util.List; import java.util.Set; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.github.jbellis.jvector.annotations.VisibleForTesting; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java index 8da0ba3523d5..c9b15979c245 100644 --- a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java @@ -89,7 +89,6 @@ public Integer getValue() } }); - longestUnrepairedSec = Metrics.register(factory.createMetricName("LongestUnrepairedSec"), new Gauge() { public Integer getValue() diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetricsManager.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetricsManager.java index e293945c9846..e97ce34e5a73 100644 --- a/src/java/org/apache/cassandra/metrics/AutoRepairMetricsManager.java +++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetricsManager.java @@ -23,6 +23,9 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +/** + * AutoRepair metrics manager holding all the auto-repair related metrics. + */ public class AutoRepairMetricsManager { private static final Map metrics = new ConcurrentHashMap<>(); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 9ee955d07fed..798578fbe34e 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -63,6 +63,9 @@ import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_DUE_TO_PRIORITY; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_FORCE_REPAIR; +/** + * AutoRepair scheduler responsible for running different types of repairs. + */ public class AutoRepair { private static final Logger logger = LoggerFactory.getLogger(AutoRepair.class); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 07d894c4f1ff..021eff330efb 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -38,6 +38,9 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.utils.FBUtilities; +/** + * Defines configurations for AutoRepair. + */ public class AutoRepairConfig implements Serializable { // Enable/Disable the auto-repair scheduler. diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index e4d449004136..87ed93881bcd 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -55,7 +55,9 @@ import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; -// AutoRepairState represents the state of automated repair for a given repair type. +/** + * AutoRepairState represents the state of automated repair for a given repair type. + */ public abstract class AutoRepairState implements ProgressListener { protected static final Logger logger = LoggerFactory.getLogger(AutoRepairState.class); diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index e65f8767687c..f19006717663 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -36,6 +36,9 @@ import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; +/** + * An implementation that splits token ranges into a fixed number of subranges. + */ public class FixedSplitTokenRangeSplitter implements IAutoRepairTokenRangeSplitter { private static final Logger logger = LoggerFactory.getLogger(FixedSplitTokenRangeSplitter.class); diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index b012a48188d8..34e4f9f5b97f 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -31,6 +31,9 @@ import static java.lang.String.format; +/** + * AutoRepair table parameters - used to define the auto-repair configuration for a table. + */ public final class AutoRepairParams { public enum Option diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 11264d221f4d..fa28ff726532 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -135,8 +135,7 @@ public static Builder builder(TableParams params) .extensions(params.extensions) .cdc(params.cdc) .readRepair(params.readRepair) - .automatedRepair(params.autoRepair) - ; + .automatedRepair(params.autoRepair); } public Builder unbuild() diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 1f7004c31d48..1c96ab685d2c 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -36,6 +36,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; +/** + * Implement all the MBeans for AutoRepair. + */ public class AutoRepairService implements AutoRepairServiceMBean { public static final String MBEAN_NAME = "org.apache.cassandra.db:type=AutoRepairService"; diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index be6cedba9329..40948f3711a5 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -20,6 +20,9 @@ import java.util.Set; +/** + * Defines all the MBeans exposed for AutoRepair. + */ public interface AutoRepairServiceMBean { /** diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index d966b0d802d6..b427a415eb9f 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2164,22 +2164,22 @@ public void setAutoRepairEnabled(String repairType, boolean enabled) autoRepairProxy.setAutoRepairEnabled(repairType, enabled); } - public void setRepairThreads(String repairType, int repairThreads) + public void setAutoRepairThreads(String repairType, int repairThreads) { autoRepairProxy.setRepairThreads(repairType, repairThreads); } - public void setRepairPriorityForHosts(String repairType, String commaSeparatedHostSet) + public void setAutoRepairPriorityForHosts(String repairType, String commaSeparatedHostSet) { autoRepairProxy.setRepairPriorityForHosts(repairType, commaSeparatedHostSet); } - public void setForceRepairForHosts(String repairType, String commaSeparatedHostSet) + public void setAutoRepairForceRepairForHosts(String repairType, String commaSeparatedHostSet) { autoRepairProxy.setForceRepairForHosts(repairType, commaSeparatedHostSet); } - public void setRepairMinInterval(String repairType, String minRepairInterval) + public void setAutoRepairMinInterval(String repairType, String minRepairInterval) { autoRepairProxy.setRepairMinInterval(repairType, minRepairInterval); } @@ -2189,7 +2189,7 @@ public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration) autoRepairProxy.setAutoRepairHistoryClearDeleteHostsBufferDuration(duration); } - public void startScheduler() + public void startAutoRepairScheduler() { autoRepairProxy.startScheduler(); } @@ -2209,7 +2209,7 @@ public void setAutoRepairMinRepairTaskDuration(String duration) autoRepairProxy.setAutoRepairMinRepairTaskDuration(duration); } - public void setRepairSSTableCountHigherThreshold(String repairType, int ssTableHigherThreshold) + public void setAutoRepairSSTableCountHigherThreshold(String repairType, int ssTableHigherThreshold) { autoRepairProxy.setRepairSSTableCountHigherThreshold(repairType, ssTableHigherThreshold); } @@ -2224,22 +2224,22 @@ public void setAutoRepairIgnoreDCs(String repairType, Set ignoreDCs) autoRepairProxy.setIgnoreDCs(repairType, ignoreDCs); } - public void setParallelRepairPercentage(String repairType, int percentage) + public void setAutoRepairParallelRepairPercentage(String repairType, int percentage) { autoRepairProxy.setParallelRepairPercentage(repairType, percentage); } - public void setParallelRepairCount(String repairType, int count) + public void setAutoRepairParallelRepairCount(String repairType, int count) { autoRepairProxy.setParallelRepairCount(repairType, count); } - public void setPrimaryTokenRangeOnly(String repairType, boolean primaryTokenRangeOnly) + public void setAutoRepairPrimaryTokenRangeOnly(String repairType, boolean primaryTokenRangeOnly) { autoRepairProxy.setPrimaryTokenRangeOnly(repairType, primaryTokenRangeOnly); } - public void setMaterializedViewRepairEnabled(String repairType, boolean enabled) + public void setAutoRepairMaterializedViewRepairEnabled(String repairType, boolean enabled) { autoRepairProxy.setMVRepairEnabled(repairType, enabled); } @@ -2249,17 +2249,17 @@ public List mutateSSTableRepairedState(boolean repair, boolean preview, return ssProxy.mutateSSTableRepairedState(repair, preview, keyspace, tables); } - public List getTablesForKeyspace(String keyspace) + public List getAutoRepairTablesForKeyspace(String keyspace) { return ssProxy.getTablesForKeyspace(keyspace); } - public void setRepairSessionTimeout(String repairType, String timeout) + public void setAutoRepairSessionTimeout(String repairType, String timeout) { autoRepairProxy.setRepairSessionTimeout(repairType, timeout); } - public Set getOnGoingRepairHostIds(String repairType) + public Set getAutoRepairOnGoingRepairHostIds(String repairType) { return autoRepairProxy.getOnGoingRepairHostIds(repairType); } diff --git a/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java index 22f9afc43fe1..bb594a010ff1 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java +++ b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java @@ -31,6 +31,9 @@ import static com.google.common.base.Preconditions.checkArgument; +/** + * Provides currently running auto-repair tasks. + */ @Command(name = "autorepairstatus", description = "Print autorepair status") public class AutoRepairStatus extends NodeTool.NodeToolCmd { @@ -52,7 +55,7 @@ public void execute(NodeProbe probe) TableBuilder table = new TableBuilder(); table.add("Active Repairs"); - Set ongoingRepairHostIds = probe.getOnGoingRepairHostIds(repairType); + Set ongoingRepairHostIds = probe.getAutoRepairOnGoingRepairHostIds(repairType); table.add(getSetString(ongoingRepairHostIds)); table.printTo(out); } diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java index 961fb71fb4e4..9744498de757 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java @@ -25,6 +25,9 @@ import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.NodeTool.NodeToolCmd; +/** + * Prints all the configurations for AutoRepair through nodetool. + */ @Command(name = "getautorepairconfig", description = "Print autorepair configurations") public class GetAutoRepairConfig extends NodeToolCmd { diff --git a/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java b/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java index e1773fe47b7f..2a7b56732ac9 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java @@ -31,6 +31,9 @@ import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.NodeTool; +/** + * Provides a way to set the repaired state of SSTables without any downtime through nodetool. + */ @Command(name = "sstablerepairedset", description = "Set the repaired state of SSTables for given keyspace/tables") public class SSTableRepairedSet extends NodeTool.NodeToolCmd { @@ -87,7 +90,7 @@ public void execute(NodeProbe probe) { sstableList.addAll(probe.mutateSSTableRepairedState(isRepaired, !reallySet, keyspace, tables.isEmpty() - ? probe.getTablesForKeyspace(keyspace) // mutate all tables + ? probe.getAutoRepairTablesForKeyspace(keyspace) // mutate all tables : tables)); // mutate specific tables } catch (InvalidRequestException e) diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index d55685e72426..9f22a43c4eb9 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -35,6 +35,9 @@ import static com.google.common.base.Preconditions.checkArgument; +/** + * Allows to set AutoRepair configuration through nodetool. + */ @Command(name = "setautorepairconfig", description = "sets the autorepair configuration") public class SetAutoRepairConfig extends NodeToolCmd { @@ -77,7 +80,7 @@ public void execute(NodeProbe probe) case "start_scheduler": if (Boolean.parseBoolean(paramVal)) { - probe.startScheduler(); + probe.startAutoRepairScheduler(); } return; case "history_clear_delete_hosts_buffer_interval": @@ -113,13 +116,13 @@ public void execute(NodeProbe probe) probe.setAutoRepairEnabled(repairTypeStr, Boolean.parseBoolean(paramVal)); break; case "number_of_repair_threads": - probe.setRepairThreads(repairTypeStr, Integer.parseInt(paramVal)); + probe.setAutoRepairThreads(repairTypeStr, Integer.parseInt(paramVal)); break; case "min_repair_interval": - probe.setRepairMinInterval(repairTypeStr, paramVal); + probe.setAutoRepairMinInterval(repairTypeStr, paramVal); break; case "sstable_upper_threshold": - probe.setRepairSSTableCountHigherThreshold(repairTypeStr, Integer.parseInt(paramVal)); + probe.setAutoRepairSSTableCountHigherThreshold(repairTypeStr, Integer.parseInt(paramVal)); break; case "table_max_repair_time": probe.setAutoRepairTableMaxRepairTime(repairTypeStr, paramVal); @@ -127,11 +130,11 @@ public void execute(NodeProbe probe) case "priority_hosts": if (paramVal!= null && !paramVal.isEmpty()) { - probe.setRepairPriorityForHosts(repairTypeStr, paramVal); + probe.setAutoRepairPriorityForHosts(repairTypeStr, paramVal); } break; case "forcerepair_hosts": - probe.setForceRepairForHosts(repairTypeStr, paramVal); + probe.setAutoRepairForceRepairForHosts(repairTypeStr, paramVal); break; case "ignore_dcs": Set ignoreDCs = new HashSet<>(); @@ -142,19 +145,19 @@ public void execute(NodeProbe probe) probe.setAutoRepairIgnoreDCs(repairTypeStr, ignoreDCs); break; case "repair_primary_token_range_only": - probe.setPrimaryTokenRangeOnly(repairTypeStr, Boolean.parseBoolean(paramVal)); + probe.setAutoRepairPrimaryTokenRangeOnly(repairTypeStr, Boolean.parseBoolean(paramVal)); break; case "parallel_repair_count": - probe.setParallelRepairCount(repairTypeStr, Integer.parseInt(paramVal)); + probe.setAutoRepairParallelRepairCount(repairTypeStr, Integer.parseInt(paramVal)); break; case "parallel_repair_percentage": - probe.setParallelRepairPercentage(repairTypeStr, Integer.parseInt(paramVal)); + probe.setAutoRepairParallelRepairPercentage(repairTypeStr, Integer.parseInt(paramVal)); break; case "materialized_view_repair_enabled": - probe.setMaterializedViewRepairEnabled(repairTypeStr, Boolean.parseBoolean(paramVal)); + probe.setAutoRepairMaterializedViewRepairEnabled(repairTypeStr, Boolean.parseBoolean(paramVal)); break; case "repair_session_timeout": - probe.setRepairSessionTimeout(repairTypeStr, paramVal); + probe.setAutoRepairSessionTimeout(repairTypeStr, paramVal); break; default: throw new IllegalArgumentException("Unknown parameter: " + paramType); diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index 31664fb6a886..175a32fda2bc 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -42,6 +42,9 @@ import static org.apache.cassandra.schema.SchemaConstants.DISTRIBUTED_KEYSPACE_NAME; import static org.junit.Assert.assertEquals; +/** + * Unit tests for {@link org.apache.cassandra.repair.autorepair.AutoRepair} + */ public class AutoRepairSchedulerTest extends TestBaseImpl { diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamReceiverTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamReceiverTest.java index 3b9b3b271206..466cfe9eab21 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamReceiverTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamReceiverTest.java @@ -35,6 +35,9 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.when; +/** + * Unit tests for {@link org.apache.cassandra.db.streaming.CassandraStreamReceiver} + */ public class CassandraStreamReceiverTest extends CQLTester { @Mock diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigRepairTypeTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigRepairTypeTest.java index be14b8b912ea..ec36bb5cfd0c 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigRepairTypeTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigRepairTypeTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.repair.autorepair; - import org.junit.Assert; import org.junit.Test; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 3bef45b8bd53..29e564cc3e12 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -44,6 +44,9 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +/** + * Unit tests for {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig} + */ @RunWith(Parameterized.class) public class AutoRepairConfigTest extends CQLTester { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java index f7ca33077661..ccfbbba57b4e 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java @@ -34,6 +34,9 @@ import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableMetadata; +/** + * Unit tests for {@link org.apache.cassandra.schema.SystemDistributedKeyspace} + */ public class AutoRepairKeyspaceTest { private static final Set tables = ImmutableSet.of( diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 1f169ef80685..aa65fca85f43 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -85,6 +85,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +/** + * Unit tests for {@link org.apache.cassandra.repair.autorepair.AutoRepair} + */ @RunWith(Parameterized.class) public class AutoRepairParameterizedTest extends CQLTester { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java index 15d62c536a06..97e80364eed7 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java @@ -26,6 +26,9 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +/** + * Unit tests for {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType} + */ public class AutoRepairStateFactoryTest { @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java index 63e1ee4aaa0d..b4bb20c7eafd 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -49,6 +49,9 @@ import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; +/** + * Unit tests for {@link org.apache.cassandra.repair.autorepair.AutoRepairState} + */ @RunWith(Parameterized.class) public class AutoRepairStateTest extends CQLTester { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index 348e5d89a565..27a6da7cc478 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -41,6 +41,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +/** + * Unit tests for {@link org.apache.cassandra.repair.autorepair.AutoRepair} + */ public class AutoRepairTest extends CQLTester { @BeforeClass diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index 9fd588b2ce79..834357598a6b 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -68,6 +68,9 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.when; +/** + * Unit tests for {@link org.apache.cassandra.repair.autorepair.AutoRepairUtils} + */ public class AutoRepairUtilsTest extends CQLTester { static RepairType repairType = RepairType.INCREMENTAL; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java index e743948d1f23..dc4ca6c11ace 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java @@ -48,6 +48,9 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +/** + * Unit tests for {@link org.apache.cassandra.repair.autorepair.FixedSplitTokenRangeSplitter} + */ @RunWith(Parameterized.class) public class FixedSplitTokenRangeSplitterTest extends CQLTester { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java index 7b95acb879de..3c4efc25ebab 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java @@ -30,6 +30,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +/** + * Unit tests for {@link org.apache.cassandra.repair.autorepair.PrioritizedRepairPlan} + */ public class PrioritizedRepairPlanTest extends CQLTester { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index e15ce7a0b2aa..468d622c11e4 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -31,6 +31,8 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.apache.cassandra.config.DataStorageSpec.LongMebibytesBound; import org.apache.cassandra.config.DatabaseDescriptor; @@ -53,6 +55,10 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +/** + * Unit tests for {@link org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter} + */ +@RunWith(Parameterized.class) public class RepairTokenRangeSplitterTest extends CQLTester { private RepairTokenRangeSplitter repairRangeSplitter; diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index 35239d3b0bb0..a7d3a8a7bf9b 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -52,6 +52,9 @@ import static org.junit.Assert.assertEquals; import static org.assertj.core.api.Assertions.assertThat; +/** + * Unit tests for {@link org.apache.cassandra.service.AutoRepairService} + */ @RunWith(Suite.class) @Suite.SuiteClasses({ AutoRepairServiceTest.BasicTests.class, AutoRepairServiceTest.SetterTests.class }) public class AutoRepairServiceTest diff --git a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java index 1c7b04d50bcb..82293581d807 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/AutoRepairStatusTest.java @@ -40,6 +40,9 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.when; +/** + * Unit tests for {@link org.apache.cassandra.tools.nodetool.AutoRepairStatus} + */ @RunWith(Parameterized.class) public class AutoRepairStatusTest { @@ -94,7 +97,7 @@ public void testExecuteWithNoNodes() @Test public void testExecute() { - when(probe.getOnGoingRepairHostIds(repairType.name())).thenReturn(ImmutableSet.of("host1", "host2", "host3", "host4")); + when(probe.getAutoRepairOnGoingRepairHostIds(repairType.name())).thenReturn(ImmutableSet.of("host1", "host2", "host3", "host4")); cmd.repairType = repairType.name(); cmd.execute(probe); diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java index bc168c0752d7..edd57447ad13 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java @@ -39,6 +39,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +/** + * Unit tests for {@link org.apache.cassandra.tools.nodetool.SSTableRepairedSetTest} + */ public class SSTableRepairedSetTest { @Mock @@ -58,8 +61,8 @@ public void setUp() { public void testNoKeyspace() { when(probe.getNonLocalStrategyKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("ks1", "ks2"))); when(probe.getKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("ks1", "ks2"))); - when(probe.getTablesForKeyspace("ks1")).thenReturn(new ArrayList<>(Arrays.asList("table1", "table2"))); - when(probe.getTablesForKeyspace("ks2")).thenReturn(new ArrayList<>(Arrays.asList("table3", "table4"))); + when(probe.getAutoRepairTablesForKeyspace("ks1")).thenReturn(new ArrayList<>(Arrays.asList("table1", "table2"))); + when(probe.getAutoRepairTablesForKeyspace("ks2")).thenReturn(new ArrayList<>(Arrays.asList("table3", "table4"))); cmd.isRepaired = true; cmd.reallySet = true; diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index b0b499f67a08..89df6ad93c97 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -45,6 +45,9 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +/** + * Unit tests for {@link org.apache.cassandra.tools.nodetool.SetAutoRepairConfig} + */ @RunWith(Suite.class) @Suite.SuiteClasses({ SetAutoRepairConfigTest.NoParamTests.class, SetAutoRepairConfigTest.RepairTypeParamTests.class, SetAutoRepairConfigTest.RepairTypeAndArgsParamsTests.class }) @@ -129,13 +132,13 @@ public void testStartScheduler() cmd.execute(probe); - verify(probe, times(0)).startScheduler(); + verify(probe, times(0)).startAutoRepairScheduler(); cmd.args = ImmutableList.of("start_scheduler", "true"); cmd.execute(probe); - verify(probe, times(1)).startScheduler(); + verify(probe, times(1)).startAutoRepairScheduler(); } } @@ -186,7 +189,7 @@ public void testRepairSchedulingDisabled() cmd.execute(probe); verify(out, times(1)).println("Auto-repair is not enabled"); - verify(probe, times(0)).setRepairThreads(repairType.name(), 1); + verify(probe, times(0)).setAutoRepairThreads(repairType.name(), 1); } @Test @@ -197,7 +200,7 @@ public void testRepairTypeDisabled() cmd.execute(probe); - verify(probe, times(1)).setRepairThreads(repairType.name(), 1); + verify(probe, times(1)).setAutoRepairThreads(repairType.name(), 1); } @@ -218,7 +221,7 @@ public void testV2FlagMissing() // expected } - verify(probe, times(0)).setRepairThreads(repairType.name(), 0); + verify(probe, times(0)).setAutoRepairThreads(repairType.name(), 0); } @Test(expected = IllegalArgumentException.class) @@ -239,7 +242,7 @@ public void testPriorityHosts() cmd.execute(probe); - verify(probe, times(1)).setRepairPriorityForHosts(repairType.name(), commaSeparatedHostSet); + verify(probe, times(1)).setAutoRepairPriorityForHosts(repairType.name(), commaSeparatedHostSet); } @Test @@ -251,7 +254,7 @@ public void testForceRepairHosts() cmd.execute(probe); - verify(probe, times(1)).setForceRepairForHosts(repairType.name(), commaSeparatedHostSet); + verify(probe, times(1)).setAutoRepairForceRepairForHosts(repairType.name(), commaSeparatedHostSet); } } @@ -275,14 +278,14 @@ public static Collection testCases() { return Stream.of( forEachRepairType("enabled", "true", (type) -> verify(probe, times(1)).setAutoRepairEnabled(type.name(), true)), - forEachRepairType("number_of_repair_threads", "1", (type) -> verify(probe, times(1)).setRepairThreads(type.name(), 1)), - forEachRepairType("min_repair_interval", "3h", (type) -> verify(probe, times(1)).setRepairMinInterval(type.name(), "3h")), - forEachRepairType("sstable_upper_threshold", "4", (type) -> verify(probe, times(1)).setRepairSSTableCountHigherThreshold(type.name(), 4)), + forEachRepairType("number_of_repair_threads", "1", (type) -> verify(probe, times(1)).setAutoRepairThreads(type.name(), 1)), + forEachRepairType("min_repair_interval", "3h", (type) -> verify(probe, times(1)).setAutoRepairMinInterval(type.name(), "3h")), + forEachRepairType("sstable_upper_threshold", "4", (type) -> verify(probe, times(1)).setAutoRepairSSTableCountHigherThreshold(type.name(), 4)), forEachRepairType("table_max_repair_time", "5s", (type) -> verify(probe, times(1)).setAutoRepairTableMaxRepairTime(type.name(), "5s")), - forEachRepairType("repair_primary_token_range_only", "true", (type) -> verify(probe, times(1)).setPrimaryTokenRangeOnly(type.name(), true)), - forEachRepairType("parallel_repair_count", "6", (type) -> verify(probe, times(1)).setParallelRepairCount(type.name(), 6)), - forEachRepairType("parallel_repair_percentage", "7", (type) -> verify(probe, times(1)).setParallelRepairPercentage(type.name(), 7)), - forEachRepairType("materialized_view_repair_enabled", "true", (type) -> verify(probe, times(1)).setMaterializedViewRepairEnabled(type.name(), true)), + forEachRepairType("repair_primary_token_range_only", "true", (type) -> verify(probe, times(1)).setAutoRepairPrimaryTokenRangeOnly(type.name(), true)), + forEachRepairType("parallel_repair_count", "6", (type) -> verify(probe, times(1)).setAutoRepairParallelRepairCount(type.name(), 6)), + forEachRepairType("parallel_repair_percentage", "7", (type) -> verify(probe, times(1)).setAutoRepairParallelRepairPercentage(type.name(), 7)), + forEachRepairType("materialized_view_repair_enabled", "true", (type) -> verify(probe, times(1)).setAutoRepairMaterializedViewRepairEnabled(type.name(), true)), forEachRepairType("ignore_dcs", "dc1,dc2", (type) -> verify(probe, times(1)).setAutoRepairIgnoreDCs(type.name(), ImmutableSet.of("dc1", "dc2"))), forEachRepairType("token_range_splitter.max_bytes_per_schedule", "500GiB", (type) -> verify(probe, times(1)).setAutoRepairTokenRangeSplitterParameter(type.name(), "max_bytes_per_schedule", "500GiB")) ).flatMap(Function.identity()).collect(Collectors.toList()); From 9051235cacc6861bc9551d2100aeeeaf8d1a2b73 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 11 Feb 2025 15:34:20 -0800 Subject: [PATCH 127/199] Fix review comments from Andy --- src/java/org/apache/cassandra/config/ParameterizedClass.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/java/org/apache/cassandra/config/ParameterizedClass.java b/src/java/org/apache/cassandra/config/ParameterizedClass.java index f14ee1e3ed28..9b001786e35b 100644 --- a/src/java/org/apache/cassandra/config/ParameterizedClass.java +++ b/src/java/org/apache/cassandra/config/ParameterizedClass.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.config; -import java.io.Serializable; import java.util.List; import java.util.Map; From d84c59ab8d5c64420b42fe6c9eb6b502cd78d485 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 11 Feb 2025 17:04:09 -0800 Subject: [PATCH 128/199] Additional review comments from Jaydeepkumar --- .../cassandra/metrics/AutoRepairMetrics.java | 21 +++++++++---------- .../repair/autorepair/AutoRepairConfig.java | 4 ++-- .../repair/autorepair/AutoRepairState.java | 4 ---- .../repair/autorepair/AutoRepairUtils.java | 11 ---------- .../FixedSplitTokenRangeSplitter.java | 3 --- .../autorepair/RepairAssignmentIterator.java | 1 - .../cassandra/schema/AutoRepairParams.java | 3 +-- .../cassandra/schema/SchemaKeyspace.java | 1 - .../schema/SystemDistributedKeyspace.java | 1 - .../apache/cassandra/schema/TableParams.java | 3 +-- .../service/AutoRepairServiceMBean.java | 3 --- .../org/apache/cassandra/tools/NodeProbe.java | 1 - .../test/repair/AutoRepairSchedulerTest.java | 1 - .../config/YamlConfigurationLoaderTest.java | 10 ++++----- .../autorepair/AutoRepairConfigTest.java | 2 +- .../autorepair/AutoRepairKeyspaceTest.java | 1 - .../AutoRepairParameterizedTest.java | 1 - .../autorepair/AutoRepairUtilsTest.java | 4 +--- .../autorepair/PrioritizedRepairPlanTest.java | 2 -- .../RepairTokenRangeSplitterTest.java | 1 - .../nodetool/SetAutoRepairConfigTest.java | 1 - 21 files changed, 21 insertions(+), 58 deletions(-) diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java index c9b15979c245..b83d2d8a512c 100644 --- a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java @@ -31,20 +31,19 @@ */ public class AutoRepairMetrics { - public Gauge repairsInProgress; - public Gauge nodeRepairTimeInSec; - public Gauge clusterRepairTimeInSec; - public Gauge longestUnrepairedSec; - public Gauge succeededTokenRangesCount; - public Gauge failedTokenRangesCount; - public Gauge skippedTokenRangesCount; - public Gauge skippedTablesCount; + public final Gauge repairsInProgress; + public final Gauge nodeRepairTimeInSec; + public final Gauge clusterRepairTimeInSec; + public final Gauge longestUnrepairedSec; + public final Gauge succeededTokenRangesCount; + public final Gauge failedTokenRangesCount; + public final Gauge skippedTokenRangesCount; + public final Gauge skippedTablesCount; + public final Gauge totalMVTablesConsideredForRepair; + public final Gauge totalDisabledRepairTables; public Counter repairTurnMyTurn; public Counter repairTurnMyTurnDueToPriority; public Counter repairTurnMyTurnForceRepair; - public Gauge totalMVTablesConsideredForRepair; - public Gauge totalDisabledRepairTables; - public AutoRepairMetrics(RepairType repairType) { AutoRepairMetricsFactory factory = new AutoRepairMetricsFactory(repairType); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 021eff330efb..6ebbc6e086fa 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -452,8 +452,8 @@ protected static Options getDefaultOptions() opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h"); opts.materialized_view_repair_enabled = false; opts.token_range_splitter = new ParameterizedClass(DEFAULT_SPLITTER.getName(), Collections.emptyMap()); - opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("5m"); // 5 minutes - opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h"); // 3 hours + opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("5m"); + opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h"); return opts; } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index 87ed93881bcd..6794c1ec2162 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -83,17 +83,14 @@ public abstract class AutoRepairState implements ProgressListener protected int totalMVTablesConsideredForRepair = 0; @VisibleForTesting protected int totalDisabledTablesRepairCount = 0; - @VisibleForTesting protected int failedTokenRangesCount = 0; @VisibleForTesting protected int succeededTokenRangesCount = 0; @VisibleForTesting protected int skippedTokenRangesCount = 0; - @VisibleForTesting protected int skippedTablesCount = 0; - @VisibleForTesting protected AutoRepairHistory longestUnrepairedNode; @VisibleForTesting @@ -114,7 +111,6 @@ protected RepairRunnable getRepairRunnable(String keyspace, RepairOption options { RepairRunnable task = new RepairRunnable(StorageService.instance, StorageService.nextRepairCommand.incrementAndGet(), options, keyspace); - task.addProgressListener(this); return task; diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index b8cd8e0a940f..fe4a49e9b89f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -132,7 +132,6 @@ public class AutoRepairUtils COL_REPAIR_TYPE, COL_HOST_ID); final static String RECORD_FINISH_REPAIR_HISTORY = String.format( - "UPDATE %s.%s SET %s= ?, %s=false WHERE %s = ? AND %s = ?" , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_FINISH_TS, COL_FORCE_REPAIR, COL_REPAIR_TYPE, COL_HOST_ID); @@ -179,10 +178,8 @@ public static void setup() .forInternalCalls()); selectStatementRepairPriority = (SelectStatement) QueryProcessor.getStatement(SELECT_REPAIR_PRIORITY, ClientState .forInternalCalls()); - selectLastRepairTimeForNode = (SelectStatement) QueryProcessor.getStatement(SELECT_LAST_REPAIR_TIME_FOR_NODE, ClientState .forInternalCalls()); - delStatementPriorityStatus = (ModificationStatement) QueryProcessor.getStatement(DEL_REPAIR_PRIORITY, ClientState .forInternalCalls()); addPriorityHost = (ModificationStatement) QueryProcessor.getStatement(ADD_PRIORITY_HOST, ClientState @@ -386,14 +383,11 @@ public static long getLastRepairTimeForNode(RepairType repairType, UUID hostId) ByteBufferUtil.bytes(repairType.toString()), ByteBufferUtil.bytes(hostId))), Dispatcher.RequestTime.forImmediateExecution()); - UntypedResultSet repairTime = UntypedResultSet.create(rows.result); - if (repairTime.isEmpty()) { return 0; } - return repairTime.one().getLong(COL_REPAIR_FINISH_TS); } @@ -558,11 +552,9 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) int parallelRepairNumber = getMaxNumberOfNodeRunAutoRepair(repairType, autoRepairHistories == null ? 0 : autoRepairHistories.size()); logger.info("Will run repairs concurrently on {} node(s)", parallelRepairNumber); - if (currentRepairStatus == null || parallelRepairNumber > currentRepairStatus.hostIdsWithOnGoingRepair.size()) { // more repairs can be run, I might be the new one - if (autoRepairHistories != null) { logger.info("Auto repair history table has {} records", autoRepairHistories.size()); @@ -604,13 +596,11 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) "First node in priority list is {}", StorageService.instance.getTokenMetadata().getEndpointForHostId(priorityHostId)); return NOT_MY_TURN; } - if (myId.equals(priorityHostId)) { //I have a priority for repair hence its my turn now return MY_TURN_DUE_TO_PRIORITY; } - // get the longest unrepaired node from the nodes which are not running repair AutoRepairHistory defaultNodeToBeRepaired = getHostWithLongestUnrepairTime(currentRepairStatus.historiesWithoutOnGoingRepair); //check who is next, which is helpful for debugging @@ -811,7 +801,6 @@ public static boolean shouldConsiderKeyspace(Keyspace ks) return repair; } - public static boolean tableMaxRepairTimeExceeded(RepairType repairType, long startTime) { long tableRepairTimeSoFar = TimeUnit.MILLISECONDS.toSeconds diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index f19006717663..bdcb355e7098 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -75,7 +75,6 @@ public Iterator getRepairAssignments(boolean primaryR { return new RepairAssignmentIterator(repairPlans) { - @Override protected KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repairPlan) { @@ -125,7 +124,6 @@ private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(boolean primar } } } - return new KeyspaceRepairAssignments(priority, keyspaceName, repairAssignments); } @@ -136,7 +134,6 @@ public void setParameter(String key, String value) { throw new IllegalArgumentException("Unexpected parameter '" + key + "', must be " + NUMBER_OF_SUBRANGES); } - logger.info("Setting {} to {} for repair type {}", key, value, repairType); this.numberOfSubranges = Integer.parseInt(value); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java index fc5573a48944..44d9f5ef5e55 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java @@ -51,7 +51,6 @@ private synchronized Iterator currentIterator() currentIterator = currentPlan.getKeyspaceRepairPlans().iterator(); } } - return currentIterator; } diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index 34e4f9f5b97f..8e82df1cbd52 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -50,7 +50,7 @@ public String toString() } } - private ImmutableMap options; + private final ImmutableMap options; public static final Map DEFAULT_OPTIONS = ImmutableMap.of( Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(true), @@ -150,7 +150,6 @@ public static boolean isValidInt(String value) return StringUtils.isNumeric(value); } - public Map options() { return options; diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index 8c45e74f1ccb..3f032ccaaf78 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -557,7 +557,6 @@ private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBui .add("extensions", params.extensions) .add("auto_repair", params.autoRepair.asMap()); - // Only add CDC-enabled flag to schema if it's enabled on the node. This is to work around RTE's post-8099 if a 3.8+ // node sends table schema to a < 3.8 versioned node with an unknown column. if (DatabaseDescriptor.isCDCEnabled()) diff --git a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java index 02cb9f228e62..94e33d2b69f9 100644 --- a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java @@ -185,7 +185,6 @@ private SystemDistributedKeyspace() + "repair_priority set," + "PRIMARY KEY (repair_type))").build(); - private static TableMetadata.Builder parse(String table, String description, String cql) { return CreateTableStatement.parse(format(cql, table), SchemaConstants.DISTRIBUTED_KEYSPACE_NAME) diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index fa28ff726532..e2ec65294f0c 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -58,8 +58,7 @@ public enum Option CRC_CHECK_CHANCE, CDC, READ_REPAIR, - AUTO_REPAIR, - ; + AUTO_REPAIR; @Override public String toString() diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index 40948f3711a5..f3ec919c8f1b 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -25,9 +25,6 @@ */ public interface AutoRepairServiceMBean { - /** - * Enable or disable auto-repair for a given repair type - */ public void setAutoRepairEnabled(String repairType, boolean enabled); public void setRepairThreads(String repairType, int repairThreads); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index b427a415eb9f..b887fc97fe73 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -157,7 +157,6 @@ public class NodeProbe implements AutoCloseable protected PermissionsCacheMBean pcProxy; protected RolesCacheMBean rcProxy; protected AutoRepairServiceMBean autoRepairProxy; - protected Output output; private boolean failed; diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index 175a32fda2bc..dc24e5847b99 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -47,7 +47,6 @@ */ public class AutoRepairSchedulerTest extends TestBaseImpl { - private static Cluster cluster; static SimpleDateFormat sdf; diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java index ead42a9f039b..07081f370acf 100644 --- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java +++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java @@ -228,11 +228,11 @@ public void fromMapTest() "optional", false, "enabled", true); Map autoRepairConfig = ImmutableMap.of("enabled", true, - "global_settings", ImmutableMap.of("number_of_repair_threads", - 1), - "repair_type_overrides", ImmutableMap.of( - "full", ImmutableMap.of("number_of_repair_threads", - 2))); + "global_settings", + ImmutableMap.of("number_of_repair_threads", 1), + "repair_type_overrides", + ImmutableMap.of("full", + ImmutableMap.of("number_of_repair_threads", 2))); Map map = new ImmutableMap.Builder() .put("storage_port", storagePort) .put("commitlog_sync", commitLogSync) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 29e564cc3e12..e8c60f2d9b50 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -52,7 +52,7 @@ public class AutoRepairConfigTest extends CQLTester { private AutoRepairConfig config; - private Set testSet = ImmutableSet.of("dc1"); + private final Set testSet = ImmutableSet.of("dc1"); @Parameterized.Parameter public AutoRepairConfig.RepairType repairType; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java index ccfbbba57b4e..f3c0702082cc 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairKeyspaceTest.java @@ -51,7 +51,6 @@ public static void setupDatabaseDescriptor() AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); } - @Test public void testEnsureAutoRepairTablesArePresent() { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index aa65fca85f43..d1c3adde8a49 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -758,7 +758,6 @@ public void testRepairThrowsForIRWithMVReplay() } } - @Test public void testRepairThrowsForIRWithCDCReplay() { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index 834357598a6b..0ff78b08479d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -73,7 +73,7 @@ */ public class AutoRepairUtilsTest extends CQLTester { - static RepairType repairType = RepairType.INCREMENTAL; + static final RepairType repairType = RepairType.INCREMENTAL; static UUID hostId; static InetAddressAndPort localEndpoint; @@ -148,7 +148,6 @@ public void testSetForceRepairNewNode() assertTrue(result.one().getBoolean(COL_FORCE_REPAIR)); } - @Test public void testClearDeleteHosts() { @@ -282,7 +281,6 @@ public void testGetMaxNumberOfNodeRunAutoRepairInGroup_0_group_size() assertEquals(2, count); } - @Test public void testGetMaxNumberOfNodeRunAutoRepairInGroup_percentage() { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java index 3c4efc25ebab..38f9c8538846 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java @@ -35,7 +35,6 @@ */ public class PrioritizedRepairPlanTest extends CQLTester { - @Test public void testBuildWithDifferentPriorities() { @@ -162,5 +161,4 @@ public void testBuildWithOneTable() assertEquals(5, prioritizedRepairPlans.get(0).getPriority()); assertEquals(table1, prioritizedRepairPlans.get(0).getKeyspaceRepairPlans().get(0).getTableNames().get(0)); } - } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index 468d622c11e4..01a81be45473 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -356,7 +356,6 @@ private SizeEstimate sizeEstimateByBytes(LongMebibytesBound sizeInRange, LongMeb return new SizeEstimate(RepairType.INCREMENTAL, KEYSPACE, "table1", FULL_RANGE, 1, sizeInRange.toBytes(), totalSize.toBytes()); } - private void insertAndFlushSingleTable() throws Throwable { execute("INSERT INTO %s (k, v) values (?, ?)", 1, 1); diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index 89df6ad93c97..8027eadb6232 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -203,7 +203,6 @@ public void testRepairTypeDisabled() verify(probe, times(1)).setAutoRepairThreads(repairType.name(), 1); } - @Test public void testV2FlagMissing() { From 3cb5e2fa9930166bfc6adc324ed409bc84751987 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 13 Feb 2025 11:32:43 -0800 Subject: [PATCH 129/199] Additional review comments from Andy From 1ed3f42a84635f0e67e5d6be331dad305d0ecbcf Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 13 Feb 2025 21:06:15 -0800 Subject: [PATCH 130/199] 4.1 only Remove Parameterized from RepairTokenRangeSplitterTest --- .../repair/autorepair/RepairTokenRangeSplitterTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index 01a81be45473..a427463631d7 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -31,8 +31,6 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; import org.apache.cassandra.config.DataStorageSpec.LongMebibytesBound; import org.apache.cassandra.config.DatabaseDescriptor; @@ -58,7 +56,6 @@ /** * Unit tests for {@link org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter} */ -@RunWith(Parameterized.class) public class RepairTokenRangeSplitterTest extends CQLTester { private RepairTokenRangeSplitter repairRangeSplitter; From ffa6961e149e46ee4fad706d62d0c7e09d4a4b36 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 13 Feb 2025 12:05:15 -0800 Subject: [PATCH 131/199] Additional review comments from Jaydeepkumar --- NEWS.txt | 9 +++++++++ .../config/YamlConfigurationLoaderTest.java | 2 +- .../autorepair/RepairTokenRangeSplitterTest.java | 15 ++++++++------- .../service/ActiveRepairServiceTest.java | 1 - 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/NEWS.txt b/NEWS.txt index 21bf7e76f3e9..d5e6bcb4ab70 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -51,6 +51,15 @@ restore snapshots created with the previous major version using the 'sstableloader' tool. You can upgrade the file format of your snapshots using the provided 'sstableupgrade' tool. +New features +------------ + [The following is a placeholder, to be revised asap] + - CEP-37 Auto Repair is a fully automated scheduler that provides repair orchestration within Apache Cassandra. This + significantly reduces operational overhead by eliminating the need for operators to deploy external tools to submit + and manage repairs. See + https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-37+Apache+Cassandra+Unified+Repair+Solution for more + details on the motivation and design + 4.1.3 ===== diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java index 07081f370acf..fdeeae56c262 100644 --- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java +++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java @@ -252,7 +252,7 @@ public void fromMapTest() assertEquals(true, config.client_encryption_options.enabled); // Check a nested object assertEquals(new DataStorageSpec.IntBytesBound("5B"), config.internode_socket_send_buffer_size); // Check names backward compatibility (CASSANDRA-17141 and CASSANDRA-15234) assertEquals(new DataStorageSpec.IntBytesBound("5B"), config.internode_socket_receive_buffer_size); // Check names backward compatibility (CASSANDRA-17141 and CASSANDRA-15234) - assertEquals(true, config.auto_repair.enabled); + assertTrue(config.auto_repair.enabled); assertEquals(new DurationSpec.IntSecondsBound("6h"), config.auto_repair.getAutoRepairTableMaxRepairTime(AutoRepairConfig.RepairType.INCREMENTAL)); config.auto_repair.setMaterializedViewRepairEnabled(AutoRepairConfig.RepairType.INCREMENTAL, false); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index a427463631d7..fe7b2815fe9d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -102,8 +102,8 @@ public void testSizePartitionCountSplit() throws Throwable Range tokenRange2 = range.next(); Assert.assertFalse(range.hasNext()); - try(Refs sstables1 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange1); - Refs sstables2 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange2)) + try (Refs sstables1 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange1); + Refs sstables2 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange2)) { SizeEstimate sizes1 = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange1, sstables1); SizeEstimate sizes2 = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange2, sstables2); @@ -306,23 +306,23 @@ public void testGetRepairAssignmentsSplitsBySubrangeSizeAndFilterLimitsByMaxByte FilteredRepairAssignments filteredRepairAssignments = repairRangeSplitter.filterRepairAssignments(0, KEYSPACE, assignments, 0); List finalRepairAssignments = filteredRepairAssignments.repairAssignments; assertEquals(2, finalRepairAssignments.size()); - assertEquals(expectedBytes*2, filteredRepairAssignments.newBytesSoFar); + assertEquals(expectedBytes * 2, filteredRepairAssignments.newBytesSoFar); } - @Test(expected=IllegalArgumentException.class) + @Test(expected = IllegalArgumentException.class) public void testSetParameterShouldNotAllowUnknownParameter() { repairRangeSplitter.setParameter("unknown", "x"); } - @Test(expected=IllegalArgumentException.class) + @Test(expected = IllegalArgumentException.class) public void testSetParameterShouldNotAllowSettingBytesPerAssignmentGreaterThanMaxBytesPerSchedule() { repairRangeSplitter.setParameter(MAX_BYTES_PER_SCHEDULE, "500GiB"); repairRangeSplitter.setParameter(BYTES_PER_ASSIGNMENT, "600GiB"); } - @Test(expected=IllegalArgumentException.class) + @Test(expected = IllegalArgumentException.class) public void testSetParameterShouldNotAllowSettingMaxBytesPerScheduleLessThanBytesPerAssignment() { repairRangeSplitter.setParameter(BYTES_PER_ASSIGNMENT, "100MiB"); @@ -362,7 +362,8 @@ private void insertAndFlushSingleTable() throws Throwable private List createAndInsertTables(int count) throws Throwable { List tableNames = new ArrayList<>(); - for (int i = 0; i < count; i++) { + for (int i = 0; i < count; i++) + { String tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); tableNames.add(tableName); insertAndFlushTable(tableName); diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java index 5a158b61b455..669fd335d50c 100644 --- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java @@ -252,7 +252,6 @@ public void testGetNeighborsSpecifiedHostsWithNoLocalHost() throws Throwable ActiveRepairService.getNeighbors(KEYSPACE5, ranges, ranges.iterator().next(), null, hosts); } - @Test public void testParentRepairStatus() throws Throwable { From b314fbddfacd00d1a17cfd19273cb8495627f6d1 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 14 Feb 2025 12:37:37 -0800 Subject: [PATCH 132/199] Incorporate NEWS.txt review comment --- NEWS.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.txt b/NEWS.txt index d5e6bcb4ab70..234ba4ba169c 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -58,7 +58,7 @@ New features significantly reduces operational overhead by eliminating the need for operators to deploy external tools to submit and manage repairs. See https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-37+Apache+Cassandra+Unified+Repair+Solution for more - details on the motivation and design + details on the motivation and design. 4.1.3 ===== From ab68d11965a742cc722419328fa9973eb3995ef9 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Thu, 30 Jan 2025 11:41:56 -0800 Subject: [PATCH 133/199] Fix race condition in auto-repair scheduler --- CHANGES.txt | 1 + .../repair/autorepair/AutoRepair.java | 74 ++++++- .../repair/autorepair/AutoRepairState.java | 61 +----- .../AutoRepairParameterizedTest.java | 207 ++++++++++++++---- .../autorepair/AutoRepairStateTest.java | 101 +-------- 5 files changed, 246 insertions(+), 198 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index c1b281600896..3ac558b08e9b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1.6 + * Fix race condition in auto-repair scheduler (CASSANDRA-20265) * Implement minimum repair task duration setting for auto-repair scheduler (CASSANDRA-20160) * Implement preview_repaired auto-repair type (CASSANDRA-20046) * Refresh stale paxos commit (CASSANDRA-19617) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 798578fbe34e..1a23eff5662d 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -18,6 +18,7 @@ package org.apache.cassandra.repair.autorepair; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.EnumMap; import java.util.HashSet; @@ -56,12 +57,17 @@ import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn; +import org.apache.cassandra.utils.concurrent.Condition; import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.progress.ProgressEvent; +import org.apache.cassandra.utils.progress.ProgressEventType; +import org.apache.cassandra.utils.progress.ProgressListener; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_DUE_TO_PRIORITY; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_FORCE_REPAIR; +import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; /** * AutoRepair scheduler responsible for running different types of repairs. @@ -69,12 +75,11 @@ public class AutoRepair { private static final Logger logger = LoggerFactory.getLogger(AutoRepair.class); + private static final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); @VisibleForTesting protected static Supplier timeFunc = Clock.Global::currentTimeMillis; - public static AutoRepair instance = new AutoRepair(); - // Sleep for 5 seconds if repair finishes quickly to flush JMX metrics; it happens only for Cassandra nodes with tiny amount of data. public static DurationSpec.IntSecondsBound SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("5s"); @@ -95,6 +100,7 @@ public class AutoRepair protected static BiConsumer sleepFunc = Uninterruptibles::sleepUninterruptibly; private boolean isSetupDone = false; + public static AutoRepair instance = new AutoRepair(); @VisibleForTesting protected AutoRepair() @@ -304,6 +310,7 @@ private void repairKeyspace(AutoRepairConfig.RepairType repairType, boolean prim if ((totalProcessedAssignments % config.getRepairThreads(repairType) == 0) || (totalProcessedAssignments == totalRepairAssignments)) { + boolean success = false; int retryCount = 0; Future f = null; while (retryCount <= config.getRepairMaxRetries()) @@ -311,19 +318,21 @@ private void repairKeyspace(AutoRepairConfig.RepairType repairType, boolean prim RepairRunnable task = repairState.getRepairRunnable(keyspaceName, Lists.newArrayList(curRepairAssignment.getTableNames()), ranges, primaryRangeOnly); - repairState.resetWaitCondition(); + RepairProgressListener listener = new RepairProgressListener(repairType); + task.addProgressListener(listener); f = repairRunnableExecutors.get(repairType).submit(task); try { long jobStartTime = timeFunc.get(); - repairState.waitForRepairToComplete(config.getRepairSessionTimeout(repairType)); + listener.await(); + success = listener.isSuccess(); soakAfterRepair(jobStartTime, config.getRepairTaskMinDuration().toMilliseconds()); } catch (InterruptedException e) { logger.error("Exception in cond await:", e); } - if (repairState.isSuccess()) + if (success) { break; } @@ -339,7 +348,7 @@ else if (retryCount < config.getRepairMaxRetries()) retryCount++; } //check repair status - if (repairState.isSuccess()) + if (success) { logger.info("Repair completed for range {}-{} for {} tables {}, total assignments: {}," + "processed assignments: {}", tokenRange.left, tokenRange.right, @@ -500,4 +509,57 @@ static class CollectedRepairStats int skippedTokenRanges = 0; int skippedTables = 0; } + + @VisibleForTesting + protected static class RepairProgressListener implements ProgressListener + { + private final AutoRepairConfig.RepairType repairType; + @VisibleForTesting + protected boolean success; + @VisibleForTesting + protected final Condition condition = newOneTimeCondition(); + + public RepairProgressListener(AutoRepairConfig.RepairType repairType) + { + this.repairType = repairType; + } + + public void await() throws InterruptedException + { + //if for some reason we don't hear back on repair progress for sometime + if (!condition.await(12, TimeUnit.HOURS)) + { + success = false; + } + } + + public boolean isSuccess() + { + return success; + } + + @Override + public void progress(String tag, ProgressEvent event) + { + ProgressEventType type = event.getType(); + String message = String.format("[%s] %s", format.format(System.currentTimeMillis()), event.getMessage()); + if (type == ProgressEventType.ERROR) + { + logger.error("Repair failure for repair {}: {}", repairType.toString(), message); + success = false; + condition.signalAll(); + } + if (type == ProgressEventType.PROGRESS) + { + message = message + " (progress: " + (int) event.getProgressPercentage() + "%)"; + logger.debug("Repair progress for repair {}: {}", repairType.toString(), message); + } + if (type == ProgressEventType.COMPLETE) + { + logger.debug("Repair completed for repair {}: {}", repairType.toString(), message); + success = true; + condition.signalAll(); + } + } + } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index 6794c1ec2162..6d14e6ac74be 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -20,7 +20,6 @@ import com.google.common.annotations.VisibleForTesting; -import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.view.TableViews; @@ -37,10 +36,6 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.Clock; -import org.apache.cassandra.utils.concurrent.Condition; -import org.apache.cassandra.utils.progress.ProgressEvent; -import org.apache.cassandra.utils.progress.ProgressEventType; -import org.apache.cassandra.utils.progress.ProgressListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +47,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +<<<<<<< HEAD import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; @@ -59,6 +55,10 @@ * AutoRepairState represents the state of automated repair for a given repair type. */ public abstract class AutoRepairState implements ProgressListener +======= +// AutoRepairState represents the state of automated repair for a given repair type. +public abstract class AutoRepairState +>>>>>>> 0542e95397 (Fix race condition in auto-repair scheduler) { protected static final Logger logger = LoggerFactory.getLogger(AutoRepairState.class); private final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); @@ -93,10 +93,6 @@ public abstract class AutoRepairState implements ProgressListener protected int skippedTablesCount = 0; @VisibleForTesting protected AutoRepairHistory longestUnrepairedNode; - @VisibleForTesting - protected Condition condition = newOneTimeCondition(); - @VisibleForTesting - protected boolean success = true; protected final AutoRepairMetrics metrics; protected AutoRepairState(RepairType repairType) @@ -109,43 +105,8 @@ protected AutoRepairState(RepairType repairType) protected RepairRunnable getRepairRunnable(String keyspace, RepairOption options) { - RepairRunnable task = new RepairRunnable(StorageService.instance, StorageService.nextRepairCommand.incrementAndGet(), + return new RepairRunnable(StorageService.instance, StorageService.nextRepairCommand.incrementAndGet(), options, keyspace); - task.addProgressListener(this); - - return task; - } - - @Override - public void progress(String tag, ProgressEvent event) - { - ProgressEventType type = event.getType(); - String message = String.format("[%s] %s", format.format(currentTimeMillis()), event.getMessage()); - if (type == ProgressEventType.ERROR) - { - logger.error("Repair failure for {} repair: {}", repairType.toString(), message); - success = false; - condition.signalAll(); - } - if (type == ProgressEventType.PROGRESS) - { - message = message + " (progress: " + (int) event.getProgressPercentage() + "%)"; - logger.debug("Repair progress for {} repair: {}", repairType.toString(), message); - } - if (type == ProgressEventType.COMPLETE) - { - success = true; - condition.signalAll(); - } - } - - public void waitForRepairToComplete(DurationSpec.IntSecondsBound repairSessionTimeout) throws InterruptedException - { - //if for some reason we don't hear back on repair progress for sometime - if (!condition.await(repairSessionTimeout.toSeconds(), TimeUnit.SECONDS)) - { - success = false; - } } public long getLastRepairTime() @@ -272,11 +233,6 @@ public int getSkippedTablesCount() return skippedTablesCount; } - public boolean isSuccess() - { - return success; - } - public void recordTurn(AutoRepairUtils.RepairTurn turn) { metrics.recordTurn(turn); @@ -291,11 +247,6 @@ public int getTotalDisabledTablesRepairCount() { return totalDisabledTablesRepairCount; } - - public void resetWaitCondition() - { - condition = newOneTimeCondition(); - } } class PreviewRepairedState extends AutoRepairState diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index d1c3adde8a49..05551bcf007d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -24,8 +24,10 @@ import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; @@ -34,7 +36,9 @@ import org.apache.cassandra.cql3.statements.schema.TableAttributes; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.LocalStrategy; +import org.apache.cassandra.repair.RepairParallelism; import org.apache.cassandra.repair.RepairRunnable; +import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.schema.AutoRepairParams; import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableParams; @@ -65,10 +69,14 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.progress.ProgressEvent; +import org.apache.cassandra.utils.progress.ProgressEventType; +import org.apache.cassandra.utils.progress.ProgressListener; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import org.mockito.invocation.InvocationOnMock; import static org.apache.cassandra.Util.setAutoRepairEnabled; import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; @@ -82,6 +90,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -127,7 +137,6 @@ public static void setupClass() throws Exception requireNetwork(); AutoRepairUtils.setup(); - cfm = TableMetadata.builder(KEYSPACE, TABLE) .addPartitionKeyColumn("k", UTF8Type.instance) .addStaticColumn("s", UTF8Type.instance) @@ -276,7 +285,7 @@ public void testRepairAsync() AutoRepair.instance.repairAsync(repairType); - verify(mockExecutor, Mockito.times(1)).submit(any(Runnable.class)); + verify(mockExecutor, times(1)).submit(any(Runnable.class)); } @Test @@ -532,6 +541,10 @@ public void testMetrics() AutoRepair.instance.repairStates.put(repairType, autoRepairState); when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())) .thenReturn(repairRunnable); + doAnswer(invocation -> { + invocation.getArgument(0, ProgressListener.class).progress("test", new ProgressEvent(ProgressEventType.COMPLETE, 0, 0)); + return null; + }).when(repairRunnable).addProgressListener(any()); when(autoRepairState.getFailedTokenRangesCount()).thenReturn(10); when(autoRepairState.getSucceededTokenRangesCount()).thenReturn(11); when(autoRepairState.getLongestUnrepairedSec()).thenReturn(10); @@ -544,29 +557,30 @@ public void testMetrics() } @Test - public void testRepairWaitsForRepairToFinishBeforeSchedullingNewSession() throws Exception + public void testRepairWaitsForRepairToFinishBeforeSchedullingNewSession() { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - config.setMaterializedViewRepairEnabled(repairType, false); - config.setRepairRetryBackoff("0s"); - when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())) - .thenReturn(repairRunnable); + config.setMVRepairEnabled(repairType, false); AutoRepair.instance.repairStates.put(repairType, autoRepairState); when(autoRepairState.getLastRepairTime()).thenReturn((long) 0); - AtomicInteger resetWaitConditionCalls = new AtomicInteger(); - AtomicInteger waitForRepairCompletedCalls = new AtomicInteger(); + AtomicInteger getRepairRunnableCalls = new AtomicInteger(); + AtomicReference prevListener = new AtomicReference<>(); doAnswer(invocation -> { - resetWaitConditionCalls.getAndIncrement(); - assertEquals("waitForRepairToComplete was called before resetWaitCondition", - resetWaitConditionCalls.get(), waitForRepairCompletedCalls.get() + 1); - return null; - }).when(autoRepairState).resetWaitCondition(); + if (getRepairRunnableCalls.getAndIncrement() > 0) + { + // progress listener from previous repair should be signalled before starting new repair + assertTrue(prevListener.get().condition.isSignalled()); + } + getRepairRunnableCalls.incrementAndGet(); + return repairRunnable; + }).when(autoRepairState).getRepairRunnable(any(), any(), any(), anyBoolean()); doAnswer(invocation -> { - waitForRepairCompletedCalls.getAndIncrement(); - assertEquals("resetWaitCondition was not called before waitForRepairToComplete", - resetWaitConditionCalls.get(), waitForRepairCompletedCalls.get()); + // sending out a COMPLETE event with a 10ms delay + Executors.newScheduledThreadPool(1).schedule(() -> { + invocation.getArgument(0, AutoRepair.RepairProgressListener.class).progress("test", new ProgressEvent(ProgressEventType.COMPLETE, 0, 0)); + }, 10, TimeUnit.MILLISECONDS); return null; - }).when(autoRepairState).waitForRepairToComplete(config.getRepairSessionTimeout(repairType)); + }).when(repairRunnable).addProgressListener(any()); AutoRepair.instance.repair(repairType); AutoRepair.instance.repair(repairType); @@ -662,7 +676,10 @@ public void testRepairTakesLastRepairTimeFromDB() public void testRepairMaxRetries() { when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())).thenReturn(repairRunnable); - when(autoRepairState.isSuccess()).thenReturn(false); + doAnswer(invocation -> { + invocation.getArgument(0, ProgressListener.class).progress("test", new ProgressEvent(ProgressEventType.ERROR, 0, 0)); + return null; + }).when(repairRunnable).addProgressListener(any()); AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); AtomicInteger sleepCalls = new AtomicInteger(); AutoRepair.sleepFunc = (Long duration, TimeUnit unit) -> { @@ -676,10 +693,10 @@ public void testRepairMaxRetries() AutoRepair.instance.repair(repairType); // Expect configured retries for each table expected to be repaired - assertEquals(config.getRepairMaxRetries() * expectedRepairAssignments, sleepCalls.get()); - verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(0); - verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); - verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(expectedRepairAssignments); + assertEquals(config.getRepairMaxRetries()*expectedTablesGoingThroughRepair, sleepCalls.get()); + verify(autoRepairState, times(1)).setSucceededTokenRangesCount(0); + verify(autoRepairState, times(1)).setSkippedTokenRangesCount(0); + verify(autoRepairState, times(1)).setFailedTokenRangesCount(expectedTablesGoingThroughRepair); } @Test @@ -694,20 +711,33 @@ public void testRepairSuccessAfterRetry() assertEquals(TimeUnit.SECONDS, unit); assertEquals(config.getRepairRetryBackoff().toSeconds(), (long) duration); }; - when(autoRepairState.isSuccess()).then((invocationOnMock) -> { - if (sleepCalls.get() == 0) { - return false; + doAnswer(invocation -> { + if (sleepCalls.get() == 0) + { + invocation.getArgument(0, ProgressListener.class).progress("test", new ProgressEvent(ProgressEventType.ERROR, 0, 0)); + } + else + { + invocation.getArgument(0, ProgressListener.class).progress("test", new ProgressEvent(ProgressEventType.COMPLETE, 0, 0)); } - return true; - }); + + return null; + }).when(repairRunnable).addProgressListener(any()); config.setRepairMinInterval(repairType, "0s"); + config.setRepairMaxRetries(1); AutoRepair.instance.repairStates.put(repairType, autoRepairState); AutoRepair.instance.repair(repairType); assertEquals(1, sleepCalls.get()); +<<<<<<< HEAD verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedRepairAssignments); verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); +======= + verify(autoRepairState, times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, times(1)).setSkippedTokenRangesCount(0); + verify(autoRepairState, times(1)).setFailedTokenRangesCount(0); +>>>>>>> 0542e95397 (Fix race condition in auto-repair scheduler) } @Test @@ -786,7 +816,10 @@ public void testRepairThrowsForIRWithCDCReplay() public void testSoakAfterImmediateRepair() { when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())).thenReturn(repairRunnable); - when(autoRepairState.isSuccess()).thenReturn(true); + doAnswer(invocation -> { + invocation.getArgument(0, ProgressListener.class).progress("test", new ProgressEvent(ProgressEventType.COMPLETE, 0, 0)); + return null; + }).when(repairRunnable).addProgressListener(any()); AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.repair_task_min_duration = new DurationSpec.LongSecondsBound("10s"); AtomicInteger sleepCalls = new AtomicInteger(); @@ -802,16 +835,19 @@ public void testSoakAfterImmediateRepair() AutoRepair.instance.repair(repairType); assertEquals(1, sleepCalls.get()); - verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedRepairAssignments); - verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); - verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); + verify(autoRepairState, times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, times(1)).setSkippedTokenRangesCount(0); + verify(autoRepairState, times(1)).setFailedTokenRangesCount(0); } @Test public void testNoSoakAfterRepair() { when(autoRepairState.getRepairRunnable(any(), any(), any(), anyBoolean())).thenReturn(repairRunnable); - when(autoRepairState.isSuccess()).thenReturn(true); + doAnswer(invocation -> { + invocation.getArgument(0, ProgressListener.class).progress("test", new ProgressEvent(ProgressEventType.COMPLETE, 0, 0)); + return null; + }).when(repairRunnable).addProgressListener(any()); AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.repair_task_min_duration = new DurationSpec.LongSecondsBound("0s"); AutoRepair.sleepFunc = (Long duration, TimeUnit unit) -> { @@ -822,8 +858,105 @@ public void testNoSoakAfterRepair() AutoRepair.instance.repair(repairType); - verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedRepairAssignments); - verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); - verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); + verify(autoRepairState, times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, times(1)).setSkippedTokenRangesCount(0); + verify(autoRepairState, times(1)).setFailedTokenRangesCount(0); + } + + @Test + public void testSchedulerIgnoresErrorsFromUnrelatedRepairRunables() + { + RepairOption options = new RepairOption(RepairParallelism.PARALLEL, true, repairType == AutoRepairConfig.RepairType.INCREMENTAL, false, + AutoRepairService.instance.getAutoRepairConfig().getRepairThreads(repairType), Set.of(), + false, false, false, PreviewKind.NONE, false, true, false, false); + AutoRepairState repairState = AutoRepair.instance.repairStates.get(repairType); + AutoRepairState spyState = spy(repairState); + AtomicReference failingRepair = new AtomicReference<>(new RepairRunnable(StorageService.instance, StorageService.nextRepairCommand.incrementAndGet(), options, keyspace.getName())); + AtomicReference failingListener = new AtomicReference<>(); + AtomicReference succeedingRepair = new AtomicReference<>(new RepairRunnable(StorageService.instance, StorageService.nextRepairCommand.incrementAndGet(), options, keyspace.getName())); + AtomicInteger repairRunableCalls = new AtomicInteger(); + doAnswer((InvocationOnMock inv ) -> { + RepairRunnable runnable = spy(repairState.getRepairRunnable(inv.getArgument(0), inv.getArgument(1), inv.getArgument(2), + inv.getArgument(3))); + if (repairRunableCalls.getAndIncrement() == 0) + { + // this will be used for first repair job + doAnswer(invocation -> { + // repair runnable for the first repair job will immediately fail + failingListener.set(invocation.getArgument(0, AutoRepair.RepairProgressListener.class)); + invocation.getArgument(0, ProgressListener.class).progress("test", new ProgressEvent(ProgressEventType.ERROR, 0, 0)); + return null; + }).when(runnable).addProgressListener(any()); + failingRepair.set(runnable); + } + else + { + // this will be used for subsequent repair jobs + doAnswer(invocation -> { + if (repairRunableCalls.get() > 0) + { + // repair runnable for the subsequent repair jobs will immediately complete + invocation.getArgument(0, ProgressListener.class).progress("test", new ProgressEvent(ProgressEventType.COMPLETE, 0, 0)); + + } + // repair runnable for the first repair job will continue firing ERROR events + failingListener.get().progress("test", new ProgressEvent(ProgressEventType.ERROR, 0, 0)); + return null; + }).when(runnable).addProgressListener(any()); + succeedingRepair.set(runnable); + } + return runnable; + }).when(spyState).getRepairRunnable(any(), any(), any(), anyBoolean()); + when(spyState.getLastRepairTime()).thenReturn((long) 0); + AutoRepairService.instance.getAutoRepairConfig().setRepairMaxRetries(0); + AutoRepair.instance.repairStates.put(repairType, spyState); + + AutoRepair.instance.repair(repairType); + + assertEquals(1, (int) AutoRepairMetricsManager.getMetrics(repairType).failedTokenRangesCount.getValue()); + // only the first repair job should have failed despite it continuously firing ERROR events + verify(spyState, times(1)).setFailedTokenRangesCount(1); + } + + @Test + public void testProgressError() + { + AutoRepair.RepairProgressListener listener = new AutoRepair.RepairProgressListener(repairType); + + listener.progress("test", new ProgressEvent(ProgressEventType.ERROR, 0, 0, "test")); + + assertFalse(listener.success); + assertTrue(listener.condition.isSignalled()); + } + + @Test + public void testProgressProgress() + { + AutoRepair.RepairProgressListener listener = new AutoRepair.RepairProgressListener(repairType); + + listener.progress("test", new ProgressEvent(ProgressEventType.PROGRESS, 0, 0, "test")); + + assertFalse(listener.success); + assertFalse(listener.condition.isSignalled()); + } + + @Test + public void testProgresComplete() + { + AutoRepair.RepairProgressListener listener = new AutoRepair.RepairProgressListener(repairType); + + listener.progress("test", new ProgressEvent(ProgressEventType.COMPLETE, 0, 0, "test")); + + assertTrue(listener.success); + assertTrue(listener.condition.isSignalled()); + } + + @Test + public void testAwait() throws Exception + { + AutoRepair.RepairProgressListener listener = new AutoRepair.RepairProgressListener(repairType); + listener.progress("test", new ProgressEvent(ProgressEventType.COMPLETE, 0, 0, "test")); + + listener.await(); } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java index b4bb20c7eafd..13ca3d62d676 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -21,8 +21,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -36,17 +34,13 @@ import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; import org.apache.cassandra.repair.autorepair.AutoRepairUtils.AutoRepairHistory; import org.apache.cassandra.service.AutoRepairService; -import org.apache.cassandra.utils.concurrent.Condition; import org.apache.cassandra.utils.progress.ProgressEvent; -import org.apache.cassandra.utils.progress.ProgressEventType; import org.mockito.Mock; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; /** @@ -89,62 +83,7 @@ public void testGetRepairRunnable() } @Test - public void testProgressError() throws InterruptedException - { - AutoRepairState state = RepairType.getAutoRepairState(repairType); - when(progressEvent.getType()).thenReturn(ProgressEventType.ERROR); - - state.progress("test", progressEvent); - - assertFalse(state.success); - assertTrue(state.condition.await(0, TimeUnit.MILLISECONDS)); - } - - @Test - public void testProgress_progress() throws InterruptedException - { - AutoRepairState state = RepairType.getAutoRepairState(repairType); - when(progressEvent.getType()).thenReturn(ProgressEventType.PROGRESS); - - state.progress("test", progressEvent); - - assertTrue(state.success); - assertFalse(state.condition.await(0, TimeUnit.MILLISECONDS)); - } - - - @Test - public void testProgress_complete() throws InterruptedException - { - AutoRepairState state = RepairType.getAutoRepairState(repairType); - when(progressEvent.getType()).thenReturn(ProgressEventType.COMPLETE); - - state.progress("test", progressEvent); - - assertTrue(state.success); - assertTrue(state.condition.await(1, TimeUnit.MILLISECONDS)); - } - - @Test - public void testWaitForRepairToComplete() throws Exception - { - AutoRepairState state = RepairType.getAutoRepairState(repairType); - state.condition.signalAll(); - Condition finishedCondition = Condition.newOneTimeCondition(); - Callable waitForRepairToComplete = () -> { - state.waitForRepairToComplete(new DurationSpec.IntSecondsBound("12h")); - finishedCondition.signalAll(); - return null; - }; - - waitForRepairToComplete.call(); - - assertTrue(finishedCondition.await(1, TimeUnit.MILLISECONDS)); - } - - @Test - public void testGetLastRepairTime() - { + public void testGetLastRepairTime() { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.lastRepairTimeInMs = 1; @@ -372,42 +311,4 @@ public void testGetFailedTokenRangesCount() assertEquals(1, state.getFailedTokenRangesCount()); } - - @Test - public void isSuccess() - { - AutoRepairState state = RepairType.getAutoRepairState(repairType); - state.success = true; - - assertTrue(state.isSuccess()); - - state.success = false; - - assertFalse(state.isSuccess()); - } - - @Test - public void testWaitForRepairToCompleteDoesNotSetSuccessWhenProgressReceivesError() throws InterruptedException - { - AutoRepairState state = RepairType.getAutoRepairState(repairType); - when(progressEvent.getType()).thenReturn(ProgressEventType.ERROR); - - state.progress("test", progressEvent); - assertFalse(state.success); - - state.waitForRepairToComplete(new DurationSpec.IntSecondsBound("12h")); - assertFalse(state.success); - } - - @Test - public void testResetWaitCondition() - { - AutoRepairState state = RepairType.getAutoRepairState(repairType); - state.condition.signalAll(); - assertTrue(state.condition.isSignalled()); - - state.resetWaitCondition(); - - assertFalse(state.condition.isSignalled()); - } } From f3c26e1c087aed371783585d4654bc01abc3c31a Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Sun, 23 Feb 2025 23:06:29 -0800 Subject: [PATCH 134/199] Fix rebase errors --- .../repair/autorepair/AutoRepairState.java | 10 ---------- .../AutoRepairParameterizedTest.java | 18 ++++++------------ 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index 6d14e6ac74be..4a97200db0bb 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -47,18 +47,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -<<<<<<< HEAD -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; -import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; - -/** - * AutoRepairState represents the state of automated repair for a given repair type. - */ -public abstract class AutoRepairState implements ProgressListener -======= // AutoRepairState represents the state of automated repair for a given repair type. public abstract class AutoRepairState ->>>>>>> 0542e95397 (Fix race condition in auto-repair scheduler) { protected static final Logger logger = LoggerFactory.getLogger(AutoRepairState.class); private final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 05551bcf007d..de1da73cce9d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -42,6 +42,7 @@ import org.apache.cassandra.schema.AutoRepairParams; import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableParams; +import org.apache.cassandra.service.StorageService; import org.junit.After; import org.junit.Assert; @@ -560,7 +561,6 @@ public void testMetrics() public void testRepairWaitsForRepairToFinishBeforeSchedullingNewSession() { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - config.setMVRepairEnabled(repairType, false); AutoRepair.instance.repairStates.put(repairType, autoRepairState); when(autoRepairState.getLastRepairTime()).thenReturn((long) 0); AtomicInteger getRepairRunnableCalls = new AtomicInteger(); @@ -693,10 +693,10 @@ public void testRepairMaxRetries() AutoRepair.instance.repair(repairType); // Expect configured retries for each table expected to be repaired - assertEquals(config.getRepairMaxRetries()*expectedTablesGoingThroughRepair, sleepCalls.get()); + assertEquals(config.getRepairMaxRetries()*expectedRepairAssignments, sleepCalls.get()); verify(autoRepairState, times(1)).setSucceededTokenRangesCount(0); verify(autoRepairState, times(1)).setSkippedTokenRangesCount(0); - verify(autoRepairState, times(1)).setFailedTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, times(1)).setFailedTokenRangesCount(expectedRepairAssignments); } @Test @@ -729,15 +729,9 @@ public void testRepairSuccessAfterRetry() AutoRepair.instance.repair(repairType); assertEquals(1, sleepCalls.get()); -<<<<<<< HEAD - verify(autoRepairState, Mockito.times(1)).setSucceededTokenRangesCount(expectedRepairAssignments); - verify(autoRepairState, Mockito.times(1)).setSkippedTokenRangesCount(0); - verify(autoRepairState, Mockito.times(1)).setFailedTokenRangesCount(0); -======= - verify(autoRepairState, times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, times(1)).setSucceededTokenRangesCount(expectedRepairAssignments); verify(autoRepairState, times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, times(1)).setFailedTokenRangesCount(0); ->>>>>>> 0542e95397 (Fix race condition in auto-repair scheduler) } @Test @@ -835,7 +829,7 @@ public void testSoakAfterImmediateRepair() AutoRepair.instance.repair(repairType); assertEquals(1, sleepCalls.get()); - verify(autoRepairState, times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, times(1)).setSucceededTokenRangesCount(expectedRepairAssignments); verify(autoRepairState, times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, times(1)).setFailedTokenRangesCount(0); } @@ -858,7 +852,7 @@ public void testNoSoakAfterRepair() AutoRepair.instance.repair(repairType); - verify(autoRepairState, times(1)).setSucceededTokenRangesCount(expectedTablesGoingThroughRepair); + verify(autoRepairState, times(1)).setSucceededTokenRangesCount(expectedRepairAssignments); verify(autoRepairState, times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, times(1)).setFailedTokenRangesCount(0); } From 31da728822930b3f7bfbd9c65a6dbce1bc8edfc8 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Mon, 24 Feb 2025 13:46:54 -0800 Subject: [PATCH 135/199] Address comments --- .../apache/cassandra/repair/autorepair/AutoRepairState.java | 4 +++- .../repair/autorepair/AutoRepairParameterizedTest.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index 4a97200db0bb..01d2ca7ca38b 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -47,7 +47,9 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -// AutoRepairState represents the state of automated repair for a given repair type. +/** + * AutoRepairState represents the state of automated repair for a given repair type. + */ public abstract class AutoRepairState { protected static final Logger logger = LoggerFactory.getLogger(AutoRepairState.class); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index de1da73cce9d..d87623b723b9 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -861,7 +862,7 @@ public void testNoSoakAfterRepair() public void testSchedulerIgnoresErrorsFromUnrelatedRepairRunables() { RepairOption options = new RepairOption(RepairParallelism.PARALLEL, true, repairType == AutoRepairConfig.RepairType.INCREMENTAL, false, - AutoRepairService.instance.getAutoRepairConfig().getRepairThreads(repairType), Set.of(), + AutoRepairService.instance.getAutoRepairConfig().getRepairThreads(repairType), Collections.emptySet(), false, false, false, PreviewKind.NONE, false, true, false, false); AutoRepairState repairState = AutoRepair.instance.repairStates.get(repairType); AutoRepairState spyState = spy(repairState); From f25ef8735335af30e11c6fabd5f2c3b943b3336c Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Mon, 24 Feb 2025 13:52:21 -0800 Subject: [PATCH 136/199] Address comments --- .../apache/cassandra/repair/autorepair/AutoRepair.java | 8 ++++---- .../repair/autorepair/AutoRepairParameterizedTest.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 1a23eff5662d..4a5758064e89 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -324,7 +324,7 @@ private void repairKeyspace(AutoRepairConfig.RepairType repairType, boolean prim try { long jobStartTime = timeFunc.get(); - listener.await(); + listener.await(config.getRepairSessionTimeout(repairType)); success = listener.isSuccess(); soakAfterRepair(jobStartTime, config.getRepairTaskMinDuration().toMilliseconds()); } @@ -524,10 +524,10 @@ public RepairProgressListener(AutoRepairConfig.RepairType repairType) this.repairType = repairType; } - public void await() throws InterruptedException + public void await(DurationSpec.IntSecondsBound repairSessionTimeout) throws InterruptedException { //if for some reason we don't hear back on repair progress for sometime - if (!condition.await(12, TimeUnit.HOURS)) + if (!condition.await(repairSessionTimeout.to(TimeUnit.SECONDS), TimeUnit.SECONDS)) { success = false; } @@ -542,7 +542,7 @@ public boolean isSuccess() public void progress(String tag, ProgressEvent event) { ProgressEventType type = event.getType(); - String message = String.format("[%s] %s", format.format(System.currentTimeMillis()), event.getMessage()); + String message = String.format("[%s] %s", format.format(Clock.Global.currentTimeMillis()), event.getMessage()); if (type == ProgressEventType.ERROR) { logger.error("Repair failure for repair {}: {}", repairType.toString(), message); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index d87623b723b9..632c2d14b403 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -952,6 +952,6 @@ public void testAwait() throws Exception AutoRepair.RepairProgressListener listener = new AutoRepair.RepairProgressListener(repairType); listener.progress("test", new ProgressEvent(ProgressEventType.COMPLETE, 0, 0, "test")); - listener.await(); + listener.await(new DurationSpec.IntSecondsBound("12h")); } } From 4a6068365e08fc3cc96a1ad325bd8a281b28d81a Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Mon, 24 Feb 2025 15:20:30 -0800 Subject: [PATCH 137/199] Fix formatting --- .../cassandra/repair/autorepair/AutoRepairStateTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java index 13ca3d62d676..fc4ac2f03c79 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -83,7 +83,8 @@ public void testGetRepairRunnable() } @Test - public void testGetLastRepairTime() { + public void testGetLastRepairTime() + { AutoRepairState state = RepairType.getAutoRepairState(repairType); state.lastRepairTimeInMs = 1; From 4ff5d756767ab02ece070b9ab01312c186676e3b Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 20 Feb 2025 16:21:10 -0800 Subject: [PATCH 138/199] A new table property should be wrapped inside a config --- pylib/cqlshlib/cql3handling.py | 3 + pylib/cqlshlib/test/test_cqlsh_completion.py | 6 +- .../repair/autorepair/AutoRepairConfig.java | 6 ++ .../cassandra/schema/SchemaKeyspace.java | 26 ++++-- .../AutoRepairTablePropertyTest.java | 84 +++++++++++++++++++ 5 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTablePropertyTest.java diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py index 03e06d809863..fac81932bbf6 100644 --- a/pylib/cqlshlib/cql3handling.py +++ b/pylib/cqlshlib/cql3handling.py @@ -54,6 +54,7 @@ class Cql3ParsingRuleSet(CqlParsingRuleSet): ('memtable_flush_period_in_ms', None), ('cdc', None), ('read_repair', None), + ('auto_repair', None), ) columnfamily_layout_map_options = ( @@ -519,6 +520,8 @@ def cf_prop_val_completer(ctxt, cass): return [Hint('')] if this_opt in ('read_repair'): return [Hint('<\'none\'|\'blocking\'>')] + if this_opt == 'auto_repair': + return ["{'full_enabled': '"] return [Hint('')] diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py index af9d05ecbc67..cd6780ac1259 100644 --- a/pylib/cqlshlib/test/test_cqlsh_completion.py +++ b/pylib/cqlshlib/test/test_cqlsh_completion.py @@ -623,7 +623,7 @@ def create_columnfamily_table_template(self, name): 'memtable_flush_period_in_ms', 'CLUSTERING', 'COMPACT', 'caching', 'comment', - 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair']) + 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair', 'auto_repair']) self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH ', choices=['bloom_filter_fp_chance', 'compaction', 'compression', @@ -632,7 +632,7 @@ def create_columnfamily_table_template(self, name): 'memtable_flush_period_in_ms', 'CLUSTERING', 'COMPACT', 'caching', 'comment', - 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair']) + 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair', 'auto_repair']) self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH bloom_filter_fp_chance ', immediate='= ') self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH bloom_filter_fp_chance = ', @@ -680,7 +680,7 @@ def create_columnfamily_table_template(self, name): 'memtable_flush_period_in_ms', 'CLUSTERING', 'COMPACT', 'caching', 'comment', - 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair']) + 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair', 'auto_repair']) self.trycompletions(prefix + " new_table (col_a int PRIMARY KEY) WITH compaction = " + "{'class': 'DateTieredCompactionStrategy', '", choices=['base_time_seconds', 'max_sstable_age_days', diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 6ebbc6e086fa..33b4410f60a5 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -146,6 +146,12 @@ public boolean isAutoRepairSchedulingEnabled() return enabled; } + @VisibleForTesting + public void setAutoRepairSchedulingEnabled(boolean enabled) + { + this.enabled = enabled; + } + public DurationSpec.IntSecondsBound getAutoRepairHistoryClearDeleteHostsBufferInterval() { return history_clear_delete_hosts_buffer_interval; diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index 3f032ccaaf78..cd3aedccd001 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -536,7 +536,7 @@ private static void addTableToSchemaMutation(TableMetadata table, boolean withCo } } - private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBuilder builder) + public static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBuilder builder) { builder.add("bloom_filter_fp_chance", params.bloomFilterFpChance) .add("comment", params.comment) @@ -554,8 +554,7 @@ private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBui .add("compaction", params.compaction.asMap()) .add("compression", params.compression.asMap()) .add("read_repair", params.readRepair.toString()) - .add("extensions", params.extensions) - .add("auto_repair", params.autoRepair.asMap()); + .add("extensions", params.extensions); // Only add CDC-enabled flag to schema if it's enabled on the node. This is to work around RTE's post-8099 if a 3.8+ // node sends table schema to a < 3.8 versioned node with an unknown column. @@ -566,6 +565,13 @@ private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBui // in mixed operation with pre-4.1 versioned node during upgrades. if (params.memtable != MemtableParams.DEFAULT) builder.add("memtable", params.memtable.configurationKey()); + + // As above, only add the auto_repair column if the scheduler is enabled + // to avoid RTE in pre-5.1 versioned node during upgrades + if (DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + { + builder.add("auto_repair", params.autoRepair.asMap()); + } } private static void addAlterTableToSchemaMutation(TableMetadata oldTable, TableMetadata newTable, Mutation.SimpleBuilder builder) @@ -957,7 +963,7 @@ private static TableMetadata fetchTable(String keyspaceName, String tableName, T @VisibleForTesting static TableParams createTableParamsFromRow(UntypedResultSet.Row row) { - return TableParams.builder() + TableParams.Builder builder = TableParams.builder() .bloomFilterFpChance(row.getDouble("bloom_filter_fp_chance")) .caching(CachingParams.fromMap(row.getFrozenTextMap("caching"))) .comment(row.getString("comment")) @@ -978,9 +984,15 @@ static TableParams createTableParamsFromRow(UntypedResultSet.Row row) SpeculativeRetryPolicy.fromString(row.getString("additional_write_policy")) : SpeculativeRetryPolicy.fromString("99PERCENTILE")) .cdc(row.has("cdc") && row.getBoolean("cdc")) - .readRepair(getReadRepairStrategy(row)) - .automatedRepair(AutoRepairParams.fromMap(row.getFrozenTextMap("auto_repair"))) - .build(); + .readRepair(getReadRepairStrategy(row)); + + // auto_repair column was introduced in 5.1 + if (row.has("auto_repair")) + { + builder.automatedRepair(AutoRepairParams.fromMap(row.getFrozenTextMap("auto_repair"))); + } + + return builder.build(); } private static List fetchColumns(String keyspace, String table, Types types) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTablePropertyTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTablePropertyTest.java new file mode 100644 index 000000000000..51551e46369d --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTablePropertyTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.Map; + +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.SimpleBuilders; +import org.apache.cassandra.db.rows.ColumnData; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SchemaKeyspace; +import org.apache.cassandra.schema.SchemaKeyspaceTables; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +/** + * Unit tests that verifies "auto_repair" is not included in Schema mutation + * {@link org.apache.cassandra.schema.SchemaKeyspace} if AutoRepair is disabled + */ +public class AutoRepairTablePropertyTest extends CQLTester +{ + @Test + public void testSchedulerDisabledNoColumnReturned() + { + helperTestTableProperty(false); + } + + @Test + public void testSchedulerEnabledShouldReturnColumnReturned() + { + helperTestTableProperty(true); + } + + public void helperTestTableProperty(boolean autoRepairOn) + { + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairSchedulingEnabled(autoRepairOn); + + Map systemSchemaTables = Map.of(SchemaKeyspaceTables.TABLES, "table_name", SchemaKeyspaceTables.VIEWS, "view_name"); + for (Map.Entry systemSchema : systemSchemaTables.entrySet()) + { + ColumnFamilyStore tables = Keyspace.open(SchemaConstants.SCHEMA_KEYSPACE_NAME).getColumnFamilyStore(systemSchema.getKey()); + SimpleBuilders.RowBuilder builder = new SimpleBuilders.RowBuilder(tables.metadata(), systemSchema.getValue()); + SchemaKeyspace.addTableParamsToRowBuilder(tables.metadata().params, builder); + Row row = builder.build(); + ColumnMetadata autoRepair = tables.metadata().getColumn(ByteBufferUtil.bytes("auto_repair")); + ColumnData data = row.getCell(autoRepair); + if (autoRepairOn) + { + assertNotNull(data); + } + else + { + // if AutoRepair is not enabled, the column should not be returned + // as part of the system_schema.tables mutation + assertNull(data); + } + } + } +} From e7ce6b975eec3fc854267e030322bb34284dffe2 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 21 Feb 2025 08:32:38 -0800 Subject: [PATCH 139/199] Skip displaying the table property if AutoRepair is disabled --- src/java/org/apache/cassandra/schema/TableParams.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index e2ec65294f0c..2b315b09f7ea 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -25,6 +25,7 @@ import com.google.common.base.Objects; import com.google.common.collect.ImmutableMap; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Attributes; import org.apache.cassandra.cql3.CqlBuilder; import org.apache.cassandra.exceptions.ConfigurationException; @@ -318,9 +319,12 @@ public void appendCqlTo(CqlBuilder builder, boolean isView) .newLine() .append("AND read_repair = ").appendWithSingleQuotes(readRepair.toString()) .newLine() - .append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString()) - .newLine() - .append("AND auto_repair = ").append(autoRepair.asMap()); + .append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString()); + if (DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + { + builder.newLine() + .append("AND auto_repair = ").append(autoRepair.asMap()); + } } public static final class Builder From 7c1ce43a12e0084e810ec2e98b7f82473233e086 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 21 Feb 2025 09:46:59 -0800 Subject: [PATCH 140/199] Fix broken tests --- pylib/cqlshlib/cql3handling.py | 3 --- pylib/cqlshlib/test/test_cqlsh_completion.py | 6 +++--- pylib/cqlshlib/test/test_cqlsh_output.py | 3 +-- .../cassandra/cql3/statements/DescribeStatementTest.java | 6 ++---- test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java | 4 ++-- 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py index fac81932bbf6..03e06d809863 100644 --- a/pylib/cqlshlib/cql3handling.py +++ b/pylib/cqlshlib/cql3handling.py @@ -54,7 +54,6 @@ class Cql3ParsingRuleSet(CqlParsingRuleSet): ('memtable_flush_period_in_ms', None), ('cdc', None), ('read_repair', None), - ('auto_repair', None), ) columnfamily_layout_map_options = ( @@ -520,8 +519,6 @@ def cf_prop_val_completer(ctxt, cass): return [Hint('')] if this_opt in ('read_repair'): return [Hint('<\'none\'|\'blocking\'>')] - if this_opt == 'auto_repair': - return ["{'full_enabled': '"] return [Hint('')] diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py index cd6780ac1259..af9d05ecbc67 100644 --- a/pylib/cqlshlib/test/test_cqlsh_completion.py +++ b/pylib/cqlshlib/test/test_cqlsh_completion.py @@ -623,7 +623,7 @@ def create_columnfamily_table_template(self, name): 'memtable_flush_period_in_ms', 'CLUSTERING', 'COMPACT', 'caching', 'comment', - 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair', 'auto_repair']) + 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair']) self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH ', choices=['bloom_filter_fp_chance', 'compaction', 'compression', @@ -632,7 +632,7 @@ def create_columnfamily_table_template(self, name): 'memtable_flush_period_in_ms', 'CLUSTERING', 'COMPACT', 'caching', 'comment', - 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair', 'auto_repair']) + 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair']) self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH bloom_filter_fp_chance ', immediate='= ') self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH bloom_filter_fp_chance = ', @@ -680,7 +680,7 @@ def create_columnfamily_table_template(self, name): 'memtable_flush_period_in_ms', 'CLUSTERING', 'COMPACT', 'caching', 'comment', - 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair', 'auto_repair']) + 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair']) self.trycompletions(prefix + " new_table (col_a int PRIMARY KEY) WITH compaction = " + "{'class': 'DateTieredCompactionStrategy', '", choices=['base_time_seconds', 'max_sstable_age_days', diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index 377aa9f4d007..52f564ab29a2 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -690,8 +690,7 @@ def test_describe_columnfamily_output(self): AND memtable_flush_period_in_ms = 0 AND min_index_interval = 128 AND read_repair = 'BLOCKING' - AND speculative_retry = '99p' - AND auto_repair = {'full_enabled': 'true', 'incremental_enabled': 'true', 'preview_repaired_enabled': 'true', 'priority': '0'};""" % quote_name(get_keyspace())) + AND speculative_retry = '99p';""" % quote_name(get_keyspace())) with testrun_cqlsh(tty=True, env=self.default_env) as c: for cmdword in ('describe table', 'desc columnfamily'): diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index e64a43eab522..16ca40691d69 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -1004,8 +1004,7 @@ private static String tableParametersCql() " AND memtable_flush_period_in_ms = 0\n" + " AND min_index_interval = 128\n" + " AND read_repair = 'BLOCKING'\n" + - " AND speculative_retry = '99p'\n" + - " AND auto_repair = {'full_enabled': 'true', 'incremental_enabled': 'true', 'preview_repaired_enabled': 'true', 'priority': '0'};"; + " AND speculative_retry = '99p';"; } private static String mvParametersCql() @@ -1025,8 +1024,7 @@ private static String mvParametersCql() " AND memtable_flush_period_in_ms = 0\n" + " AND min_index_interval = 128\n" + " AND read_repair = 'BLOCKING'\n" + - " AND speculative_retry = '99p'\n" + - " AND auto_repair = {'full_enabled': 'true', 'incremental_enabled': 'true', 'preview_repaired_enabled': 'true', 'priority': '0'};"; + " AND speculative_retry = '99p';"; } private static String keyspaceOutput() diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java index 3e2be66a4504..1fb9e5fc2fb3 100644 --- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java @@ -312,8 +312,8 @@ public void testCfmOptionsCQL() " AND memtable_flush_period_in_ms = 8\n" + " AND min_index_interval = 6\n" + " AND read_repair = 'BLOCKING'\n" + - " AND speculative_retry = 'ALWAYS'\n" + - " AND auto_repair = {'full_enabled': 'true', 'incremental_enabled': 'true', 'preview_repaired_enabled': 'true', 'priority': '0'};" + " AND speculative_retry = 'ALWAYS';" + )); } From 5cc5816da3aa6cdf867adf48db09474ae830e767 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 21 Feb 2025 14:26:45 -0800 Subject: [PATCH 141/199] Extend DescribeStatementTest to cover properties when AutoRepair scheduler is enabled --- .../statements/DescribeStatementTest.java | 126 +++++++++++++----- 1 file changed, 93 insertions(+), 33 deletions(-) diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index 16ca40691d69..2ba54480742e 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -261,6 +261,19 @@ public void testDescribeVirtualTables() throws Throwable @Test public void testDescribe() throws Throwable + { + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairSchedulingEnabled(false); + helperTestDescribe(); + } + + @Test + public void testDescribeWithAutoRepair() throws Throwable + { + DatabaseDescriptor.getAutoRepairConfig().setAutoRepairSchedulingEnabled(true); + helperTestDescribe(); + } + + public void helperTestDescribe() throws Throwable { try { @@ -988,43 +1001,90 @@ private static String testTableOutput() private static String tableParametersCql() { - return "additional_write_policy = '99p'\n" + - " AND bloom_filter_fp_chance = 0.01\n" + - " AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" + - " AND cdc = false\n" + - " AND comment = ''\n" + - " AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}\n" + - " AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}\n" + - " AND memtable = 'default'\n" + - " AND crc_check_chance = 1.0\n" + - " AND default_time_to_live = 0\n" + - " AND extensions = {}\n" + - " AND gc_grace_seconds = 864000\n" + - " AND max_index_interval = 2048\n" + - " AND memtable_flush_period_in_ms = 0\n" + - " AND min_index_interval = 128\n" + - " AND read_repair = 'BLOCKING'\n" + - " AND speculative_retry = '99p';"; + if (!DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + { + return "additional_write_policy = '99p'\n" + + " AND bloom_filter_fp_chance = 0.01\n" + + " AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" + + " AND cdc = false\n" + + " AND comment = ''\n" + + " AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}\n" + + " AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}\n" + + " AND memtable = 'default'\n" + + " AND crc_check_chance = 1.0\n" + + " AND default_time_to_live = 0\n" + + " AND extensions = {}\n" + + " AND gc_grace_seconds = 864000\n" + + " AND max_index_interval = 2048\n" + + " AND memtable_flush_period_in_ms = 0\n" + + " AND min_index_interval = 128\n" + + " AND read_repair = 'BLOCKING'\n" + + " AND speculative_retry = '99p';"; + } + else + { + return "additional_write_policy = '99p'\n" + + " AND bloom_filter_fp_chance = 0.01\n" + + " AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" + + " AND cdc = false\n" + + " AND comment = ''\n" + + " AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}\n" + + " AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}\n" + + " AND memtable = 'default'\n" + + " AND crc_check_chance = 1.0\n" + + " AND default_time_to_live = 0\n" + + " AND extensions = {}\n" + + " AND gc_grace_seconds = 864000\n" + + " AND max_index_interval = 2048\n" + + " AND memtable_flush_period_in_ms = 0\n" + + " AND min_index_interval = 128\n" + + " AND read_repair = 'BLOCKING'\n" + + " AND speculative_retry = '99p'\n" + + " AND auto_repair = {'full_enabled': 'true', 'incremental_enabled': 'true', 'preview_repaired_enabled': 'true', 'priority': '0'};"; + } } private static String mvParametersCql() { - return "additional_write_policy = '99p'\n" + - " AND bloom_filter_fp_chance = 0.01\n" + - " AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" + - " AND cdc = false\n" + - " AND comment = ''\n" + - " AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}\n" + - " AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}\n" + - " AND memtable = 'default'\n" + - " AND crc_check_chance = 1.0\n" + - " AND extensions = {}\n" + - " AND gc_grace_seconds = 864000\n" + - " AND max_index_interval = 2048\n" + - " AND memtable_flush_period_in_ms = 0\n" + - " AND min_index_interval = 128\n" + - " AND read_repair = 'BLOCKING'\n" + - " AND speculative_retry = '99p';"; + if (!DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + { + return "additional_write_policy = '99p'\n" + + " AND bloom_filter_fp_chance = 0.01\n" + + " AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" + + " AND cdc = false\n" + + " AND comment = ''\n" + + " AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}\n" + + " AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}\n" + + " AND memtable = 'default'\n" + + " AND crc_check_chance = 1.0\n" + + " AND extensions = {}\n" + + " AND gc_grace_seconds = 864000\n" + + " AND max_index_interval = 2048\n" + + " AND memtable_flush_period_in_ms = 0\n" + + " AND min_index_interval = 128\n" + + " AND read_repair = 'BLOCKING'\n" + + " AND speculative_retry = '99p';"; + } + else + { + return "additional_write_policy = '99p'\n" + + " AND bloom_filter_fp_chance = 0.01\n" + + " AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}\n" + + " AND cdc = false\n" + + " AND comment = ''\n" + + " AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}\n" + + " AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}\n" + + " AND memtable = 'default'\n" + + " AND crc_check_chance = 1.0\n" + + " AND extensions = {}\n" + + " AND gc_grace_seconds = 864000\n" + + " AND max_index_interval = 2048\n" + + " AND memtable_flush_period_in_ms = 0\n" + + " AND min_index_interval = 128\n" + + " AND read_repair = 'BLOCKING'\n" + + " AND speculative_retry = '99p'\n" + + " AND auto_repair = {'full_enabled': 'true', 'incremental_enabled': 'true', 'preview_repaired_enabled': 'true', 'priority': '0'};"; + } } private static String keyspaceOutput() From 6d89f59497eb6de90e81b54d0a57aa9cb72c9e3d Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Mon, 3 Mar 2025 09:26:27 -0600 Subject: [PATCH 142/199] Account for vnodes with repair_by_keyspace in RepairTokenRangeSplitter Updates RepairTokenRangeSplitter to call getRepairAssignmentsForKeyspace for each individual token range so repair_by_keyspace grouping works correctly. Patch by Andy Tolbert; Reviewed by Jaydeepkumar Chovatia for CASSANDRA-20392 --- .../autorepair/RepairTokenRangeSplitter.java | 46 +++++----- .../AutoRepairParameterizedTest.java | 46 +++++----- .../RepairTokenRangeSplitterTest.java | 83 +++++++++++++++---- 3 files changed, 114 insertions(+), 61 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index c8456d4cfcb6..18415f5d3551 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -121,7 +121,7 @@ *
      • * max_bytes_per_schedule: The maximum number of bytes to cover an individual schedule. This serves * as a mechanism for throttling the amount of work that can be done on each repair cycle. One may opt to - * reduce this value if the impact of repairs is causing too many load on the cluster, or increase it if + * reduce this value if the impact of repairs is causing too much load on the cluster, or increase it if * writes outpace the amount of data being repaired. Alternatively, one may want to choose tuning down or up * the min_repair_interval. *
        • @@ -293,8 +293,16 @@ protected KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repair return new KeyspaceRepairAssignments(priority, repairPlan.getKeyspaceName(), Collections.emptyList()); } - Collection> tokenRanges = getTokenRanges(primaryRangeOnly, repairPlan.getKeyspaceName()); - List repairAssignments = getRepairAssignmentsForKeyspace(repairType, repairPlan.getKeyspaceName(), repairPlan.getTableNames(), tokenRanges); + List> tokenRanges = getTokenRanges(primaryRangeOnly, repairPlan.getKeyspaceName()); + // shuffle token ranges to unbias selection of ranges + Collections.shuffle(tokenRanges); + List repairAssignments = new ArrayList<>(); + // Generate assignments for each range speparately + for (Range tokenRange : tokenRanges) + { + repairAssignments.addAll(getRepairAssignmentsForKeyspace(repairType, repairPlan.getKeyspaceName(), repairPlan.getTableNames(), tokenRange)); + } + FilteredRepairAssignments filteredRepairAssignments = filterRepairAssignments(priority, repairPlan.getKeyspaceName(), repairAssignments, bytesSoFar); bytesSoFar = filteredRepairAssignments.newBytesSoFar; return new KeyspaceRepairAssignments(priority, repairPlan.getKeyspaceName(), filteredRepairAssignments.repairAssignments); @@ -302,7 +310,7 @@ protected KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repair } @VisibleForTesting - List getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType repairType, String keyspaceName, List tableNames, Collection> tokenRanges) + List getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType repairType, String keyspaceName, List tableNames, Range tokenRange) { List repairAssignments = new ArrayList<>(); // this is used for batching minimal single assignment tables together @@ -312,9 +320,10 @@ List getRepairAssignmentsForKeyspace(AutoRepairConfig.Rep // If we can repair by keyspace, sort the tables by size so can batch the smallest ones together boolean repairByKeyspace = config.getRepairByKeyspace(repairType); + List tablesToProcess = tableNames; if (repairByKeyspace) { - tableNames.sort((t1, t2) -> { + tablesToProcess = tableNames.stream().sorted((t1, t2) -> { ColumnFamilyStore cfs1 = ColumnFamilyStore.getIfExists(keyspaceName, t1); ColumnFamilyStore cfs2 = ColumnFamilyStore.getIfExists(keyspaceName, t2); // If for whatever reason the CFS is not retrievable, we can assume it has been deleted, so give the @@ -330,12 +339,12 @@ else if (cfs2 == null) return 1; } return Long.compare(cfs1.metric.totalDiskSpaceUsed.getCount(), cfs2.metric.totalDiskSpaceUsed.getCount()); - }); + }).collect(Collectors.toList()); } - for (String tableName : tableNames) + for (String tableName : tablesToProcess) { - List tableAssignments = getRepairAssignmentsForTable(keyspaceName, tableName, tokenRanges); + List tableAssignments = getRepairAssignmentsForTable(keyspaceName, tableName, tokenRange); if (tableAssignments.isEmpty()) continue; @@ -517,9 +526,9 @@ static SizedRepairAssignment merge(List assignments) } @VisibleForTesting - protected List getRepairAssignmentsForTable(String keyspaceName, String tableName, Collection> tokenRanges) + protected List getRepairAssignmentsForTable(String keyspaceName, String tableName, Range tokenRange) { - List sizeEstimates = getRangeSizeEstimate(keyspaceName, tableName, tokenRanges); + List sizeEstimates = getRangeSizeEstimate(keyspaceName, tableName, tokenRange); return getRepairAssignments(sizeEstimates); } @@ -630,7 +639,7 @@ private int calculateNumberOfSplits(SizeEstimate estimate) return splits; } - private Collection> getTokenRanges(boolean primaryRangeOnly, String keyspaceName) + private List> getTokenRanges(boolean primaryRangeOnly, String keyspaceName) { // Collect all applicable token ranges Collection> wrappedRanges; @@ -652,18 +661,15 @@ private Collection> getTokenRanges(boolean primaryRangeOnly, String return ranges; } - private List getRangeSizeEstimate(String keyspace, String table, Collection> tokenRanges) + private List getRangeSizeEstimate(String keyspace, String table, Range tokenRange) { List sizeEstimates = new ArrayList<>(); - for (Range tokenRange : tokenRanges) + logger.debug("Calculating size estimate for {}.{} for range {}", keyspace, table, tokenRange); + try (Refs refs = getSSTableReaderRefs(repairType, keyspace, table, tokenRange)) { - logger.debug("Calculating size estimate for {}.{} for range {}", keyspace, table, tokenRange); - try (Refs refs = getSSTableReaderRefs(repairType, keyspace, table, tokenRange)) - { - SizeEstimate estimate = getSizesForRangeOfSSTables(repairType, keyspace, table, tokenRange, refs); - logger.debug("Generated size estimate {}", estimate); - sizeEstimates.add(estimate); - } + SizeEstimate estimate = getSizesForRangeOfSSTables(repairType, keyspace, table, tokenRange, refs); + logger.debug("Generated size estimate {}", estimate); + sizeEstimates.add(estimate); } return sizeEstimates; } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 632c2d14b403..d38e3c5a762b 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -177,10 +177,9 @@ public static void setupClass() throws Exception DatabaseDescriptor.setMaterializedViewsEnabled(false); DatabaseDescriptor.setCDCEnabled(false); - // Calculate the expected number of tables to be repaired, this should be all system keyspaces that are - // distributed, plus 1 for the table we created (ks.tbl), excluding the 'mv' materialized view and - // 'tbl_disabled_auto_repair' we created. - int expectedTablesGoingThroughRepair = 0; + // Calculate the expected number of keyspaces to be repaired, this should be all system keyspaces that are + // distributed, plus 1 for the table we created (ks.tbl). + int expectedKeyspacesGoingThroughRepair = 0; for (Keyspace keyspace : Keyspace.all()) { // skip LocalStrategy keyspaces as these aren't repaired. @@ -193,11 +192,10 @@ public static void setupClass() throws Exception { continue; } - int expectedTables = keyspace.getName().equals("ks") ? 1 : keyspace.getColumnFamilyStores().size(); - expectedTablesGoingThroughRepair += expectedTables; + expectedKeyspacesGoingThroughRepair += 1; } // Since the splitter will unwrap a full token range, we expect twice as many repairs. - expectedRepairAssignments = expectedTablesGoingThroughRepair * 2; + expectedRepairAssignments = expectedKeyspacesGoingThroughRepair * 2; } @Before @@ -205,6 +203,7 @@ public void setup() { DatabaseDescriptor.setCDCOnRepairEnabled(false); DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); + //noinspection resource MockitoAnnotations.initMocks(this); Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).truncateBlocking(); @@ -294,7 +293,7 @@ public void testRepairAsync() public void testRepairTurn() { UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); - Assert.assertTrue("Expected my turn for the repair", AutoRepairUtils.myTurnToRunRepair(repairType, myId) != NOT_MY_TURN); + Assert.assertNotEquals("Expected my turn for the repair", NOT_MY_TURN, AutoRepairUtils.myTurnToRunRepair(repairType, myId)); } @Test @@ -319,8 +318,8 @@ public void testTooFrequentRepairs() AutoRepair.instance.repair(repairType); long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); int consideredTables = AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair(); - Assert.assertNotSame(String.format("Expected total repaired tables > 0, actual value %s ", consideredTables), - consideredTables, 0); + Assert.assertNotEquals(String.format("Expected total repaired tables > 0, actual value %s ", consideredTables), + 0, consideredTables); //if repair was done in last 24 hours then it should not trigger another repair config.setRepairMinInterval(repairType, "24h"); @@ -344,12 +343,12 @@ public void testNonFrequentRepairs() Assert.assertTrue(String.format("Expected lastRepairTime1 > 0, actual value lastRepairTime1 %d", lastRepairTime1), lastRepairTime1 > 0); UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); - Assert.assertTrue("Expected my turn for the repair", - AutoRepairUtils.myTurnToRunRepair(repairType, myId) != NOT_MY_TURN); + Assert.assertNotEquals("Expected my turn for the repair", + NOT_MY_TURN, AutoRepairUtils.myTurnToRunRepair(repairType, myId)); AutoRepair.instance.repair(repairType); long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); Assert.assertNotSame(String.format("Expected repair time to be same, actual value lastRepairTime1 %d, " + - "lastRepairTime2 ", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); + "lastRepairTime2 %d", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); assertEquals(prevCount, state.getTotalMVTablesConsideredForRepair()); assertEquals(prevMetricsCount, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue()); } @@ -361,16 +360,15 @@ public void testGetPriorityHosts() AutoRepairState state = AutoRepair.instance.repairStates.get(repairType); long prevCount = state.getTotalMVTablesConsideredForRepair(); AutoRepairService.instance.getAutoRepairConfig().setRepairMinInterval(repairType, "0s"); - Assert.assertSame(String.format("Priority host count is not same, actual value %d, expected value %d", - AutoRepairUtils.getPriorityHosts(repairType).size(), 0), AutoRepairUtils.getPriorityHosts(repairType).size(), 0); + Assert.assertEquals(String.format("Priority host count is not same, actual value %d, expected value %d", + AutoRepairUtils.getPriorityHosts(repairType).size(), 0), 0, AutoRepairUtils.getPriorityHosts(repairType).size()); UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); - Assert.assertTrue("Expected my turn for the repair", AutoRepairUtils.myTurnToRunRepair(repairType, myId) != - NOT_MY_TURN); + Assert.assertNotEquals("Expected my turn for the repair", NOT_MY_TURN, AutoRepairUtils.myTurnToRunRepair(repairType, myId)); AutoRepair.instance.repair(repairType); AutoRepairUtils.addPriorityHosts(repairType, Sets.newHashSet(FBUtilities.getBroadcastAddressAndPort())); AutoRepair.instance.repair(repairType); - Assert.assertSame(String.format("Priority host count is not same actual value %d, expected value %d", - AutoRepairUtils.getPriorityHosts(repairType).size(), 0), AutoRepairUtils.getPriorityHosts(repairType).size(), 0); + Assert.assertEquals(String.format("Priority host count is not same actual value %d, expected value %d", + AutoRepairUtils.getPriorityHosts(repairType).size(), 0), 0, AutoRepairUtils.getPriorityHosts(repairType).size()); assertEquals(prevCount, state.getTotalMVTablesConsideredForRepair()); assertEquals(prevMetricsCount, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue()); } @@ -419,7 +417,7 @@ public void testGetAllMVs() config.setMaterializedViewRepairEnabled(repairType, true); assertTrue(config.getMaterializedViewRepairEnabled(repairType)); - assertEquals(Arrays.asList(MV), AutoRepairUtils.getAllMVs(repairType, keyspace, cfm)); + assertEquals(List.of(MV), AutoRepairUtils.getAllMVs(repairType, keyspace, cfm)); config.setMaterializedViewRepairEnabled(repairType, false); } @@ -602,7 +600,7 @@ public void testDisabledAutoRepairForATableThroughTableLevelConfiguration() AutoRepair.instance.repair(repairType); int consideredTables = AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair(); Assert.assertNotSame(String.format("Expected total repaired tables > 0, actual value %s ", consideredTables), - consideredTables, 0); + 0, consideredTables); int disabledTablesRepairCountAfter = AutoRepair.instance.repairStates.get(repairType).getTotalDisabledTablesRepairCount(); Assert.assertTrue(String.format("A table %s should be skipped from auto repair, expected value: %d, actual value %d ", TABLE_DISABLED_AUTO_REPAIR, disabledTablesRepairCountBefore + 1, disabledTablesRepairCountAfter), disabledTablesRepairCountBefore < disabledTablesRepairCountAfter); @@ -693,7 +691,7 @@ public void testRepairMaxRetries() AutoRepair.instance.repair(repairType); - // Expect configured retries for each table expected to be repaired + // Expect configured retries for each keyspace expected to be repaired assertEquals(config.getRepairMaxRetries()*expectedRepairAssignments, sleepCalls.get()); verify(autoRepairState, times(1)).setSucceededTokenRangesCount(0); verify(autoRepairState, times(1)).setSkippedTokenRangesCount(0); @@ -866,9 +864,7 @@ public void testSchedulerIgnoresErrorsFromUnrelatedRepairRunables() false, false, false, PreviewKind.NONE, false, true, false, false); AutoRepairState repairState = AutoRepair.instance.repairStates.get(repairType); AutoRepairState spyState = spy(repairState); - AtomicReference failingRepair = new AtomicReference<>(new RepairRunnable(StorageService.instance, StorageService.nextRepairCommand.incrementAndGet(), options, keyspace.getName())); AtomicReference failingListener = new AtomicReference<>(); - AtomicReference succeedingRepair = new AtomicReference<>(new RepairRunnable(StorageService.instance, StorageService.nextRepairCommand.incrementAndGet(), options, keyspace.getName())); AtomicInteger repairRunableCalls = new AtomicInteger(); doAnswer((InvocationOnMock inv ) -> { RepairRunnable runnable = spy(repairState.getRepairRunnable(inv.getArgument(0), inv.getArgument(1), inv.getArgument(2), @@ -882,7 +878,6 @@ public void testSchedulerIgnoresErrorsFromUnrelatedRepairRunables() invocation.getArgument(0, ProgressListener.class).progress("test", new ProgressEvent(ProgressEventType.ERROR, 0, 0)); return null; }).when(runnable).addProgressListener(any()); - failingRepair.set(runnable); } else { @@ -898,7 +893,6 @@ public void testSchedulerIgnoresErrorsFromUnrelatedRepairRunables() failingListener.get().progress("test", new ProgressEvent(ProgressEventType.ERROR, 0, 0)); return null; }).when(runnable).addProgressListener(any()); - succeedingRepair.set(runnable); } return runnable; }).when(spyState).getRepairRunnable(any(), any(), any(), anyBoolean()); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index fe7b2815fe9d..78a922111670 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; @@ -32,11 +31,11 @@ import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.auth.AuthKeyspace; import org.apache.cassandra.config.DataStorageSpec.LongMebibytesBound; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -51,6 +50,8 @@ import static org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.BYTES_PER_ASSIGNMENT; import static org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.MAX_TABLES_PER_ASSIGNMENT; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -62,6 +63,8 @@ public class RepairTokenRangeSplitterTest extends CQLTester private String tableName; private static Range FULL_RANGE; + private static List AUTH_TABLE_NAMES = Arrays.asList(AuthKeyspace.ROLES, AuthKeyspace.ROLE_MEMBERS, AuthKeyspace.ROLE_PERMISSIONS, AuthKeyspace.RESOURCE_ROLE_INDEX, AuthKeyspace.NETWORK_PERMISSIONS); + @BeforeClass public static void setUpClass() { @@ -73,6 +76,7 @@ public static void setUpClass() @Before public void setUp() { + AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(RepairType.FULL, true); repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.emptyMap()); tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); } @@ -119,17 +123,15 @@ public void testSizePartitionCountSplit() throws Throwable public void testGetRepairAssignmentsForTable_NoSSTables() { // Should return 1 assignment if there are no SSTables - Collection> ranges = Collections.singleton(new Range<>(Murmur3Partitioner.instance.getMinimumToken(), Murmur3Partitioner.instance.getMaximumToken())); - List assignments = repairRangeSplitter.getRepairAssignmentsForTable(CQLTester.KEYSPACE, tableName, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(CQLTester.KEYSPACE, tableName, FULL_RANGE); assertEquals(1, assignments.size()); } @Test public void testGetRepairAssignmentsForTable_Single() throws Throwable { - Collection> ranges = Collections.singleton(new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken())); insertAndFlushSingleTable(); - List assignments = repairRangeSplitter.getRepairAssignmentsForTable(CQLTester.KEYSPACE, tableName, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(CQLTester.KEYSPACE, tableName, FULL_RANGE); assertEquals(1, assignments.size()); } @@ -137,10 +139,9 @@ public void testGetRepairAssignmentsForTable_Single() throws Throwable public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable { repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "2")); - Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, FULL_RANGE); // We expect two assignments, one with table1 and table2 batched, and one with table3 assertEquals(2, assignments.size()); @@ -152,10 +153,9 @@ public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable { repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "2")); - Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(2); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, FULL_RANGE); // We expect one assignment, with two tables batched assertEquals(1, assignments.size()); @@ -166,10 +166,9 @@ public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable { repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "1")); - Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(3); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, FULL_RANGE); assertEquals(3, assignments.size()); } @@ -178,10 +177,9 @@ public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable { repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "100")); - Collection> ranges = Collections.singleton(FULL_RANGE); List tableNames = createAndInsertTables(5); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, ranges); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, FULL_RANGE); assertEquals(1, assignments.size()); } @@ -309,6 +307,62 @@ public void testGetRepairAssignmentsSplitsBySubrangeSizeAndFilterLimitsByMaxByte assertEquals(expectedBytes * 2, filteredRepairAssignments.newBytesSoFar); } + @Test + public void testTokenRangesRepairByKeyspace() + { + AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(RepairType.FULL, true); + + final KeyspaceRepairPlan repairPlan = new KeyspaceRepairPlan("system_auth", AUTH_TABLE_NAMES); + final PrioritizedRepairPlan prioritizedRepairPlan = new PrioritizedRepairPlan(0, List.of(repairPlan)); + + Iterator keyspaceAssignments = repairRangeSplitter.getRepairAssignments(true, List.of(prioritizedRepairPlan)); + + // should be only 1 entry for the keyspace. + assertTrue(keyspaceAssignments.hasNext()); + KeyspaceRepairAssignments keyspace = keyspaceAssignments.next(); + assertFalse(keyspaceAssignments.hasNext()); + + List assignments = keyspace.getRepairAssignments(); + assertNotNull(assignments); + + // Should only be two assignments (since single node encompasses the whole range, should get 2 primary ranges) + // to account for the range wrapping the ring. + assertEquals(2, assignments.size()); + + for (RepairAssignment assignment : assignments) + { + assertEquals(AUTH_TABLE_NAMES.size(), assignment.getTableNames().size()); + } + } + + @Test + public void testTokenRangesRepairByKeyspaceFalse() + { + AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(RepairType.FULL, false); + + final KeyspaceRepairPlan repairPlan = new KeyspaceRepairPlan("system_auth", AUTH_TABLE_NAMES); + final PrioritizedRepairPlan prioritizedRepairPlan = new PrioritizedRepairPlan(0, List.of(repairPlan)); + + Iterator keyspaceAssignments = repairRangeSplitter.getRepairAssignments(true, List.of(prioritizedRepairPlan)); + + // should be only 1 entry for the keyspace. + assertTrue(keyspaceAssignments.hasNext()); + KeyspaceRepairAssignments keyspace = keyspaceAssignments.next(); + assertFalse(keyspaceAssignments.hasNext()); + + List assignments = keyspace.getRepairAssignments(); + assertNotNull(assignments); + + // Should be two ranges * X system_auth table names assignments + assertEquals(2 * AUTH_TABLE_NAMES.size(), assignments.size()); + + // each assignment should only include one table. + for (RepairAssignment assignment : assignments) + { + assertEquals(1, assignment.getTableNames().size()); + } + } + @Test(expected = IllegalArgumentException.class) public void testSetParameterShouldNotAllowUnknownParameter() { @@ -374,7 +428,6 @@ private List createAndInsertTables(int count) throws Throwable private void insertAndFlushTable(String tableName) throws Throwable { insertAndFlushTable(tableName, 1); - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, tableName); } private void insertAndFlushTable(String tableName, int... vals) throws Throwable From aca1d61353c8ce6ca68b1cd9ffe80f11843abfde Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Mon, 3 Mar 2025 15:25:16 -0600 Subject: [PATCH 143/199] Make work with JDK11 (and JDK8) --- .../repair/autorepair/AutoRepairParameterizedTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index d38e3c5a762b..a05ce6593f13 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -417,7 +417,7 @@ public void testGetAllMVs() config.setMaterializedViewRepairEnabled(repairType, true); assertTrue(config.getMaterializedViewRepairEnabled(repairType)); - assertEquals(List.of(MV), AutoRepairUtils.getAllMVs(repairType, keyspace, cfm)); + assertEquals(Collections.singletonList(MV), AutoRepairUtils.getAllMVs(repairType, keyspace, cfm)); config.setMaterializedViewRepairEnabled(repairType, false); } From fd816723868dbcb9236dac4693ee0b5948748738 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 3 Mar 2025 16:11:46 -0800 Subject: [PATCH 144/199] Type & Remove whitespace --- test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java index 1fb9e5fc2fb3..63ef86110177 100644 --- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java @@ -313,7 +313,6 @@ public void testCfmOptionsCQL() " AND min_index_interval = 6\n" + " AND read_repair = 'BLOCKING'\n" + " AND speculative_retry = 'ALWAYS';" - )); } From 6e1a356c4a00ef1cacaa763c0d4a880cd183e381 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 6 Mar 2025 12:19:57 -0800 Subject: [PATCH 145/199] Ensure the main config obj has been initialized before checking the yaml property --- src/java/org/apache/cassandra/schema/SchemaKeyspace.java | 2 +- src/java/org/apache/cassandra/schema/TableParams.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index cd3aedccd001..4058157d4873 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -568,7 +568,7 @@ public static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBuil // As above, only add the auto_repair column if the scheduler is enabled // to avoid RTE in pre-5.1 versioned node during upgrades - if (DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + if (DatabaseDescriptor.getRawConfig() != null && DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) { builder.add("auto_repair", params.autoRepair.asMap()); } diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 2b315b09f7ea..3c09c1b81545 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -320,7 +320,7 @@ public void appendCqlTo(CqlBuilder builder, boolean isView) .append("AND read_repair = ").appendWithSingleQuotes(readRepair.toString()) .newLine() .append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString()); - if (DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + if (DatabaseDescriptor.getRawConfig() != null && DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) { builder.newLine() .append("AND auto_repair = ").append(autoRepair.asMap()); From c9cf27add72f496468f93c19bf39198657773d6b Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 6 Mar 2025 13:59:01 -0800 Subject: [PATCH 146/199] Use the last repair stat to confirm that the repair ran indeed instead of sleep to avoid flaky test --- .../repair/autorepair/AutoRepair.java | 2 +- .../test/repair/AutoRepairSchedulerTest.java | 61 +++++++++++++------ 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 4a5758064e89..8fcead64004f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -89,7 +89,7 @@ public class AutoRepair protected final Map repairRunnableExecutors; @VisibleForTesting - protected final Map repairStates; + public final Map repairStates; @VisibleForTesting // Auto-repair is likely to be run on multiple nodes independently, we want to avoid running multiple repair diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index dc24e5847b99..89ef40640317 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -21,22 +21,22 @@ import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.concurrent.TimeUnit; import java.util.UUID; import com.google.common.collect.ImmutableMap; -import com.google.common.util.concurrent.Uninterruptibles; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.schema.SystemDistributedKeyspace; + import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; -import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.service.AutoRepairService; import static org.apache.cassandra.schema.SchemaConstants.DISTRIBUTED_KEYSPACE_NAME; @@ -63,19 +63,19 @@ public static void init() throws IOException ImmutableMap.of( "repair_type_overrides", ImmutableMap.of(AutoRepairConfig.RepairType.FULL.getConfigName(), - ImmutableMap.of( - "initial_scheduler_delay", "5s", - "enabled", "true", - "parallel_repair_count", "1", - "parallel_repair_percentage", "0", - "min_repair_interval", "1s"), + ImmutableMap.of( + "initial_scheduler_delay", "5s", + "enabled", "true", + "parallel_repair_count", "1", + "parallel_repair_percentage", "0", + "min_repair_interval", "1s"), AutoRepairConfig.RepairType.INCREMENTAL.getConfigName(), - ImmutableMap.of( - "initial_scheduler_delay", "5s", - "enabled", "true", - "parallel_repair_count", "1", - "parallel_repair_percentage", "0", - "min_repair_interval", "1s")))) + ImmutableMap.of( + "initial_scheduler_delay", "5s", + "enabled", "true", + "parallel_repair_count", "1", + "parallel_repair_percentage", "0", + "min_repair_interval", "1s")))) .set("auto_repair.enabled", "true") .set("auto_repair.global_settings.repair_by_keyspace", "true") .set("auto_repair.repair_task_min_duration", "0s") @@ -105,9 +105,34 @@ public void testScheduler() throws ParseException throw new RuntimeException(e); } })); - // wait for a couple of minutes for repair to go through on all three nodes - Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MINUTES); + // validate that the repair ran on all nodes + cluster.forEach(i -> i.runOnInstance(() -> { + try + { + while(true) + { + if (AutoRepair.instance.repairStates.get(AutoRepairConfig.RepairType.FULL).getLastRepairTime() == 0 + || AutoRepair.instance.repairStates.get(AutoRepairConfig.RepairType.INCREMENTAL).getLastRepairTime() == 0) + { + try + { + Thread.sleep(5000); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + continue; + } + break; + } + } + catch (Exception e) + { + throw new RuntimeException(e); + } + })); validate(AutoRepairConfig.RepairType.FULL.toString()); validate(AutoRepairConfig.RepairType.INCREMENTAL.toString()); } From eb49eb17c665c515f5b6e1650c9db11d3825aebf Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 6 Mar 2025 14:29:09 -0800 Subject: [PATCH 147/199] Incorporate the code review comments --- src/java/org/apache/cassandra/schema/SchemaKeyspace.java | 3 ++- src/java/org/apache/cassandra/schema/TableParams.java | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index 4058157d4873..5a3b2d57bb3e 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -568,7 +568,8 @@ public static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBuil // As above, only add the auto_repair column if the scheduler is enabled // to avoid RTE in pre-5.1 versioned node during upgrades - if (DatabaseDescriptor.getRawConfig() != null && DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + if (DatabaseDescriptor.getRawConfig() != null + && DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) { builder.add("auto_repair", params.autoRepair.asMap()); } diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 3c09c1b81545..7c218e5f25ff 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -320,7 +320,9 @@ public void appendCqlTo(CqlBuilder builder, boolean isView) .append("AND read_repair = ").appendWithSingleQuotes(readRepair.toString()) .newLine() .append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString()); - if (DatabaseDescriptor.getRawConfig() != null && DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) + + if (DatabaseDescriptor.getRawConfig() != null + && DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) { builder.newLine() .append("AND auto_repair = ").append(autoRepair.asMap()); From 99f9891270ce8c7a9da31114bf75d0ca67f84b19 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 6 Mar 2025 14:34:54 -0800 Subject: [PATCH 148/199] Incorporate the code review comments From f603c4506e3434171e1c0aadaef2ef0646ef2d76 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 6 Mar 2025 14:37:26 -0800 Subject: [PATCH 149/199] Group all public statements together --- .../org/apache/cassandra/repair/autorepair/AutoRepair.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 8fcead64004f..b793d91e650d 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -83,14 +83,14 @@ public class AutoRepair // Sleep for 5 seconds if repair finishes quickly to flush JMX metrics; it happens only for Cassandra nodes with tiny amount of data. public static DurationSpec.IntSecondsBound SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("5s"); + @VisibleForTesting + public final Map repairStates; + @VisibleForTesting protected final Map repairExecutors; protected final Map repairRunnableExecutors; - @VisibleForTesting - public final Map repairStates; - @VisibleForTesting // Auto-repair is likely to be run on multiple nodes independently, we want to avoid running multiple repair // sessions on overlapping datasets at the same time. Shuffling keyspaces reduces the likelihood of this happening. From d03a71e7df6ae107d1fad4f71f9d71a46e4adfca Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 6 Mar 2025 14:41:44 -0800 Subject: [PATCH 150/199] No need for try/catch in the test case --- .../cassandra/repair/autorepair/AutoRepairParameterizedTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index a05ce6593f13..5a4be0d800ec 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -43,7 +43,6 @@ import org.apache.cassandra.schema.AutoRepairParams; import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableParams; -import org.apache.cassandra.service.StorageService; import org.junit.After; import org.junit.Assert; From 3988cf836feed849ded1a6a82efc34497faaddb8 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sat, 8 Mar 2025 22:07:49 -0800 Subject: [PATCH 151/199] make it Java8 compatible --- .../repair/autorepair/AutoRepairTablePropertyTest.java | 9 ++++++++- .../repair/autorepair/RepairTokenRangeSplitterTest.java | 9 +++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTablePropertyTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTablePropertyTest.java index 51551e46369d..244411f2ea49 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTablePropertyTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTablePropertyTest.java @@ -18,6 +18,7 @@ package org.apache.cassandra.repair.autorepair; +import java.util.HashMap; import java.util.Map; import org.junit.Test; @@ -60,7 +61,13 @@ public void helperTestTableProperty(boolean autoRepairOn) { DatabaseDescriptor.getAutoRepairConfig().setAutoRepairSchedulingEnabled(autoRepairOn); - Map systemSchemaTables = Map.of(SchemaKeyspaceTables.TABLES, "table_name", SchemaKeyspaceTables.VIEWS, "view_name"); + Map systemSchemaTables = new HashMap() + { + { + put(SchemaKeyspaceTables.TABLES, "table_name"); + put(SchemaKeyspaceTables.VIEWS, "view_name"); + } + }; for (Map.Entry systemSchema : systemSchemaTables.entrySet()) { ColumnFamilyStore tables = Keyspace.open(SchemaConstants.SCHEMA_KEYSPACE_NAME).getColumnFamilyStore(systemSchema.getKey()); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index 78a922111670..8bb8a6b81573 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -71,6 +71,7 @@ public static void setUpClass() CQLTester.setUpClass(); AutoRepairService.setup(); FULL_RANGE = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMaximumToken()); + requireNetwork(); } @Before @@ -313,9 +314,9 @@ public void testTokenRangesRepairByKeyspace() AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(RepairType.FULL, true); final KeyspaceRepairPlan repairPlan = new KeyspaceRepairPlan("system_auth", AUTH_TABLE_NAMES); - final PrioritizedRepairPlan prioritizedRepairPlan = new PrioritizedRepairPlan(0, List.of(repairPlan)); + final PrioritizedRepairPlan prioritizedRepairPlan = new PrioritizedRepairPlan(0, Arrays.asList(repairPlan)); - Iterator keyspaceAssignments = repairRangeSplitter.getRepairAssignments(true, List.of(prioritizedRepairPlan)); + Iterator keyspaceAssignments = repairRangeSplitter.getRepairAssignments(true, Arrays.asList(prioritizedRepairPlan)); // should be only 1 entry for the keyspace. assertTrue(keyspaceAssignments.hasNext()); @@ -341,9 +342,9 @@ public void testTokenRangesRepairByKeyspaceFalse() AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(RepairType.FULL, false); final KeyspaceRepairPlan repairPlan = new KeyspaceRepairPlan("system_auth", AUTH_TABLE_NAMES); - final PrioritizedRepairPlan prioritizedRepairPlan = new PrioritizedRepairPlan(0, List.of(repairPlan)); + final PrioritizedRepairPlan prioritizedRepairPlan = new PrioritizedRepairPlan(0, Arrays.asList(repairPlan)); - Iterator keyspaceAssignments = repairRangeSplitter.getRepairAssignments(true, List.of(prioritizedRepairPlan)); + Iterator keyspaceAssignments = repairRangeSplitter.getRepairAssignments(true, Arrays.asList(prioritizedRepairPlan)); // should be only 1 entry for the keyspace. assertTrue(keyspaceAssignments.hasNext()); From 7a9d1dd034381ceb236adeeecce423c98aa3c137 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 7 Mar 2025 14:42:25 -0800 Subject: [PATCH 152/199] Avoid micros splitting of token ranges for FixedSplitTokenRangeSplitter --- .../autorepair/FixedSplitTokenRangeSplitter.java | 4 +++- .../autorepair/FixedSplitTokenRangeSplitterTest.java | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index bdcb355e7098..54f9b346c91d 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -100,9 +100,11 @@ private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(boolean primar boolean byKeyspace = config.getRepairByKeyspace(repairType); // collect all token ranges. List> allRanges = new ArrayList<>(); + // this is done to avoid micro splits in the case of vnodes + int splitsPerRange = Math.max(1, numberOfSubranges / tokens.size()); for (Range token : tokens) { - allRanges.addAll(split(token, numberOfSubranges)); + allRanges.addAll(split(token, splitsPerRange)); } if (byKeyspace) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java index dc4ca6c11ace..6cd6ad054383 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java @@ -58,8 +58,11 @@ public class FixedSplitTokenRangeSplitterTest extends CQLTester private static final String TABLE1 = "tbl1"; private static final String TABLE2 = "tbl2"; private static final String TABLE3 = "tbl3"; + private static final int numberOfSubRanges = 4; - private static final Map splitterParams = Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(4)); + private static final int totalTokenRanges = 3; + private static int numberOfSplits; + private static final Map splitterParams = Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRanges)); @Parameterized.Parameter() public AutoRepairConfig.RepairType repairType; @@ -81,16 +84,16 @@ public static void setupClass() throws Exception AutoRepairUtils.setup(); SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1)); + + numberOfSplits = numberOfSubRanges/totalTokenRanges; } @Test public void testTokenRangesSplitByTable() { AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(repairType, false); - int totalTokenRanges = 3; Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); assertEquals(totalTokenRanges, tokens.size()); - int numberOfSplits = 4; List tables = Arrays.asList(TABLE1, TABLE2, TABLE3); List> expectedToken = new ArrayList<>(); for (int i = 0; i> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); assertEquals(totalTokenRanges, tokens.size()); - int numberOfSplits = 4; List tables = Arrays.asList(TABLE1, TABLE2, TABLE3); List> expectedToken = new ArrayList<>(); for (Range range : tokens) @@ -180,7 +181,6 @@ public void testTokenRangesSplitByKeyspace() public void testTokenRangesNoSplitByDefault() { Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); - int totalTokenRanges = 3; assertEquals(totalTokenRanges, tokens.size()); List> expectedToken = new ArrayList<>(tokens); From 05bdb8ebf1778e0e9327301d6199d9ee32766a36 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 7 Mar 2025 18:07:00 -0800 Subject: [PATCH 153/199] Make the number_of_subranges parameterized in test case --- .../FixedSplitTokenRangeSplitterTest.java | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java index 6cd6ad054383..0fd1f6839313 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java @@ -24,8 +24,8 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Map; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -58,19 +58,27 @@ public class FixedSplitTokenRangeSplitterTest extends CQLTester private static final String TABLE1 = "tbl1"; private static final String TABLE2 = "tbl2"; private static final String TABLE3 = "tbl3"; - private static final int numberOfSubRanges = 4; - private static final int totalTokenRanges = 3; private static int numberOfSplits; - private static final Map splitterParams = Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRanges)); - @Parameterized.Parameter() + @Parameterized.Parameter(0) public AutoRepairConfig.RepairType repairType; - @Parameterized.Parameters(name = "repairType={0}") - public static Collection repairTypes() + @Parameterized.Parameter(1) + public int numberOfSubRange; + + @Parameterized.Parameters(name = "repairType={0}, numberOfSubRange={1}") + public static Collection parameters() { - return Arrays.asList(AutoRepairConfig.RepairType.values()); + List params = new ArrayList<>(); + for (AutoRepairConfig.RepairType type : AutoRepairConfig.RepairType.values()) + { + for (int subRange : Arrays.asList(4, 16, 64, 128)) + { + params.add(new Object[]{ type, subRange }); + } + } + return params; } @BeforeClass @@ -84,8 +92,12 @@ public static void setupClass() throws Exception AutoRepairUtils.setup(); SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1)); + } - numberOfSplits = numberOfSubRanges/totalTokenRanges; + @Before + public void before() + { + numberOfSplits = numberOfSubRange / totalTokenRanges; } @Test @@ -106,7 +118,7 @@ public void testTokenRangesSplitByTable() List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); - Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, splitterParams) + Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRange))) .getRepairAssignments(true, plan); // should be only 1 entry for the keyspace. @@ -115,7 +127,7 @@ public void testTokenRangesSplitByTable() assertFalse(keyspaceAssignments.hasNext()); List assignments = keyspace.getRepairAssignments(); - assertEquals(totalTokenRanges*numberOfSplits*tables.size(), assignments.size()); + assertEquals(totalTokenRanges * numberOfSplits * tables.size(), assignments.size()); assertEquals(expectedToken.size(), assignments.size()); int expectedTableIndex = -1; @@ -129,9 +141,9 @@ public void testTokenRangesSplitByTable() expectedTableIndex = -1; // should be a set of ranges for each table. - for (int i = 0; i plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); - Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, splitterParams) + Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRange))) .getRepairAssignments(true, plan); // should be only 1 entry for the keyspace. @@ -166,11 +178,11 @@ public void testTokenRangesSplitByKeyspace() List assignments = keyspace.getRepairAssignments(); assertNotNull(assignments); - assertEquals(totalTokenRanges*numberOfSplits, assignments.size()); + assertEquals(totalTokenRanges * numberOfSplits, assignments.size()); assertEquals(expectedToken.size(), assignments.size()); // should only be one set of ranges for the entire keyspace. - for (int i = 0; i Date: Fri, 7 Mar 2025 23:06:35 -0600 Subject: [PATCH 154/199] Auto Repair Documentation improvements, bytes_per_assignment to 50GiB Updates auto repair documentation and moves it under managing/operating/auto_repair.adoc. Updates existing repair documentation to reference the auto repair documentation. Updates bytes_per_assignment default to 50GiB (from 200GiB) for all report types to be more conservative. Clean up some syntactial issues and formatting in cassandra.yaml files. patch by Andy Tolbert; reviewed by ____ for CASSANDRA-20421 --- conf/cassandra.yaml | 69 ++- .../managing/operating/auto_repair.adoc | 410 ++++++++++++++++++ doc/modules/cassandra/nav.adoc | 2 +- .../cassandra/pages/auto-repair/overview.adoc | 211 --------- .../cassandra/pages/operating/repair.adoc | 24 +- .../repair/autorepair/AutoRepairConfig.java | 9 +- .../autorepair/RepairTokenRangeSplitter.java | 147 ++----- .../autorepair/AutoRepairConfigTest.java | 13 +- 8 files changed, 518 insertions(+), 367 deletions(-) create mode 100644 doc/modules/cassandra/managing/operating/auto_repair.adoc delete mode 100644 doc/modules/cassandra/pages/auto-repair/overview.adoc diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 89aa58714fd5..1a3408a62b14 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1870,9 +1870,15 @@ drop_compact_storage_enabled: false # incremental repair as anti-compaction during incremental repair may contribute to additional space temporarily. # if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason) # then set the ratio to 0.0 -# incremental_repair_disk_headroom_reject_ratio = 0.2; +# incremental_repair_disk_headroom_reject_ratio: 0.2; -# Configuration for AutoRepair Scheduler. +# Configuration for Auto Repair Scheduler. +# +# This feature is disabled by default. +# +# See: https://cassandra.apache.org/doc/latest/cassandra/managing/operating/auto_repair.html for an overview of this +# feature. +# # auto_repair: # # Enable/Disable the auto-repair scheduler. # # If set to false, the scheduler thread will not be started. @@ -1884,17 +1890,33 @@ drop_compact_storage_enabled: false # full: # # Enable/Disable full auto-repair # enabled: true -# # Minimum duration between repairing the same node again. This is useful for tiny clusters, such as -# # clusters with 5 nodes that finish repairs quickly. The default is 24 hours. This means that if the scheduler -# # completes one round on all nodes in less than 24 hours, it will not start a new repair round on a given node -# # until 24 hours have passed since the last repair. +# # Minimum duration between repairing the same node again. This is useful for tiny clusters, +# # such as clusters with 5 nodes that finish repairs quickly. This means that if the scheduler completes one +# # round on all nodes in less than this duration, it will not start a new repair round on a given node until +# # this much time has passed since the last repair completed. Consider increasing to a larger value to reduce +# # the impact of repairs, however note that one should attempt to run repairs at a smaller interval than +# # gc_grace_seconds to avoid zombies. # min_repair_interval: 24h # token_range_splitter: +# # Implementation of IAutoRepairTokenRangeSplitter; responsible for splitting token ranges +# # for repair assignments. +# # +# # Out of the box, Cassandra provides org.apache.cassandra.repair.autorepair.{RepairTokenRangeSplitter, +# # FixedTokenRangeSplitter}. +# # +# # - RepairTokenRangeSplitter (default) attempts to intelligently split ranges based on data size and partition +# # count. +# # - FixedTokenRangeSplitter splits into fixed ranges based on the 'number_of_subranges' option. +# # class_name: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter +# +# # Optional parameters can be specified in the form of: +# # parameters: +# # param_key1: param_value1 # parameters: # # The target and maximum amount of bytes that should be included in a repair # # assignment. This scopes the amount of work involved in a repair and includes # # the data covering the range being repaired. -# bytes_per_assignment: 200GiB +# bytes_per_assignment: 50GiB # # The maximum number of bytes to cover in an individual schedule. This serves as # # a mechanism to throttle the work done in each repair cycle. You may reduce this # # value if the impact of repairs is causing too much load on the cluster or increase it @@ -1903,29 +1925,32 @@ drop_compact_storage_enabled: false # # This is set to a large value for full repair to attempt to repair all data per repair schedule. # max_bytes_per_schedule: 100000GiB # incremental: -# enabled: true -# # Incremental repairs operate over unrepaired data and should finish quickly. Running them more frequently keeps -# # the unrepaired set smaller and thus causes repairs to operate over a smaller set of data, so a more frequent -# # schedule such as 1h is recommended. -# # When turning on incremental repair for the first time with a decent amount of data it may be advisable to -# # increase this interval to 24h or longer to reduce the impact of anticompaction caused by incremental repair. -# min_repair_interval: 1h +# enabled: false +# # Incremental repairs operate over unrepaired data and should finish quickly. Running incremental repair +# # frequently keeps the unrepaired set smaller and thus causes repairs to operate over a smaller set of data, +# # so a more frequent schedule such as 1h is recommended. +# # NOTE: When turning on incremental repair for the first time with a decent amount of data it is adviable to +# # use a larger interval such as 24h or longer to reduce the impact of anticompaction caused by incremental +# # repair. Once a majority of data has been repaired, one may consider reducing this to 1h to keep to unrepaired +# # set small. +# min_repair_interval: 24h # token_range_splitter: # parameters: # # Configured to attempt repairing 50GiB of data per repair. -# # This throttles the amount of incremental repair and anticompaction done per schedule -# # after incremental repairs are turned on. +# # This throttles the amount of incremental repair and anticompaction done per schedule after incremental +# # repairs are turned on. # bytes_per_assignment: 50GiB -# # The maximum number of bytes to cover in an individual schedule to 100GiB. -# # Consider increasing max_bytes_per_schedule if -# # more data is written than this limit within the min_repair_interval. +# # Resricts the maximum number of bytes to cover in an individual schedule to 100GiB. +# # Consider increasing this if more data is written than this limit within the min_repair_interval. # max_bytes_per_schedule: 100GiB # preview_repaired: -# enabled: true +# # Performs preview repair over repaired SSTables, useful to detect possible inconsistencies in the repaired +# # data set. +# enabled: false # min_repair_interval: 24h # token_range_splitter: # parameters: -# bytes_per_assignment: 200GiB +# bytes_per_assignment: 50GiB # max_bytes_per_schedule: 100000GiB # # Time interval between successive checks to see if ongoing repairs are complete or if it is time to schedule # # repairs. @@ -1951,7 +1976,7 @@ drop_compact_storage_enabled: false # # Number of nodes running repair in parallel. If parallel_repair_percentage is set, the larger value is used. # parallel_repair_count: 3 # # Percentage of nodes in the cluster running repair in parallel. If parallel_repair_count is set, the larger value -# # is used. Recommendation is that the repair cycle on the cluster should finish within gc_grace_seconds. +# # is used. # parallel_repair_percentage: 3 # # Repairs materialized views if true. # materialized_view_repair_enabled: false diff --git a/doc/modules/cassandra/managing/operating/auto_repair.adoc b/doc/modules/cassandra/managing/operating/auto_repair.adoc new file mode 100644 index 000000000000..db50a8c4524b --- /dev/null +++ b/doc/modules/cassandra/managing/operating/auto_repair.adoc @@ -0,0 +1,410 @@ += Auto Repair +:navtitle: Auto Repair +:description: Auto Repair concepts - How it works, how to configure it, and more. +:keywords: CEP-37, Repair, Incremental, Preview + +Auto Repair is a fully automated scheduler that provides repair orchestration within Apache Cassandra. This +significantly reduces operational overhead by eliminating the need for operators to deploy external tools to submit and +manage repairs. + +At a high level, a dedicated thread pool is assigned to the repair scheduler. The repair scheduler in Cassandra +maintains a new replicated table, `system_distributed.auto_repair_history`, which stores the repair history for all +nodes, including details such as the last repair time. The scheduler selects the node(s) to begin repairs and +orchestrates the process to ensure that every table and its token ranges are repaired. + +The algorithm can run repairs simultaneously on multiple nodes and splits token ranges into subranges, with necessary +retries to handle transient failures. Automatic repair starts as soon as a Cassandra cluster is launched, similar to +compaction, and if configured appropriately, does not require human intervention. + +The scheduler currently supports Full, Incremental, and Preview repair types with the following features. New repair +types, such as Paxos repair or other future repair mechanisms, can be integrated with minimal development effort! + + +== Features +- Capability to run repairs on multiple nodes simultaneously. +- A default implementation and an interface to override the dataset being repaired per session. +- Extendable token split algorithms with two implementations readily available: +. Splits token ranges by placing a cap on the size of data repaired in one session and a maximum cap at the schedule +level using xref:#repair-token-range-splitter[RepairTokenRangeSplitter] (default). +. Splits tokens evenly based on the specified number of splits using +xref:#fixed-split-token-range-splitter[FixedSplitTokenRangeSplitter]. +- A new xref:#table-configuration[CQL table property] (`auto_repair`) offering: +. The ability to disable specific repair types at the table level, allowing the scheduler to skip one or more tables. +. Configuring repair priorities for certain tables to prioritize them over others. +- Dynamic enablement or disablement of the scheduler for each repair type. +- Configurable settings tailored to each repair job. +- Rich configuration options for each repair type (e.g., Full, Incremental, or Preview repairs). +- Comprehensive observability features that allow operators to configure alarms as needed. + +== Considerations + +Before enabling auto repair, please consult the xref:managing/operating/repair.adoc[Repair] guide to establish a base +understanding of repairs. + +=== Full Repair + +Full Repairs operate over all data in the token range being repaired. It is therefore important to run full repair +with a longer schedule and with smaller assignments. + +=== Incremental Repair + +When enabled from the inception of a cluster, incremental repairs operate over unrepaired data and should finish +quickly when run more frequently. + +Once incremental repair has been run, SSTables will be separated between data that have been incrementally repaired +and data that have not. Therefore, it is important to continually run incremental repair once it has been enabled so +newly written data can be compacted together with previously repaired data, allowing overwritten and expired data to +be eventually purged. + +Running incremental repair more frequently keeps the unrepaired set smaller and thus causes repairs to operate over +a smaller set of data, so a shorter `min_repair_interval` such as `1h` is recommended for new clusters. + +NOTE: When turning on incremental repair for the first time on an existing cluser with a large amount of data it is +advisable to use a larger interval such as `24h` or longer to reduce the impact of anticompaction caused by incremental +repair. Once the entire token range has been repaired, one may consider reducing this to a shorter interval. It is +strongly recommended to not enable incremental repair for an existing cluster unless full repair has previously +actively run. + +=== Previewing Repaired Data + +The `preview_repaired` repair type executes repairs over the repaired data set to detect possible data inconsistencies. + +Inconsistencies in the repaired data set should not happen in practice and could indicate a possible bug in incremental +repair. + +Running preview repairs is useful when considering using the +xref:cassandra:managing/operating/compaction/tombstones.adoc#deletion[only_purge_repaired_tombstones] table compaction +option to prevent data from possibly being resurrected when inconsistent replicas are missing tombstones from deletes. + +When enabled, the `BytesPreviewedDesynchronized` and `TokenRangesPreviewedDesynchronized` +xref:cassandra:managing/operating/metrics.adoc#table-metrics[table metrics] can be used to detect inconsistencies in the +repaired data set. + +== Configuring auto repair in cassandra.yaml + +Configuration for auto repair is managed in the `cassandra.yaml` file by the `auto_repair` property. + +A rich set of configuration exists for configuring auto repair with sensible defaults. However, the expectation +is that some tuning might be needed particulary when it comes to tuning how often repair should run +(`min_repair_interval`) and how repair assignments as created (`token_range_splitter`). + +The following is a practical example of an auto_repair configuration that one might use. + +[source, yaml] +---- +auto_repair: + enabled: true + repair_type_overrides: + full: + enabled: true + min_repair_interval: 5d + incremental: + enabled: true + min_repair_interval: 1h + token_range_splitter: + parameters: + bytes_per_assignment: 50GiB + max_bytes_per_schedule: 100GiB + preview_repaired: + enabled: true + min_repair_interval: 1d + global_settings: + repair_by_keyspace: true + parallel_repair_count: 1 +---- + + +=== Top level settings +The following settings are defined at the top level of the configuration file and apply universally across all +repair types. + +[cols=",,",options="header",] +|=== +| Name | Default | Description +| enabled | false | Enable/Disable the auto-repair scheduler. If set to false, the scheduler thread will not be started. +If set to true, the repair scheduler thread will be created. The thread will check for secondary configuration available +for each repair type (full, incremental, and preview_repaired), and based on that, it will schedule repairs. +| repair_check_interval | 5m | Time interval between successive checks to see if ongoing repairs are complete or if it +is time to schedule repairs. +| repair_max_retries | 3 | Maximum number of retries for a repair session. +| repair_retry_backoff | 30s | Backoff time before retrying a repair session. +| repair_task_min_duration | 5s | Minimum duration for the execution of a single repair task. This prevents the +scheduler from overwhelming the node by scheduling too many repair tasks in a short period of time. +| history_clear_delete_hosts_buffer_interval | 2h | The scheduler needs to adjust its order when nodes leave the ring. +Deleted hosts are tracked in metadata for a specified duration to ensure they are indeed removed before adjustments +are made to the schedule. +|=== + + +=== Repair level settings +The following settings can be configured globally using `global_settings` or tailored individually for each repair +type by using `repair_type_overrides`. + +[cols=",,",options="header",] +|=== +| Name | Default | Description +| enabled | false | Whether the given repair types should be enabled +| min_repair_interval | 24h | Minimum duration between repairing the same node again. This is useful for tiny clusters, +such as clusters with 5 nodes that finish repairs quickly. This means that if the scheduler completes one round on all +nodes in less than this duration, it will not start a new repair round on a given node until this much time has +passed since the last repair completed. Consider increasing to a larger value to reduce the impact of repairs, +however *note that one should attempt to run repairs at a smaller interval than gc_grace_seconds to +avoid xref:cassandra:managing/operating/compaction/tombstones.adoc#zombies[zombies]*. +| class_name | org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter | Implementation of +IAutoRepairTokenRangeSplitter to use; responsible for splitting token ranges for repair assignments. Out of the box, +Cassandra provides org.apache.cassandra.repair.autorepair.{RepairTokenRangeSplitter,FixedTokenRangeSplitter}. +| repair_by_keyspace | true | If true, attempts to group tables in the same keyspace into one repair; otherwise, +each table is repaired individually. +| number_of_repair_threads | 1 | Number of threads to use for each repair job scheduled by the scheduler. Similar to +the -j option in nodetool repair. +| parallel_repair_count | 3 | Number of nodes running repair in parallel. If `parallel_repair_percentage` is set, the +larger value is used. +| parallel_repair_percentage | 3 | Percentage of nodes in the cluster running repair in parallel. If +`parallel_repair_count is set`, the larger value is used. +| materialized_view_repair_enabled | false | Repairs materialized views if true. +| initial_scheduler_delay | 5m | Delay before starting repairs after a node restarts to avoid repairs starting +immediately after a restart. +| repair_session_timeout | 3h | Timeout for resuming stuck repair sessions. +| force_repair_new_node | false | Force immediate repair on new nodes after they join the ring. +| sstable_upper_threshold | 10000 | Threshold to skip repairing tables with too many SSTables. +| table_max_repair_time | 6h | Maximum time allowed for repairing one table on a given node. If exceeded, the repair +proceeds to the next table. +| ignore_dcs | [] | Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify +data centers to exclude in this list. Note that repair sessions will still consider all replicas from excluded data +centers. Useful if you have keyspaces that are not replicated in certain data centers, and you want to not run repair +schedule in certain data centers. +| repair_primary_token_range_only | true | Repair only the primary ranges owned by a node. Equivalent to the -pr option +in nodetool repair. General advice is to keep this true. +|=== + +=== `RepairTokenRangeSplitter` configuration +[#repair-token-range-splitter] + +`RepairTokenRangeSplitter` is the default implementation of `IAutoRepairTokenRangeSplitter` that attempts to create +token range assignments meeting the following goals: + +1. *Create smaller, consistent repair times*: Long repairs, such as those lasting 15 hours, can be problematic. If a +node fails 14 hours into the repair, the entire process must be restarted. The goal is to reduce the impact of +disturbances or failures. However, making the repairs too short can lead to overhead from repair orchestration becoming +the main bottleneck. + +2. *Minimize the impact on hosts*: Repairs should not heavily affect the host systems. For incremental repairs, this +might involve anti-compaction work. In full repairs, streaming large amounts of data—especially with wide partitions +can lead to issues with disk usage and higher compaction costs. + +3. *Reduce overstreaming*: The Merkle tree, which represents data within each partition and range, has a maximum size. +If a repair covers too many partitions, the tree’s leaves represent larger data ranges. Even a small change in a leaf +can trigger excessive data streaming, making the process inefficient. + +4. *Reduce number of repairs*: If there are many small tables, it's beneficial to batch these tables together under a +single parent repair. This prevents the repair overhead from becoming a bottleneck, especially when dealing with +hundreds of tables. Running individual repairs for each table can significantly impact performance and efficiency. + +To achieve these goals, this implementation inspects SSTable metadata to estimate the bytes and number of partitions +within a range and splits it accordingly to bound the size of the token ranges used for repair assignments. + +==== Parameter defaults + +The following `parameters` include the same defaults for all repair types. + +[cols=",,",options="header",] +|=== +| Name | Default | Description +| partitions_per_assignment | 1048576 | Maximum number of partitions to include in a repair +assignment. Used to reduce number of partitions present in merkle tree leaf nodes to avoid overstreaming. +| max_tables_per_assignment | 64 | Maximum number of tables to include in a repair assignment. +This reduces the number of repairs, especially in keyspaces with many tables. The splitter avoids batching tables +together if they exceed other configuration parameters like `bytes_per_assignment` or `partitions_per_assignment`. +|=== + +==== Full & Preview Repaired repair defaults + +The following `parameters` defaults are established for both `full` and `preview_repaired` repair scheduling: + +[cols=",,",options="header",] +|=== +| Name | Default | Description +| bytes_per_assignment | 50GiB | The target and maximum amount of *compressed* bytes that should be included in a +repair assignment. *Note*: For full and preview_repaired, only the portion of an SSTable that covers the ranges +being repaired are accounted for in this calculation. +| max_bytes_per_schedule | 100000GiB | The maximum number of bytes to cover in an individual +schedule. This serves as a mechanism to throttle the work done in each repair cycle. You may reduce this value if the +impact of repairs is causing too much load on the cluster or increase it if writes outpace the amount of data being +repaired. Alternatively, adjust the `min_repair_interval`. This is set to a large value for full repair to attempt to +repair all data per repair schedule. +|=== + +==== Incremental repair defaults + +The following `parameters` defaults are established for `incremental` repair scheduling: + +[cols=",,",options="header",] +|=== +| Name | Default | Description +| bytes_per_assignment | 50GiB | The target and maximum amount of *compressed* bytes that should be +included in a repair assignment. *Note*: For incremental repair, the *entire size* of *unrepaired* SSTables +including ranges being repaired are accounted for in this calculation. This is to account for the anticompaction +work required to split the candidate data to repair from the data that won't be repaired. +| max_bytes_per_schedule | 100GiB | The maximum number of bytes to cover in an individual schedule. +Consider increasing if more data is written than this limit within the `min_repair_interval`. +|=== + +=== `FixedSplitTokenRangeSplitter` configuration +[#fixed-split-token-range-splitter] + +`FixedSplitTokenRangeSplitter` is a more simple implementation of `IAutoRepairTokenRangeSplitter` that creates repair +assignments by splitting a node's token ranges into an even number of splits. + +The following `parameters` apply for `FixedSplitTokenRangeSplitter` configuration: + +[cols=",,",options="header",] +|=== +| Name | Default | Description +| number_of_subranges | 64 | Number of evenly split subranges to create for each node that repair runs for. +If vnodes are configured using `num_tokens`, attempts to evenly subdivide subranges by each range. For example, for +`num_tokens: 16` and `number_of_subranges: 64`, 4 (64/16) repair assignments will be created for each token range. At +least one repair assignment will be created for each token range. +|=== + +=== Other cassandra.yaml Considerations + +==== Enable `reject_repair_compaction_threshold` + +When enabling auto_repair, it is advisable to configure the top level `reject_repair_compaction_threshold` +configuration in cassandra.yaml as a backpressure mechanism to reject new repairs on instances that have many +pending compactions. + +==== Tune `incremental_repair_disk_headroom_reject_ratio` + +By default, incremental repairs will be rejected if less than 20% of disk is available. If one wishes to be +conservative this top level configuration could be increased to a larger value to prevent filling your data directories. + +== Table configuration + +If auto repair is enabled in cassandra.yaml, the `auto_repair` property may be optionally configured at the table +level, e.g.: + +[source,cql] +---- +ALTER TABLE cycling.cyclist_races +WITH auto_repair = {'incremental_enabled': 'false', 'priority': '0'}; +---- + +[cols=",,",options="header",] +|=== +| Name | Default | Description +| priority | 0 | Indicates the priority at which this table should be given when issuing repairs. The higher the number +the more priority will be given to repair the table (e.g. 3 will be repaired before 2). When `repair_by_keyspace` is +set to `true` tables sharing the same priority may be grouped in the same repair assignment. +| full_enabled | true | Whether full repair is enabled for this table. If full.enabled is not true in cassandra.yaml +this will not be evaluated. +| incremental_enabled | true | Whether incremental repair is enabled for this table. If incremental.enabled is not +true in cassandra.yaml this will not be evaluated. +| preview_repaired_enabled | true | Whether preview repair is enabled for this table. If preview_repaired.enabled is +not true in cassandra.yaml this will not be evaluated. +|=== + +== Nodetool Configuration +=== nodetool getautorepairconfig + +Retrieves the runtime configuration of Auto Repair for the targeted node. + +[source,none] +---- +$> nodetool getautorepairconfig +repair scheduler configuration: + repair_check_interval: 5m + repair_max_retries: 3 + repair_retry_backoff: 30s + repair_task_min_duration: 5s + history_clear_delete_hosts_buffer_interval: 2h +configuration for repair_type: full + enabled: true + min_repair_interval: 24h + repair_by_keyspace: true + number_of_repair_threads: 1 + sstable_upper_threshold: 10000 + table_max_repair_time: 6h + ignore_dcs: [] + repair_primary_token_range_only: true + parallel_repair_count: 3 + parallel_repair_percentage: 3 + materialized_view_repair_enabled: false + initial_scheduler_delay: 5m + repair_session_timeout: 3h + force_repair_new_node: false + token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter + token_range_splitter.bytes_per_assignment: 50GiB + token_range_splitter.partitions_per_assignment: 1048576 + token_range_splitter.max_tables_per_assignment: 64 + token_range_splitter.max_bytes_per_schedule: 100000GiB +configuration for repair_type: incremental + enabled: true + min_repair_interval: 1h + repair_by_keyspace: true + number_of_repair_threads: 1 + sstable_upper_threshold: 10000 + table_max_repair_time: 6h + ignore_dcs: [] + repair_primary_token_range_only: true + parallel_repair_count: 3 + parallel_repair_percentage: 3 + materialized_view_repair_enabled: false + initial_scheduler_delay: 5m + repair_session_timeout: 3h + force_repair_new_node: false + token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter + token_range_splitter.bytes_per_assignment: 50GiB + token_range_splitter.partitions_per_assignment: 1048576 + token_range_splitter.max_tables_per_assignment: 64 + token_range_splitter.max_bytes_per_schedule: 100GiB +configuration for repair_type: preview_repaired + enabled: false +---- + +=== nodetool autorepairstatus + +Provides currently running Auto Repair status. + +[source,none] +---- +$> nodetool autorepairstatus -t incremental +Active Repairs +425cea55-09aa-46e0-8911-9f37a4424574 + + +$> nodetool autorepairstatus -t full +Active Repairs +NONE + +---- + +=== nodetool setautorepairconfig + +Dynamic configuration changes can be made by using `setautorepairconfig`. Note that this only applies on the node being +targeted and these changes are not retained when a node is bounced. + +The following disables the `incremental` repair schedule: + +[source,none] +---- +$> nodetool setautorepairconfig -t incremental enabled false +---- + +The following adjusts the `min_repair_interval` option to `5d` specifically for the `full` repair schedule: + +[source,none] +---- +$> nodetool setautorepairconfig -t full min_repair_interval 5d +---- + +The following configures the `bytes_per_assignment` parameter for `incremental` repair's `token_range_splitter` to +`10GiB`: + +[source,none] +---- +$> nodetool setautorepairconfig -t incremental token_range_splitter.bytes_per_assignment 10GiB +---- + +==== More details +https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-37+Apache+Cassandra+Unified+Repair+Solution[CEP-37] diff --git a/doc/modules/cassandra/nav.adoc b/doc/modules/cassandra/nav.adoc index 4849abb7daaf..93f5ee2596fa 100644 --- a/doc/modules/cassandra/nav.adoc +++ b/doc/modules/cassandra/nav.adoc @@ -81,6 +81,7 @@ *** xref:tools/nodetool/nodetool.adoc[nodetool] *** xref:tools/sstable/index.adoc[SSTable tools] *** xref:tools/cassandra_stress.adoc[cassandra-stress] +**** xref:cassandra:managing/operating/auto_repair.adoc[Auto Repair] ** xref:troubleshooting/index.adoc[Troubleshooting] *** xref:troubleshooting/finding_nodes.adoc[Finding misbehaving nodes] @@ -104,4 +105,3 @@ ** xref:faq/index.adoc[FAQ] ** xref:plugins/index.adoc[Plug-ins] - diff --git a/doc/modules/cassandra/pages/auto-repair/overview.adoc b/doc/modules/cassandra/pages/auto-repair/overview.adoc deleted file mode 100644 index 698e53c44677..000000000000 --- a/doc/modules/cassandra/pages/auto-repair/overview.adoc +++ /dev/null @@ -1,211 +0,0 @@ -= Overview of Auto Repair -:navtitle: Auto Repair overview -:description: Auto Repair concepts - How it works, how to configure it, and more. -:keywords: CEP-37 - -Auto Repair is a fully automated scheduler that provides repair orchestration within Apache Cassandra. This -significantly reduces operational overhead by eliminating the need for operators to deploy external tools to submit and -manage repairs. - -At a high level, a dedicated thread pool is assigned to the repair scheduler. The repair scheduler in Cassandra -maintains a new replicated table, system_distributed.auto_repair_history, which stores the repair history for all nodes, -including details such as the last repair time. The scheduler selects the node(s) to begin repairs and orchestrates the -process to ensure that every table and its token ranges are repaired. The algorithm can run repairs simultaneously on -multiple nodes and splits token ranges into subranges, with necessary retries to handle transient failures. Automatic -repair starts as soon as a Cassandra cluster is launched, similar to compaction, and does not require human -intervention. - -The scheduler currently supports Full, Incremental, and Preview repair types with the following features. New repair -types, such as Paxos repair or other future repair mechanisms, can be integrated with minimal development effort! - - -=== Features -- Capability to run repairs on multiple nodes simultaneously. -- A default implementation and an interface to override the dataset being repaired per session. -- Extendable token split algorithms with two implementations readily available: -. Splits token ranges by placing a cap on the size of data repaired in one session and a maximum cap at the schedule -level (default). -. Splits tokens evenly based on the specified number of splits. -- A new CQL table property offering: -. The ability to disable specific repair types at the table level, allowing the scheduler to skip one or more tables. -. Configuring repair priorities for certain tables to prioritize them over others. -- Dynamic enablement or disablement of the scheduler for each repair type. -- Configurable settings tailored to each repair job. -- Rich configuration options for each repair type (e.g., Full, Incremental, or Preview repairs). -- Comprehensive observability features that allow operators to configure alarms as needed. - -=== Yaml Configuration - -==== Top level settings -The following settings are defined at the top level of the configuration file and apply universally across all -repair types. - -[cols=",,",options="header",] -|=== -| Name | Default | Description -| enabled | false | Enable/Disable the auto-repair scheduler. If set to false, the scheduler thread will not be started. -If set to true, the repair scheduler thread will be created. The thread will check for secondary configuration available -for each repair type (full, incremental, and preview_repaired), and based on that, it will schedule repairs. -| repair_check_interval | 5m | Time interval between successive checks to see if ongoing repairs are complete or if it -is time to schedule repairs. -| repair_max_retries | 3 | Maximum number of retries for a repair session. -| repair_retry_backoff | 30s | Backoff time before retrying a repair session. -| repair_task_min_duration | 5s | Minimum duration for the execution of a single repair task. This prevents the -scheduler from overwhelming the node by scheduling too many repair tasks in a short period of time. -| history_clear_delete_hosts_buffer_interval | 2h | The scheduler needs to adjust its order when nodes leave the ring. -Deleted hosts are tracked in metadata. for a specified duration to ensure they are indeed removed before adjustments -are made to the schedule. -|=== - - -==== Repair level settings -The following settings can be tailored individually for each repair type. -[cols=",,",options="header",] -|=== -| Name | Default | Description -| enabled | false | Enable/Disable full/incremental/preview_repaired auto-repair -| repair_by_keyspace | true | If true, attempts to group tables in the same keyspace into one repair; otherwise, -each table is repaired individually. -| number_of_repair_threads | 1 | Number of threads to use for each repair job scheduled by the scheduler. Similar to -the -j option in nodetool repair. -| parallel_repair_count | 3 | Number of nodes running repair in parallel. If parallel_repair_percentage is set, the -larger value is used. -| parallel_repair_percentage | 3 | Percentage of nodes in the cluster running repair in parallel. If -parallel_repair_count is set, the larger value is used. Recommendation is that the repair cycle on the cluster should -finish within gc_grace_seconds. -| materialized_view_repair_enabled | false | Repairs materialized views if true. -| initial_scheduler_delay | 5m | Delay before starting repairs after a node restarts to avoid repairs starting -immediately after a restart. -| repair_session_timeout | 3h | Timeout for resuming stuck repair sessions. -| force_repair_new_node | false | Force immediate repair on new nodes after they join the ring. -| sstable_upper_threshold | 10000 | Threshold to skip repairing tables with too many SSTables. Defaults to 10,000 -SSTables to avoid penalizing good tables. -| table_max_repair_time | 6h | Maximum time allowed for repairing one table on a given node. If exceeded, the repair -proceeds to the next table. -| ignore_dcs | [] | Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify -data centers to exclude in this list. Note that repair sessions will still consider all replicas from excluded data -centers. Useful if you have keyspaces that are not replicated in certain data centers, and you want to not run repair -schedule in certain data centers. -| repair_primary_token_range_only | true | Repair only the primary ranges owned by a node. Equivalent to the -pr option -in nodetool repair. Defaults to true. General advice is to keep this true. -| token_range_splitter | org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter | Splitter implementation to -generate repair assignments. Defaults to RepairTokenRangeSplitter. -| token_range_splitter.partitions_per_assignment | 1048576 | Maximum number of partitions to include in a repair -assignment. Used to reduce number of partitions present in merkle tree leaf nodes to avoid overstreaming. -| token_range_splitter.max_tables_per_assignment | 64 | Maximum number of tables to include in a repair assignment. -This reduces the number of repairs, especially in keyspaces with many tables. The splitter avoids batching tables -together if they exceed other configuration parameters like bytes_per_assignment or partitions_per_assignment. -|=== - -==== Full & Preview_Repaired repair level settings -The following settings can be tailored individually for each repair type. -[cols=",,",options="header",] -|=== -| min_repair_interval | 24h | Minimum duration between repairing the same node again. This is useful for tiny clusters, -such as clusters with 5 nodes that finish repairs quickly. The default is 24 hours. This means that if the scheduler -completes one round on all nodes in less than 24 hours, it will not start a new repair round on a given node until 24 -hours have passed since the last repair. -| token_range_splitter.bytes_per_assignment | 200GiB | The target and maximum amount of bytes that should be included -in a repair assignment. This scopes the amount of work involved in a repair and includes the data covering the range -being repaired. -| token_range_splitter.max_bytes_per_schedule | 100000GiB | The maximum number of bytes to cover in an individual -schedule. This serves as a mechanism to throttle the work done in each repair cycle. You may reduce this value if the -impact of repairs is causing too much load on the cluster or increase it if writes outpace the amount of data being -repaired. Alternatively, adjust the min_repair_interval. This is set to a large value for full repair to attempt to -repair all data per repair schedule. -|=== - -==== Incremental repair level settings -The following settings can be customized for each repair type. -[cols=",,",options="header",] -|=== -| min_repair_interval | 1h | Incremental repairs operate over unrepaired data and should finish quickly. Running them -more frequently keeps the unrepaired set smaller and thus causes repairs to operate over a smaller set of data, -so a more frequent schedule such as 1h is recommended. When turning on incremental repair for the first time with a -decent amount of data it may be advisable to increase this interval to 24h or longer to reduce the impact of -anticompaction caused by incremental repair. -| token_range_splitter.bytes_per_assignment | 50GiB | Configured to attempt repairing 50GiB of data per repair. -This throttles the amount of incremental repair and anticompaction done per schedule after incremental repairs are -turned on. -| token_range_splitter.max_bytes_per_schedule | 100GiB | The maximum number of bytes to cover in an individual schedule -to 100GiB. Consider increasing max_bytes_per_schedule if more data is written than this limit within -the min_repair_interval. -|=== - - -=== Nodetool Configuration -==== nodetool getautorepairconfig -``` -$> nodetool getautorepairconfig -repair scheduler configuration: - repair_check_interval: 5m - repair_max_retries: 3 - repair_retry_backoff: 30s - repair_task_min_duration: 5s - history_clear_delete_hosts_buffer_interval: 2h -configuration for repair_type: full - enabled: true - min_repair_interval: 24h - repair_by_keyspace: true - number_of_repair_threads: 1 - sstable_upper_threshold: 10000 - table_max_repair_time: 6h - ignore_dcs: [] - repair_primary_token_range_only: true - parallel_repair_count: 3 - parallel_repair_percentage: 3 - materialized_view_repair_enabled: false - initial_scheduler_delay: 5m - repair_session_timeout: 3h - force_repair_new_node: false - token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter - token_range_splitter.bytes_per_assignment: 200GiB - token_range_splitter.partitions_per_assignment: 1048576 - token_range_splitter.max_tables_per_assignment: 64 - token_range_splitter.max_bytes_per_schedule: 100000GiB -configuration for repair_type: incremental - enabled: true - min_repair_interval: 1h - repair_by_keyspace: true - number_of_repair_threads: 1 - sstable_upper_threshold: 10000 - table_max_repair_time: 6h - ignore_dcs: [] - repair_primary_token_range_only: true - parallel_repair_count: 3 - parallel_repair_percentage: 3 - materialized_view_repair_enabled: false - initial_scheduler_delay: 5m - repair_session_timeout: 3h - force_repair_new_node: false - token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter - token_range_splitter.bytes_per_assignment: 50GiB - token_range_splitter.partitions_per_assignment: 1048576 - token_range_splitter.max_tables_per_assignment: 64 - token_range_splitter.max_bytes_per_schedule: 100GiB -configuration for repair_type: preview_repaired - enabled: false -``` - -==== nodetool autorepairstatus -``` -$> nodetool autorepairstatus -t incremental -Active Repairs -425cea55-09aa-46e0-8911-9f37a4424574 - - -$> nodetool autorepairstatus -t full -Active Repairs -NONE - -``` - -==== nodetool setautorepairconfig -``` -$> nodetool setautorepairconfig -t incremental number_of_repair_threads 2 -``` - - - -==== More details -https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-37+Apache+Cassandra+Unified+Repair+Solution[CEP-37] \ No newline at end of file diff --git a/doc/modules/cassandra/pages/operating/repair.adoc b/doc/modules/cassandra/pages/operating/repair.adoc index 3c6b20d3a995..22745dc16617 100644 --- a/doc/modules/cassandra/pages/operating/repair.adoc +++ b/doc/modules/cassandra/pages/operating/repair.adoc @@ -29,10 +29,18 @@ for syncing up missed writes, but it doesn't protect against things like disk corruption, data loss by operator error, or bugs in Cassandra. For this reason, full repairs should still be run occasionally. -== Usage and Best Practices +== Automated Repair Scheduling -Since repair can result in a lot of disk and network io, it's not run -automatically by Cassandra. It is run by the operator via nodetool. +Since repair can result in a lot of disk and network io, it has +traditionally not been run automatically by Cassandra. + +In the latest version of Cassandra, a new feature called +xref:managing/operating/auto_repair.adoc[auto repair] was introduced to +allow Cassandra to run repairs automatically on a schedule. + +== Submitting Repairs Using Nodetool + +Repairs can also be run by the operator via nodetool. Incremental repair is the default and is run with the following command: @@ -62,13 +70,9 @@ Or even on specific tables: nodetool repair [options] ---- -The repair command only repairs token ranges on the node being repaired, -it doesn't repair the whole cluster. By default, repair will operate on -all token ranges replicated by the node you're running repair on, which -will cause duplicate work if you run it on every node. The `-pr` flag -will only repair the "primary" ranges on a node, so you can repair your -entire cluster by running `nodetool repair -pr` on each node in a single -datacenter. +The repair command repairs token ranges only on the node being repaired; it does not repair the whole cluster. +By default, repair operates on all token ranges replicated by the node on which repair is run, causing duplicate work when running it on every node. Avoid duplicate work by using the `-pr` flag to repair only the "primary" ranges on a node. +Do a full cluster repair by running the `nodetool repair -pr` command on each node in each datacenter in the cluster, until all of the nodes and datacenters are repaired. The specific frequency of repair that's right for your cluster, of course, depends on several factors. However, if you're just starting out diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 33b4410f60a5..1b1596f1a0cf 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -411,14 +411,6 @@ private static Map initializeDefaultOption options.put(RepairType.INCREMENTAL, getDefaultOptions()); options.put(RepairType.PREVIEW_REPAIRED, getDefaultOptions()); - - options.get(RepairType.FULL).min_repair_interval = new DurationSpec.IntSecondsBound("24h"); - // Incremental repairs operate over unrepaired data and should finish quickly. Running them more frequently keeps - // the unrepaired set smaller and thus causes repairs to operate over a smaller set of data, so a more frequent - // increase this interval to 24h or longer to reduce the impact of anticompaction caused by incremental repair. - options.get(RepairType.INCREMENTAL).min_repair_interval = new DurationSpec.IntSecondsBound("1h"); - options.get(RepairType.PREVIEW_REPAIRED).min_repair_interval = new DurationSpec.IntSecondsBound("24h"); - return options; } @@ -460,6 +452,7 @@ protected static Options getDefaultOptions() opts.token_range_splitter = new ParameterizedClass(DEFAULT_SPLITTER.getName(), Collections.emptyMap()); opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("5m"); opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h"); + opts.min_repair_interval = new DurationSpec.IntSecondsBound("24h"); return opts; } diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index 18415f5d3551..4fdbce726ba4 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -65,104 +65,27 @@ import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; /** - * In Apache Cassandra, tuning repair ranges has four main goals: + * The default implementation of {@link IAutoRepairTokenRangeSplitter} that attempts to: *
          - *
        1. - * Create smaller, consistent repair times: Long repairs, such as those lasting 15 hours, can be problematic. - * If a node fails 14 hours into the repair, the entire process must be restarted. The goal is to reduce the impact - * of disturbances or failures. However, making the repairs too short can lead to overhead from repair orchestration - * becoming the main bottleneck. - *
          2. - *
        2. - * Minimize the impact on hosts: Repairs should not heavily affect the host systems. For incremental repairs, - * this might involve anti-compaction work. In full repairs, streaming large amounts of data—especially with wide - * partitions—can lead to issues with disk usage and higher compaction costs. - *
          3. - *
        3. - * Reduce overstreaming: The Merkle tree, which represents data within each partition and range, - * has a maximum size. If a repair covers too many partitions, the tree’s leaves represent larger data ranges. - * Even a small change in a leaf can trigger excessive data streaming, making the process inefficient. - *
          4. - *
        4. - * Reduce number of repairs: If there are many small tables, it's beneficial to batch these tables together - * under a single parent repair. This prevents the repair overhead from becoming a bottleneck, especially when - * dealing with hundreds of tables. Running individual repairs for each table can significantly impact performance - * and efficiency. - *
          5. + *
        5. Create smaller, consistent repair times
          6. + *
        6. Minimize the impact on hosts
          7. + *
        7. Reduce overstreaming
          8. + *
        8. Reduce number of repairs
          9. *
        - * To manage these issues, the strategy involves estimating the size and number of partitions within a range and - * splitting it accordingly to bound the size of the range splits. This is established by iterating over SSTable - * index files to estimate the amount of bytes and partitions involved in the ranges being repaired and by what - * repair type is being invoked. - *

        + *

        + * To achieve these goals, this implementation inspects SSTable metadata to estimate the bytes and number of partitions + * within a range and splits it accordingly to bound the size of the token ranges used for repair assignments. + *

        + *

        + * Refer to + * Auto Repair documentation for this implementation + * for a more thorough breakdown of this implementation. + *

        + *

        * While this splitter has a lot of tuning parameters, the expectation is that the established default configuration * shall be sensible for all {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType}'s. The following * configuration parameters are offered. - *

          - *
        • - * bytes_per_assigment: The target and maximum amount of bytes that should be included in a repair - * assignment. This is meant to scope the amount of work involved in a repair. For incremental repair, this - * involves the total number of bytes in all SSTables containing unrepaired data involving the ranges being - * repaired, including data that doesn't cover the range. This is to account for the amount of anticompaction - * that is expected. For all other repair types, this involves the amount of data covering the range being - * repaired. - *
          • - *
        • - * partitions_per_assignment: The maximum number of partitions that should be included in a repair - * assignment. This configuration exists to reduce excessive overstreaming by attempting to limit the number - * of partitions present in a merkle tree leaf node. - *
          • - *
        • - * max_tables_per_assignment: The maximum number of tables that can be included in a repair assignment. - * This aims to reduce the number of repairs, especially in cases where a large amount of tables exists for - * a keyspace. Note that the splitter will avoid batching tables together if they exceed the other - * configuration parameters such as bytes_per_assignment and partitions_per_assignment. - *
          • - *
        • - * max_bytes_per_schedule: The maximum number of bytes to cover an individual schedule. This serves - * as a mechanism for throttling the amount of work that can be done on each repair cycle. One may opt to - * reduce this value if the impact of repairs is causing too much load on the cluster, or increase it if - * writes outpace the amount of data being repaired. Alternatively, one may want to choose tuning down or up - * the min_repair_interval. - *
          • - *
        - * Given the impact of what each repair type accomplishes, different defaults are established per repair type. - *
          - *
        • - * full: Configured in a way that attempts to accomplish repairing all data in a schedule, with - * individual repairs targeting at most 200GiB of data and 1048576 partitions. - * max_bytes_per_schedule is set to a large value for full repair to attempt to repair all data per - * repair schedule. - *
            - *
          • bytes_per_assignment: 200GiB
            • - *
          • partitions_per_assignment: 1048576
            • - *
          • max_tables_per_assignment: 64
            • - *
          • max_bytes_per_schedule: 100TiB
            • - *
          - *
          • - *
        • - * incremental: Configured in a way that attempts to repair 50GiB of data per repair, and 100GiB per - * schedule. This attempts to throttle the amount of IR and anticompaction done per schedule after turning - * incremental on for the first time. You may want to consider increasing max_bytes_per_schedule - * more than this much data is written per min_repair_interval. - *
            - *
          • bytes_per_assignment: 50GiB
            • - *
          • partitions_per_assignment: 1048576
            • - *
          • max_tables_per_assignment: 64
            • - *
          • max_bytes_per_schedule: 100GiB
            • - *
          - *
          • - *
        • - * preview_repaired: Configured in a way that attempts to accomplish previewing all data in a schedule, - * with previews targeting at most 200GiB of data and 1048576 partitions. - *
            - *
          • bytes_per_assignment: 200GiB
            • - *
          • partitions_per_assignment: 1048576
            • - *
          • max_tables_per_assignment: 64
            • - *
          • max_bytes_per_schedule: 100TiB
            • - *
          - *
          • - *
        + *

        */ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter { @@ -171,9 +94,25 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter // Default max bytes to 100TiB, which is much more readable than Long.MAX_VALUE private static final DataStorageSpec.LongBytesBound MAX_BYTES = new DataStorageSpec.LongBytesBound(100_000, DataStorageSpec.DataStorageUnit.GIBIBYTES); + /** + * The target bytes that should be included in a repair assignment + */ static final String BYTES_PER_ASSIGNMENT = "bytes_per_assignment"; + + /** + * Maximum number of partitions to include in a repair assignment + */ static final String PARTITIONS_PER_ASSIGNMENT = "partitions_per_assignment"; + + /** + * Maximum number of tables to include in a repair assignment if {@link AutoRepairConfig.Options#repair_by_keyspace} + * is enabled. + */ static final String MAX_TABLES_PER_ASSIGNMENT = "max_tables_per_assignment"; + + /** + * The maximum number of bytes to cover in an individual schedule + */ static final String MAX_BYTES_PER_SCHEDULE = "max_bytes_per_schedule"; static final List PARAMETERS = Arrays.asList(BYTES_PER_ASSIGNMENT, PARTITIONS_PER_ASSIGNMENT, MAX_TABLES_PER_ASSIGNMENT, MAX_BYTES_PER_SCHEDULE); @@ -193,23 +132,22 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter *

        * Defaults if not specified for the given repair type: *

      • - *
          bytes_per_assignment: 200GiB
        - *
          partitions_per_assignment: 2^repair_session_max_tree_depth
        + *
          bytes_per_assignment: 50GiB
        + *
          partitions_per_assignment: 1048576 (2^20)
        *
          max_tables_per_assignment: 64
        *
          max_bytes_per_schedule: 1000GiB
        *
        • - * It's expected that these defaults should work well for everything except incremental, so we confine - * bytes_per_assignment to 50GiB and max_bytes_per_schedule to 100GiB. This should strike a good balance - * between the amount of data that will be repaired during an initial migration to incremental repair and should - * move the entire repaired set from unrepaired to repaired at steady state, assuming not more the 100GiB of - * data is written to a node per min_repair_interval. + * It's expected that these defaults should work well for everything except incremental, where we + * max_bytes_per_schedule to 100GiB. This should strike a good balance between the amount of data that will be + * repaired during an initial migration to incremental repair and should move the entire repaired set from + * unrepaired to repaired at steady state, assuming not more the 100GiB of data is written to a node per + * min_repair_interval. */ private static final Map DEFAULTS_BY_REPAIR_TYPE = new EnumMap(AutoRepairConfig.RepairType.class) {{ put(AutoRepairConfig.RepairType.FULL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.FULL) .build()); - // Restrict incremental repair to 50GB bytes per assignment to confine the amount of possible autocompaction. + // Restrict incremental repair to 100GB max bytes per schedule to confine the amount of possible autocompaction. put(AutoRepairConfig.RepairType.INCREMENTAL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.INCREMENTAL) - .withBytesPerAssignment(new DataStorageSpec.LongBytesBound("50GiB")) .withMaxBytesPerSchedule(new DataStorageSpec.LongBytesBound("100GiB")) .build()); put(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.PREVIEW_REPAIRED) @@ -957,8 +895,8 @@ static RepairTypeDefaultsBuilder builder(AutoRepairConfig.RepairType repairType) static class RepairTypeDefaultsBuilder { private final AutoRepairConfig.RepairType repairType; - private DataStorageSpec.LongBytesBound bytesPerAssignment = new DataStorageSpec.LongBytesBound("200GiB"); - // Aims to target at most 1 partitons per leaf assuming a merkle tree of depth 20 (2^20 = 1,048,576) + private DataStorageSpec.LongBytesBound bytesPerAssignment = new DataStorageSpec.LongBytesBound("50GiB"); + // Aims to target at most 1 partitions per leaf assuming a merkle tree of depth 20 (2^20 = 1,048,576) private long partitionsPerAssignment = 1_048_576; private int maxTablesPerAssignment = 64; private DataStorageSpec.LongBytesBound maxBytesPerSchedule = MAX_BYTES; @@ -968,6 +906,7 @@ private RepairTypeDefaultsBuilder(AutoRepairConfig.RepairType repairType) this.repairType = repairType; } + @SuppressWarnings("unused") public RepairTypeDefaultsBuilder withBytesPerAssignment(DataStorageSpec.LongBytesBound bytesPerAssignment) { this.bytesPerAssignment = bytesPerAssignment; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index e8c60f2d9b50..2c3e4401ed9f 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -41,7 +41,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -437,15 +436,7 @@ public void testDefaultOptions() assertEquals(new ParameterizedClass(RepairTokenRangeSplitter.class.getName(), Collections.emptyMap()), options.token_range_splitter); assertEquals(new DurationSpec.IntSecondsBound("5m"), options.initial_scheduler_delay); assertEquals(new DurationSpec.IntSecondsBound("3h"), options.repair_session_timeout); - - if (repairType == AutoRepairConfig.RepairType.INCREMENTAL) - { - assertEquals(new DurationSpec.IntSecondsBound("1h"), options.min_repair_interval); - } - else - { - assertEquals(new DurationSpec.IntSecondsBound("24h"), options.min_repair_interval); - } + assertEquals(new DurationSpec.IntSecondsBound("24h"), options.min_repair_interval); } @Test @@ -466,6 +457,6 @@ public void testGlobalOptions() assertEquals(new ParameterizedClass(RepairTokenRangeSplitter.class.getName(), Collections.emptyMap()), config.global_settings.token_range_splitter); assertEquals(new DurationSpec.IntSecondsBound("5m"), config.global_settings.initial_scheduler_delay); assertEquals(new DurationSpec.IntSecondsBound("3h"), config.global_settings.repair_session_timeout); - assertNull(config.global_settings.min_repair_interval); + assertEquals(new DurationSpec.IntSecondsBound("24h"), config.global_settings.min_repair_interval); } } From 9b78e2b68682eb2143816c0021a2d712ad51130a Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sat, 8 Mar 2025 01:42:01 -0600 Subject: [PATCH 155/199] Revert tombstones.adoc no op changes. From 40882438adbd79520b19b6869481d82d60dff7e5 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sat, 8 Mar 2025 02:16:45 -0600 Subject: [PATCH 156/199] Add documentation for reject_repair_compaction_threshold --- conf/cassandra.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 1a3408a62b14..ce399a0373e4 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1866,6 +1866,11 @@ drop_compact_storage_enabled: false # excluded_keyspaces: # comma separated list of keyspaces to exclude from the check # excluded_tables: # comma separated list of keyspace.table pairs to exclude from the check + +# Prevents preparing a repair session or beginning a repair streaming session if pending compactions is over +# the given value. Defaults to disabled. +# reject_repair_compaction_threshold: 1024 + # At least 20% of disk must be unused to run incremental repair. It is useful to avoid disks filling up during # incremental repair as anti-compaction during incremental repair may contribute to additional space temporarily. # if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason) From c208f7d23849d578d4eb54ce0b7b1b7ace70ad2e Mon Sep 17 00:00:00 2001 From: Andrew Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 9 Mar 2025 11:06:33 -0500 Subject: [PATCH 157/199] Update src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java Co-authored-by: Francisco Guerrero --- .../cassandra/repair/autorepair/RepairTokenRangeSplitter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index 4fdbce726ba4..e60a068fa274 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -146,7 +146,7 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter private static final Map DEFAULTS_BY_REPAIR_TYPE = new EnumMap(AutoRepairConfig.RepairType.class) {{ put(AutoRepairConfig.RepairType.FULL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.FULL) .build()); - // Restrict incremental repair to 100GB max bytes per schedule to confine the amount of possible autocompaction. + // Restrict incremental repair to 100GiB max bytes per schedule to confine the amount of possible autocompaction. put(AutoRepairConfig.RepairType.INCREMENTAL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.INCREMENTAL) .withMaxBytesPerSchedule(new DataStorageSpec.LongBytesBound("100GiB")) .build()); From 93f77235b2c5f4a3b5856b06e2e2173d9b9e21c2 Mon Sep 17 00:00:00 2001 From: Andrew Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 9 Mar 2025 11:08:44 -0500 Subject: [PATCH 158/199] Update conf/cassandra.yaml Co-authored-by: Francisco Guerrero --- conf/cassandra.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index ce399a0373e4..9e35e81f0014 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1900,7 +1900,7 @@ drop_compact_storage_enabled: false # # round on all nodes in less than this duration, it will not start a new repair round on a given node until # # this much time has passed since the last repair completed. Consider increasing to a larger value to reduce # # the impact of repairs, however note that one should attempt to run repairs at a smaller interval than -# # gc_grace_seconds to avoid zombies. +# # gc_grace_seconds to avoid potential data resurrection. # min_repair_interval: 24h # token_range_splitter: # # Implementation of IAutoRepairTokenRangeSplitter; responsible for splitting token ranges From da58371c185cddb2e2b3a52c3ed9eb84da2f14d8 Mon Sep 17 00:00:00 2001 From: Andrew Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 9 Mar 2025 11:09:14 -0500 Subject: [PATCH 159/199] Update conf/cassandra.yaml Co-authored-by: Francisco Guerrero --- conf/cassandra.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 9e35e81f0014..3e0633cc4cb4 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1936,7 +1936,7 @@ drop_compact_storage_enabled: false # # so a more frequent schedule such as 1h is recommended. # # NOTE: When turning on incremental repair for the first time with a decent amount of data it is adviable to # # use a larger interval such as 24h or longer to reduce the impact of anticompaction caused by incremental -# # repair. Once a majority of data has been repaired, one may consider reducing this to 1h to keep to unrepaired +# # repair. Once a majority of data has been repaired, one may consider reducing this to 1h to keep the unrepaired # # set small. # min_repair_interval: 24h # token_range_splitter: From 1abb1a50aa490d63789bc11484e67d8bbd14f9b7 Mon Sep 17 00:00:00 2001 From: Andrew Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 9 Mar 2025 11:10:12 -0500 Subject: [PATCH 160/199] Update conf/cassandra.yaml Co-authored-by: Francisco Guerrero --- conf/cassandra.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 3e0633cc4cb4..d3e8a952cf28 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1945,7 +1945,7 @@ drop_compact_storage_enabled: false # # This throttles the amount of incremental repair and anticompaction done per schedule after incremental # # repairs are turned on. # bytes_per_assignment: 50GiB -# # Resricts the maximum number of bytes to cover in an individual schedule to 100GiB. +# # Resricts the maximum number of bytes to cover in an individual schedule to the configured max_bytes_per_schedule value (defaults to 100GiB). # # Consider increasing this if more data is written than this limit within the min_repair_interval. # max_bytes_per_schedule: 100GiB # preview_repaired: From 82d3d2c92903ca064d008f5c71597d8ffe95025e Mon Sep 17 00:00:00 2001 From: Andrew Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 9 Mar 2025 11:10:24 -0500 Subject: [PATCH 161/199] Update conf/cassandra.yaml Co-authored-by: Francisco Guerrero --- conf/cassandra.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index d3e8a952cf28..a1c5e76b139c 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1946,7 +1946,7 @@ drop_compact_storage_enabled: false # # repairs are turned on. # bytes_per_assignment: 50GiB # # Resricts the maximum number of bytes to cover in an individual schedule to the configured max_bytes_per_schedule value (defaults to 100GiB). -# # Consider increasing this if more data is written than this limit within the min_repair_interval. +# # Consider increasing this value if more data is written than this limit within the min_repair_interval. # max_bytes_per_schedule: 100GiB # preview_repaired: # # Performs preview repair over repaired SSTables, useful to detect possible inconsistencies in the repaired From 55433dee0fec2c047816bfb537acf9da10cf61ef Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 9 Mar 2025 12:27:17 -0500 Subject: [PATCH 162/199] Doc and conf fixes, initial IR enablement guidance --- conf/cassandra.yaml | 18 ++--- .../managing/operating/auto_repair.adoc | 71 ++++++++++++++----- .../cassandra/pages/operating/repair.adoc | 5 +- 3 files changed, 67 insertions(+), 27 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index a1c5e76b139c..fb07effd11ae 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1918,9 +1918,9 @@ drop_compact_storage_enabled: false # # parameters: # # param_key1: param_value1 # parameters: -# # The target and maximum amount of bytes that should be included in a repair -# # assignment. This scopes the amount of work involved in a repair and includes -# # the data covering the range being repaired. +# # The target and maximum amount of compressed bytes that should be included in a repair assignment. +# # This scopes the amount of work involved in a repair and includes the data covering the range being +# # repaired. # bytes_per_assignment: 50GiB # # The maximum number of bytes to cover in an individual schedule. This serves as # # a mechanism to throttle the work done in each repair cycle. You may reduce this @@ -1934,18 +1934,18 @@ drop_compact_storage_enabled: false # # Incremental repairs operate over unrepaired data and should finish quickly. Running incremental repair # # frequently keeps the unrepaired set smaller and thus causes repairs to operate over a smaller set of data, # # so a more frequent schedule such as 1h is recommended. -# # NOTE: When turning on incremental repair for the first time with a decent amount of data it is adviable to -# # use a larger interval such as 24h or longer to reduce the impact of anticompaction caused by incremental -# # repair. Once a majority of data has been repaired, one may consider reducing this to 1h to keep the unrepaired -# # set small. +# # NOTE: Please consult +# # https://cassandra.apache.org/doc/latest/cassandra/managing/operating/auto_repair.html#enabling-ir +# # for guidance on enabling incremental repair on ane exiting cluster. # min_repair_interval: 24h # token_range_splitter: # parameters: -# # Configured to attempt repairing 50GiB of data per repair. +# # Configured to attempt repairing 50GiB of compressed data per repair. # # This throttles the amount of incremental repair and anticompaction done per schedule after incremental # # repairs are turned on. # bytes_per_assignment: 50GiB -# # Resricts the maximum number of bytes to cover in an individual schedule to the configured max_bytes_per_schedule value (defaults to 100GiB). +# # Restricts the maximum number of bytes to cover in an individual schedule to the configured +# # max_bytes_per_schedule value (defaults to 100GiB for incremental). # # Consider increasing this value if more data is written than this limit within the min_repair_interval. # max_bytes_per_schedule: 100GiB # preview_repaired: diff --git a/doc/modules/cassandra/managing/operating/auto_repair.adoc b/doc/modules/cassandra/managing/operating/auto_repair.adoc index db50a8c4524b..e82cb95d1223 100644 --- a/doc/modules/cassandra/managing/operating/auto_repair.adoc +++ b/doc/modules/cassandra/managing/operating/auto_repair.adoc @@ -38,7 +38,7 @@ xref:#fixed-split-token-range-splitter[FixedSplitTokenRangeSplitter]. == Considerations -Before enabling auto repair, please consult the xref:managing/operating/repair.adoc[Repair] guide to establish a base +Before enabling Auto Repair, please consult the xref:managing/operating/repair.adoc[Repair] guide to establish a base understanding of repairs. === Full Repair @@ -59,11 +59,48 @@ be eventually purged. Running incremental repair more frequently keeps the unrepaired set smaller and thus causes repairs to operate over a smaller set of data, so a shorter `min_repair_interval` such as `1h` is recommended for new clusters. -NOTE: When turning on incremental repair for the first time on an existing cluser with a large amount of data it is -advisable to use a larger interval such as `24h` or longer to reduce the impact of anticompaction caused by incremental -repair. Once the entire token range has been repaired, one may consider reducing this to a shorter interval. It is -strongly recommended to not enable incremental repair for an existing cluster unless full repair has previously -actively run. +==== Enabling Incremental Repair on existing clusters with a large amount of data +[#enabling-ir] +One should be careful when enabling incremental repair on a cluster for the first time. While +xref:#repair-token-range-splitter[RepairTokenRangeSplitter] includes a default configuration to attempt to gracefully +migrate to incremental repair over time, failure to take proper precaution could overwhelm the cluster with +xref:managing/operating/compaction/overview.adoc#types-of-compaction[anticompactions]. + +No matter how one goes about enabling and running incremental repair, it is recommended to run a cycle of full repairs +for the entire cluster as pre-flight step to running incremental repair. This will put the cluster into a more +consistent state which will reduce the amount of streaming between replicas when incremental repair initially runs. + +If you do not have strong data consistency requirements, one may consider using +xref:managing/tools/sstable/sstablerepairedset.adoc[nodetool sstablerepairedset] to mark all SSTables as repaired +before enabling incremental repair scheduling using Auto Repair. This will reduce the burden of initially running +incremental repair because all existing data will be considered as repaired, so subsequent incremental repairs will +only run against new data. + +If you do have strong data consistency requirements, then one must treat all data as initially unrepaired and run +incremental repair against it. Consult +xref:#incremental-repair-defaults[RepairTokenRangeSplitter's Incremental repair defaults]. + +In particular one should be mindful of the xref:managing/operating/compaction/overview.adoc[compaction strategy] +you use for your tables and how it might impact incremental repair before running incremental repair for the first +time: + +- *Large SSTables*: When using xref:managing/operating/compaction/stcs.adoc[SizeTieredCompactionStrategy] or any + compaction strategy which can create large SSTables including many partitions the amount of + xref:managing/operating/compaction/overview.adoc#types-of-compaction[anticompaction] that might be required could be + excessive. Using a small `bytes_per_assignment` might contribute to repeated anticompactions over the same + unrepaired data. +- *Partitions overlapping many SSTables*: If partitions overlap between many SSTables, the amount of SSTables included + in a repair might be large. Therefore it is important to consider that many SSTables may be included in a repair + session and must all be anticompacted. xref:managing/operating/compaction/lcs.adoc[LeveledCompactionStrategy] is less + susceptible to this issue as it prevents overlapping of partitions within levels outside of L0, but if SSTables + start accumulating in L0 between incremental repairs, the cost of anticompaction will increase. + xref:managing/operating/compaction/ucs#sharding[UnifiedCompactionStrategy's sharding] can also be used to avoid + partitions overlapping SSTables. + +The xref:#repair-token-range-splitter[token_range_splitter] configuration for incremental repair includes a default +configuration that attempts to conservatively migrate 100GiB of compressed data every day per node. Depending on +requirements, data set and capability of a cluster's hardware, one may consider tuning these values to be more +aggressive or conservative. === Previewing Repaired Data @@ -80,11 +117,11 @@ When enabled, the `BytesPreviewedDesynchronized` and `TokenRangesPreviewedDesync xref:cassandra:managing/operating/metrics.adoc#table-metrics[table metrics] can be used to detect inconsistencies in the repaired data set. -== Configuring auto repair in cassandra.yaml +== Configuring Auto Repair in cassandra.yaml -Configuration for auto repair is managed in the `cassandra.yaml` file by the `auto_repair` property. +Configuration for Auto Repair is managed in the `cassandra.yaml` file by the `auto_repair` property. -A rich set of configuration exists for configuring auto repair with sensible defaults. However, the expectation +A rich set of configuration exists for configuring Auto Repair with sensible defaults. However, the expectation is that some tuning might be needed particulary when it comes to tuning how often repair should run (`min_repair_interval`) and how repair assignments as created (`token_range_splitter`). @@ -148,9 +185,9 @@ type by using `repair_type_overrides`. such as clusters with 5 nodes that finish repairs quickly. This means that if the scheduler completes one round on all nodes in less than this duration, it will not start a new repair round on a given node until this much time has passed since the last repair completed. Consider increasing to a larger value to reduce the impact of repairs, -however *note that one should attempt to run repairs at a smaller interval than gc_grace_seconds to -avoid xref:cassandra:managing/operating/compaction/tombstones.adoc#zombies[zombies]*. -| class_name | org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter | Implementation of +however note that one should attempt to run repairs at a smaller interval than gc_grace_seconds to +avoid xref:cassandra:managing/operating/compaction/tombstones.adoc#zombies[data resurrection]. +| token_range_splitter.class_name | org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter | Implementation of IAutoRepairTokenRangeSplitter to use; responsible for splitting token ranges for repair assignments. Out of the box, Cassandra provides org.apache.cassandra.repair.autorepair.{RepairTokenRangeSplitter,FixedTokenRangeSplitter}. | repair_by_keyspace | true | If true, attempts to group tables in the same keyspace into one repair; otherwise, @@ -183,20 +220,20 @@ in nodetool repair. General advice is to keep this true. `RepairTokenRangeSplitter` is the default implementation of `IAutoRepairTokenRangeSplitter` that attempts to create token range assignments meeting the following goals: -1. *Create smaller, consistent repair times*: Long repairs, such as those lasting 15 hours, can be problematic. If a +- *Create smaller, consistent repair times*: Long repairs, such as those lasting 15 hours, can be problematic. If a node fails 14 hours into the repair, the entire process must be restarted. The goal is to reduce the impact of disturbances or failures. However, making the repairs too short can lead to overhead from repair orchestration becoming the main bottleneck. -2. *Minimize the impact on hosts*: Repairs should not heavily affect the host systems. For incremental repairs, this +- *Minimize the impact on hosts*: Repairs should not heavily affect the host systems. For incremental repairs, this might involve anti-compaction work. In full repairs, streaming large amounts of data—especially with wide partitions can lead to issues with disk usage and higher compaction costs. -3. *Reduce overstreaming*: The Merkle tree, which represents data within each partition and range, has a maximum size. +- *Reduce overstreaming*: The Merkle tree, which represents data within each partition and range, has a maximum size. If a repair covers too many partitions, the tree’s leaves represent larger data ranges. Even a small change in a leaf can trigger excessive data streaming, making the process inefficient. -4. *Reduce number of repairs*: If there are many small tables, it's beneficial to batch these tables together under a +- *Reduce number of repairs*: If there are many small tables, it's beneficial to batch these tables together under a single parent repair. This prevents the repair overhead from becoming a bottleneck, especially when dealing with hundreds of tables. Running individual repairs for each table can significantly impact performance and efficiency. @@ -281,7 +318,7 @@ conservative this top level configuration could be increased to a larger value t == Table configuration -If auto repair is enabled in cassandra.yaml, the `auto_repair` property may be optionally configured at the table +If Auto Repair is enabled in cassandra.yaml, the `auto_repair` property may be optionally configured at the table level, e.g.: [source,cql] diff --git a/doc/modules/cassandra/pages/operating/repair.adoc b/doc/modules/cassandra/pages/operating/repair.adoc index 22745dc16617..01926255554d 100644 --- a/doc/modules/cassandra/pages/operating/repair.adoc +++ b/doc/modules/cassandra/pages/operating/repair.adoc @@ -36,7 +36,10 @@ traditionally not been run automatically by Cassandra. In the latest version of Cassandra, a new feature called xref:managing/operating/auto_repair.adoc[auto repair] was introduced to -allow Cassandra to run repairs automatically on a schedule. +allow Cassandra to submit and manage repairs automatically on a schedule. + +The introduction of this feature does not interfere with existing repair +functionality enabled via nodetool. == Submitting Repairs Using Nodetool From 6aef878063b291016988a5ff692164a817c0caf4 Mon Sep 17 00:00:00 2001 From: Andrew Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 9 Mar 2025 13:22:35 -0500 Subject: [PATCH 163/199] Update src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java Co-authored-by: Francisco Guerrero --- .../cassandra/repair/autorepair/RepairTokenRangeSplitter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index e60a068fa274..f856a6530292 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -106,7 +106,7 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter /** * Maximum number of tables to include in a repair assignment if {@link AutoRepairConfig.Options#repair_by_keyspace} - * is enabled. + * is enabled */ static final String MAX_TABLES_PER_ASSIGNMENT = "max_tables_per_assignment"; From 764d0165b10e44716c1b993a24f9a51013f4ffd0 Mon Sep 17 00:00:00 2001 From: Andrew Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Sun, 9 Mar 2025 13:22:51 -0500 Subject: [PATCH 164/199] Update src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java Co-authored-by: Francisco Guerrero --- .../cassandra/repair/autorepair/RepairTokenRangeSplitter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index f856a6530292..99c3e3add9ec 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -137,7 +137,7 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter *
          max_tables_per_assignment: 64
        *
          max_bytes_per_schedule: 1000GiB
        * - * It's expected that these defaults should work well for everything except incremental, where we + * It's expected that these defaults should work well for everything except incremental, where we set * max_bytes_per_schedule to 100GiB. This should strike a good balance between the amount of data that will be * repaired during an initial migration to incremental repair and should move the entire repaired set from * unrepaired to repaired at steady state, assuming not more the 100GiB of data is written to a node per From 2623823414a080343307d1737f61d4ba167c61fb Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 9 Mar 2025 13:51:09 -0700 Subject: [PATCH 165/199] Review comments from Andy --- .../FixedSplitTokenRangeSplitter.java | 13 ++- .../FixedSplitTokenRangeSplitterTest.java | 107 ++++++++++-------- 2 files changed, 72 insertions(+), 48 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index 54f9b346c91d..1bbd3354daef 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -43,6 +43,17 @@ public class FixedSplitTokenRangeSplitter implements IAutoRepairTokenRangeSplitt { private static final Logger logger = LoggerFactory.getLogger(FixedSplitTokenRangeSplitter.class); + /** + * Selecting the default value is tricky. If we select a small number then individual repair would be heavy, + * on the other hand, if we select large then too many repair sessions would be created. + *

        + * The default 64 is a good number for non v-nodes, but with v-nodes it can result in too many repair sessions + * because the number of repair sessions on a node = number_of_subranges * num_token. + *

        + * To keep it balances, 32 serves a good default that would cater to both v-nodes and non v-nodes. + */ + public static final int DEFAULT_SUBRANGES = 32; + /** * The number of subranges to split each to-be-repaired token range into. Defaults to 1. *

        @@ -67,7 +78,7 @@ public FixedSplitTokenRangeSplitter(AutoRepairConfig.RepairType repairType, Map< { this.repairType = repairType; - numberOfSubranges = Integer.parseInt(parameters.getOrDefault(NUMBER_OF_SUBRANGES, "1")); + numberOfSubranges = Integer.parseInt(parameters.getOrDefault(NUMBER_OF_SUBRANGES, Integer.toString(DEFAULT_SUBRANGES))); } @Override diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java index 0fd1f6839313..6549637f5302 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java @@ -24,8 +24,9 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Set; +import java.util.TreeSet; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -43,6 +44,7 @@ import static org.apache.cassandra.Util.setAutoRepairEnabled; import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; +import static org.apache.cassandra.repair.autorepair.FixedSplitTokenRangeSplitter.DEFAULT_SUBRANGES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -58,24 +60,30 @@ public class FixedSplitTokenRangeSplitterTest extends CQLTester private static final String TABLE1 = "tbl1"; private static final String TABLE2 = "tbl2"; private static final String TABLE3 = "tbl3"; - private static final int totalTokenRanges = 3; - private static int numberOfSplits; @Parameterized.Parameter(0) public AutoRepairConfig.RepairType repairType; @Parameterized.Parameter(1) - public int numberOfSubRange; + public int numberOfSubRanges; - @Parameterized.Parameters(name = "repairType={0}, numberOfSubRange={1}") + @Parameterized.Parameter(2) + public int numTokens; + + public static final int curTokenRange = 1; + + @Parameterized.Parameters(name = "repairType={0}, numberOfSubRanges={1}, numTokens={2}") public static Collection parameters() { List params = new ArrayList<>(); for (AutoRepairConfig.RepairType type : AutoRepairConfig.RepairType.values()) { - for (int subRange : Arrays.asList(4, 16, 64, 128)) + for (int subRange : Arrays.asList(1, 2, 4, 8, 16, 32, 64, 128, 256)) { - params.add(new Object[]{ type, subRange }); + for (int numToken : Arrays.asList(curTokenRange)) + { + params.add(new Object[]{ type, subRange, numToken }); + } } } return params; @@ -94,18 +102,13 @@ public static void setupClass() throws Exception SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1)); } - @Before - public void before() - { - numberOfSplits = numberOfSubRange / totalTokenRanges; - } - @Test public void testTokenRangesSplitByTable() { + int numberOfSplits = calcSplits(numberOfSubRanges); AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(repairType, false); Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); - assertEquals(totalTokenRanges, tokens.size()); + assertEquals(numTokens, tokens.size()); List tables = Arrays.asList(TABLE1, TABLE2, TABLE3); List> expectedToken = new ArrayList<>(); for (int i = 0; i plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); - Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRange))) + Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRanges))) .getRepairAssignments(true, plan); // should be only 1 entry for the keyspace. @@ -127,37 +130,31 @@ public void testTokenRangesSplitByTable() assertFalse(keyspaceAssignments.hasNext()); List assignments = keyspace.getRepairAssignments(); - assertEquals(totalTokenRanges * numberOfSplits * tables.size(), assignments.size()); + assertEquals(numTokens * numberOfSplits * tables.size(), assignments.size()); assertEquals(expectedToken.size(), assignments.size()); - int expectedTableIndex = -1; - for (int i = 0; i assignmentForATable = new ArrayList<>(); + List> expectedTokensForATable = new ArrayList<>(); + for (int j = 0; j < assignmentsPerTable; j++) { - expectedTableIndex++; + assertEquals(Arrays.asList(tables.get(i)), assignments.get(i*assignmentsPerTable + j).getTableNames()); + assignmentForATable.add(assignments.get(i*assignmentsPerTable+j)); + expectedTokensForATable.add(expectedToken.get(i*assignmentsPerTable+j)); } - } - - expectedTableIndex = -1; - // should be a set of ranges for each table. - for (int i = 0; i < totalTokenRanges * numberOfSplits * tables.size(); i++) - { - if (i % (totalTokenRanges * numberOfSplits) == 0) - { - expectedTableIndex++; - } - assertEquals(expectedToken.get(i), assignments.get(i).getTokenRange()); - assertEquals(Collections.singletonList(tables.get(expectedTableIndex)), assignments.get(i).getTableNames()); + compare(numberOfSplits, expectedTokensForATable, assignmentForATable); } } @Test public void testTokenRangesSplitByKeyspace() { + int numberOfSplits = calcSplits(numberOfSubRanges); AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(repairType, true); Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); - assertEquals(totalTokenRanges, tokens.size()); + assertEquals(numTokens, tokens.size()); List tables = Arrays.asList(TABLE1, TABLE2, TABLE3); List> expectedToken = new ArrayList<>(); for (Range range : tokens) @@ -167,7 +164,7 @@ public void testTokenRangesSplitByKeyspace() List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); - Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRange))) + Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRanges))) .getRepairAssignments(true, plan); // should be only 1 entry for the keyspace. @@ -178,23 +175,23 @@ public void testTokenRangesSplitByKeyspace() List assignments = keyspace.getRepairAssignments(); assertNotNull(assignments); - assertEquals(totalTokenRanges * numberOfSplits, assignments.size()); + assertEquals(numTokens * numberOfSplits, assignments.size()); assertEquals(expectedToken.size(), assignments.size()); - // should only be one set of ranges for the entire keyspace. - for (int i = 0; i < totalTokenRanges * numberOfSplits; i++) - { - assertEquals(expectedToken.get(i), assignments.get(i).getTokenRange()); - assertEquals(tables, assignments.get(i).getTableNames()); - } + compare(numberOfSplits, expectedToken, assignments); } @Test - public void testTokenRangesNoSplitByDefault() + public void testTokenRangesWithDefaultSplit() { + int numberOfSplits = calcSplits(DEFAULT_SUBRANGES); Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); - assertEquals(totalTokenRanges, tokens.size()); - List> expectedToken = new ArrayList<>(tokens); + assertEquals(numTokens, tokens.size()); + List> expectedToken = new ArrayList<>(); + for (Range range : tokens) + { + expectedToken.addAll(AutoRepairUtils.split(range, numberOfSplits)); + } List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1); @@ -209,10 +206,26 @@ public void testTokenRangesNoSplitByDefault() assertNotNull(assignments); // should be 3 entries for the table which covers each token range. - assertEquals(totalTokenRanges, assignments.size()); - for (int i = 0; i < totalTokenRanges; i++) + assertEquals(numTokens * numberOfSplits, assignments.size()); + + compare(numberOfSplits, expectedToken, assignments); + } + + private void compare(int numberOfSplits, List> expectedToken, List assignments) + { + assertEquals(expectedToken.size(), assignments.size()); + Set> a = new TreeSet<>(); + Set> b = new TreeSet<>(); + for (int i = 0; i < numTokens * numberOfSplits; i++) { - assertEquals(expectedToken.get(i), assignments.get(i).getTokenRange()); + a.add(expectedToken.get(i)); + b.add(assignments.get(i).getTokenRange()); } + assertEquals(a, b); + } + + private int calcSplits(int subRange) + { + return Math.max(1, subRange / numTokens); } } From 36c598a5ed68ee2f6ca05dce5cdc291002a51c1b Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 9 Mar 2025 14:14:31 -0700 Subject: [PATCH 166/199] Add v-nodes and non v-nodes test cases --- .../FixedSplitTokenRangeSplitter.java | 4 +- ...> FixedSplitTokenRangeSplitterHelper.java} | 89 +++------------ ...edSplitTokenRangeSplitterNoVNodesTest.java | 108 ++++++++++++++++++ ...ixedSplitTokenRangeSplitterVNodesTest.java | 108 ++++++++++++++++++ 4 files changed, 236 insertions(+), 73 deletions(-) rename test/unit/org/apache/cassandra/repair/autorepair/{FixedSplitTokenRangeSplitterTest.java => FixedSplitTokenRangeSplitterHelper.java} (68%) create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index 1bbd3354daef..74d9a35738fa 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -52,7 +52,7 @@ public class FixedSplitTokenRangeSplitter implements IAutoRepairTokenRangeSplitt *

        * To keep it balances, 32 serves a good default that would cater to both v-nodes and non v-nodes. */ - public static final int DEFAULT_SUBRANGES = 32; + public static final int DEFAULT_NUMBER_OF_SUBRANGES = 32; /** * The number of subranges to split each to-be-repaired token range into. Defaults to 1. @@ -78,7 +78,7 @@ public FixedSplitTokenRangeSplitter(AutoRepairConfig.RepairType repairType, Map< { this.repairType = repairType; - numberOfSubranges = Integer.parseInt(parameters.getOrDefault(NUMBER_OF_SUBRANGES, Integer.toString(DEFAULT_SUBRANGES))); + numberOfSubranges = Integer.parseInt(parameters.getOrDefault(NUMBER_OF_SUBRANGES, Integer.toString(DEFAULT_NUMBER_OF_SUBRANGES))); } @Override diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java similarity index 68% rename from test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java rename to test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java index 6549637f5302..925fa8e7482c 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java @@ -27,85 +27,34 @@ import java.util.Set; import java.util.TreeSet; -import org.junit.BeforeClass; -import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.DurationSpec; -import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.service.StorageService; -import static org.apache.cassandra.Util.setAutoRepairEnabled; -import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; -import static org.apache.cassandra.repair.autorepair.FixedSplitTokenRangeSplitter.DEFAULT_SUBRANGES; +import static org.apache.cassandra.repair.autorepair.FixedSplitTokenRangeSplitter.DEFAULT_NUMBER_OF_SUBRANGES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** - * Unit tests for {@link org.apache.cassandra.repair.autorepair.FixedSplitTokenRangeSplitter} + * Helper class for for {@link FixedSplitTokenRangeSplitterNoVNodesTest} and {@link FixedSplitTokenRangeSplitterVNodesTest} */ @RunWith(Parameterized.class) -public class FixedSplitTokenRangeSplitterTest extends CQLTester +public class FixedSplitTokenRangeSplitterHelper { - private static final String KEYSPACE = "ks"; private static final String TABLE1 = "tbl1"; private static final String TABLE2 = "tbl2"; private static final String TABLE3 = "tbl3"; + public static final String KEYSPACE = "ks"; - @Parameterized.Parameter(0) - public AutoRepairConfig.RepairType repairType; - - @Parameterized.Parameter(1) - public int numberOfSubRanges; - - @Parameterized.Parameter(2) - public int numTokens; - - public static final int curTokenRange = 1; - - @Parameterized.Parameters(name = "repairType={0}, numberOfSubRanges={1}, numTokens={2}") - public static Collection parameters() - { - List params = new ArrayList<>(); - for (AutoRepairConfig.RepairType type : AutoRepairConfig.RepairType.values()) - { - for (int subRange : Arrays.asList(1, 2, 4, 8, 16, 32, 64, 128, 256)) - { - for (int numToken : Arrays.asList(curTokenRange)) - { - params.add(new Object[]{ type, subRange, numToken }); - } - } - } - return params; - } - - @BeforeClass - public static void setupClass() throws Exception - { - AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); - SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); - setAutoRepairEnabled(true); - DatabaseDescriptor.setInitialTokens("-9223372036854775808,-3074457345618258603,3074457345618258602"); - requireNetwork(); - AutoRepairUtils.setup(); - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1)); - } - - @Test - public void testTokenRangesSplitByTable() + public static void testTokenRangesSplitByTable(int numTokens, int numberOfSubRanges, AutoRepairConfig.RepairType repairType) { - int numberOfSplits = calcSplits(numberOfSubRanges); + int numberOfSplits = calcSplits(numTokens, numberOfSubRanges); AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(repairType, false); Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); assertEquals(numTokens, tokens.size()); @@ -140,18 +89,17 @@ public void testTokenRangesSplitByTable() List> expectedTokensForATable = new ArrayList<>(); for (int j = 0; j < assignmentsPerTable; j++) { - assertEquals(Arrays.asList(tables.get(i)), assignments.get(i*assignmentsPerTable + j).getTableNames()); - assignmentForATable.add(assignments.get(i*assignmentsPerTable+j)); - expectedTokensForATable.add(expectedToken.get(i*assignmentsPerTable+j)); + assertEquals(Arrays.asList(tables.get(i)), assignments.get(i * assignmentsPerTable + j).getTableNames()); + assignmentForATable.add(assignments.get(i * assignmentsPerTable + j)); + expectedTokensForATable.add(expectedToken.get(i * assignmentsPerTable + j)); } - compare(numberOfSplits, expectedTokensForATable, assignmentForATable); + compare(numTokens, numberOfSplits, expectedTokensForATable, assignmentForATable); } } - @Test - public void testTokenRangesSplitByKeyspace() + public static void testTokenRangesSplitByKeyspace(int numTokens, int numberOfSubRanges, AutoRepairConfig.RepairType repairType) { - int numberOfSplits = calcSplits(numberOfSubRanges); + int numberOfSplits = calcSplits(numTokens, numberOfSubRanges); AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(repairType, true); Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); assertEquals(numTokens, tokens.size()); @@ -178,13 +126,12 @@ public void testTokenRangesSplitByKeyspace() assertEquals(numTokens * numberOfSplits, assignments.size()); assertEquals(expectedToken.size(), assignments.size()); - compare(numberOfSplits, expectedToken, assignments); + compare(numTokens, numberOfSplits, expectedToken, assignments); } - @Test - public void testTokenRangesWithDefaultSplit() + public static void testTokenRangesWithDefaultSplit(int numTokens, int numberOfSubRanges, AutoRepairConfig.RepairType repairType) { - int numberOfSplits = calcSplits(DEFAULT_SUBRANGES); + int numberOfSplits = calcSplits(numTokens, DEFAULT_NUMBER_OF_SUBRANGES); Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); assertEquals(numTokens, tokens.size()); List> expectedToken = new ArrayList<>(); @@ -208,10 +155,10 @@ public void testTokenRangesWithDefaultSplit() // should be 3 entries for the table which covers each token range. assertEquals(numTokens * numberOfSplits, assignments.size()); - compare(numberOfSplits, expectedToken, assignments); + compare(numTokens, numberOfSplits, expectedToken, assignments); } - private void compare(int numberOfSplits, List> expectedToken, List assignments) + private static void compare(int numTokens, int numberOfSplits, List> expectedToken, List assignments) { assertEquals(expectedToken.size(), assignments.size()); Set> a = new TreeSet<>(); @@ -224,7 +171,7 @@ private void compare(int numberOfSplits, List> expectedToken, List< assertEquals(a, b); } - private int calcSplits(int subRange) + private static int calcSplits(int numTokens, int subRange) { return Math.max(1, subRange / numTokens); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java new file mode 100644 index 000000000000..a92c7d2bc69b --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.cassandra.ServerTestUtils; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.dht.BootStrapper; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.index.sai.disk.format.Version; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.tcm.ClusterMetadata; + +import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; +import static org.apache.cassandra.cql3.CQLTester.Fuzzed.setupSeed; +import static org.apache.cassandra.cql3.CQLTester.Fuzzed.updateConfigs; + +/** + * Unit tests for a setup that does not have v-nodes {@link FixedSplitTokenRangeSplitter} + */ +@RunWith(Parameterized.class) +public class FixedSplitTokenRangeSplitterNoVNodesTest +{ + private static final int numTokens = 1; + + @Parameterized.Parameter(0) + public AutoRepairConfig.RepairType repairType; + + @Parameterized.Parameter(1) + public int numberOfSubRanges; + + @Parameterized.Parameters(name = "repairType={0}, numberOfSubRanges={1}") + public static Collection parameters() + { + List params = new ArrayList<>(); + for (AutoRepairConfig.RepairType type : AutoRepairConfig.RepairType.values()) + { + for (int subRange : Arrays.asList(1, 2, 4, 8, 16, 32, 64, 128, 256)) + { + params.add(new Object[]{ type, subRange }); + } + } + return params; + } + + @BeforeClass + public static void setupClass() throws Exception + { + setupSeed(); + updateConfigs(); + DatabaseDescriptor.setPartitioner("org.apache.cassandra.dht.Murmur3Partitioner"); + ServerTestUtils.prepareServerNoRegister(); + + Set tokens = BootStrapper.getRandomTokens(ClusterMetadata.current(), numTokens); + ServerTestUtils.registerLocal(tokens); + // Ensure that the on-disk format statics are loaded before the test run + Version.LATEST.onDiskFormat(); + StorageService.instance.doAutoRepairSetup(); + + SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); + QueryProcessor.executeInternal(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", FixedSplitTokenRangeSplitterHelper.KEYSPACE)); + } + + @Test + public void testTokenRangesSplitByTable() + { + FixedSplitTokenRangeSplitterHelper.testTokenRangesSplitByTable(numTokens, numberOfSubRanges, repairType); + } + + @Test + public void testTokenRangesSplitByKeyspace() + { + FixedSplitTokenRangeSplitterHelper.testTokenRangesSplitByKeyspace(numTokens, numberOfSubRanges, repairType); + } + + @Test + public void testTokenRangesWithDefaultSplit() + { + FixedSplitTokenRangeSplitterHelper.testTokenRangesWithDefaultSplit(numTokens, numberOfSubRanges, repairType); + } +} diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java new file mode 100644 index 000000000000..1b48537968d4 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.cassandra.ServerTestUtils; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.dht.BootStrapper; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.index.sai.disk.format.Version; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.tcm.ClusterMetadata; + +import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; +import static org.apache.cassandra.cql3.CQLTester.Fuzzed.setupSeed; +import static org.apache.cassandra.cql3.CQLTester.Fuzzed.updateConfigs; + +/** + * Unit tests for a setup that has v-nodes {@link FixedSplitTokenRangeSplitter} + */ +@RunWith(Parameterized.class) +public class FixedSplitTokenRangeSplitterVNodesTest +{ + private static final int numTokens = 16; + + @Parameterized.Parameter(0) + public AutoRepairConfig.RepairType repairType; + + @Parameterized.Parameter(1) + public int numberOfSubRanges; + + @Parameterized.Parameters(name = "repairType={0}, numberOfSubRanges={1}") + public static Collection parameters() + { + List params = new ArrayList<>(); + for (AutoRepairConfig.RepairType type : AutoRepairConfig.RepairType.values()) + { + for (int subRange : Arrays.asList(1, 2, 4, 8, 16, 32, 64, 128, 256)) + { + params.add(new Object[]{ type, subRange }); + } + } + return params; + } + + @BeforeClass + public static void setupClass() throws Exception + { + setupSeed(); + updateConfigs(); + DatabaseDescriptor.setPartitioner("org.apache.cassandra.dht.Murmur3Partitioner"); + ServerTestUtils.prepareServerNoRegister(); + + Set tokens = BootStrapper.getRandomTokens(ClusterMetadata.current(), numTokens); + ServerTestUtils.registerLocal(tokens); + // Ensure that the on-disk format statics are loaded before the test run + Version.LATEST.onDiskFormat(); + StorageService.instance.doAutoRepairSetup(); + + SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); + QueryProcessor.executeInternal(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", FixedSplitTokenRangeSplitterHelper.KEYSPACE)); + } + + @Test + public void testTokenRangesSplitByTable() + { + FixedSplitTokenRangeSplitterHelper.testTokenRangesSplitByTable(numTokens, numberOfSubRanges, repairType); + } + + @Test + public void testTokenRangesSplitByKeyspace() + { + FixedSplitTokenRangeSplitterHelper.testTokenRangesSplitByKeyspace(numTokens, numberOfSubRanges, repairType); + } + + @Test + public void testTokenRangesWithDefaultSplit() + { + FixedSplitTokenRangeSplitterHelper.testTokenRangesWithDefaultSplit(numTokens, numberOfSubRanges, repairType); + } +} From 94cb75eb2e27647e68c95b71f09d21fa388874c0 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 9 Mar 2025 14:36:52 -0700 Subject: [PATCH 167/199] Adjust the comment --- .../autorepair/FixedSplitTokenRangeSplitter.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index 74d9a35738fa..414c924f6250 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -44,13 +44,15 @@ public class FixedSplitTokenRangeSplitter implements IAutoRepairTokenRangeSplitt private static final Logger logger = LoggerFactory.getLogger(FixedSplitTokenRangeSplitter.class); /** - * Selecting the default value is tricky. If we select a small number then individual repair would be heavy, - * on the other hand, if we select large then too many repair sessions would be created. + * Selecting the default value is tricky. If we select a small number, individual repairs would be heavy. + * On the other hand, if we select a large number, too many repair sessions would be created. *

        - * The default 64 is a good number for non v-nodes, but with v-nodes it can result in too many repair sessions - * because the number of repair sessions on a node = number_of_subranges * num_token. + * The default value of 64 works well for non-vnodes, but with vnodes, it can result in too many repair sessions + * because the total number of repair sessions on a node is calculated as: *

        - * To keep it balances, 32 serves a good default that would cater to both v-nodes and non v-nodes. + * number_of_subranges * num_token + *

        + * To maintain balance, 32 serves as a good default that accommodates both vnodes and non-vnodes effectively. */ public static final int DEFAULT_NUMBER_OF_SUBRANGES = 32; From e3070bbe5dad55c3b8a74cd393eec30369dec378 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 9 Mar 2025 20:16:41 -0700 Subject: [PATCH 168/199] Add the missing nodetool way to update repair_by_keyspace --- .../org/apache/cassandra/service/AutoRepairService.java | 6 ++++++ .../apache/cassandra/service/AutoRepairServiceMBean.java | 3 +++ src/java/org/apache/cassandra/tools/NodeProbe.java | 5 +++++ .../cassandra/tools/nodetool/SetAutoRepairConfig.java | 6 +++++- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 1c96ab685d2c..0cf744296e07 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -255,6 +255,12 @@ public void setAutoRepairTokenRangeSplitterParameter(String repairType, String k config.getTokenRangeSplitterInstance(RepairType.parse(repairType)).setParameter(key, value); } + @Override + public void setRepairByKeyspace(String repairType, boolean repairByKeyspace) + { + config.setRepairByKeyspace(RepairType.parse(repairType), repairByKeyspace); + } + private String formatRepairTypeConfig(RepairType repairType, AutoRepairConfig config) { StringBuilder sb = new StringBuilder(); diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index f3ec919c8f1b..d57329e86795 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -68,4 +68,7 @@ public interface AutoRepairServiceMBean public Set getOnGoingRepairHostIds(String repairType); public void setAutoRepairTokenRangeSplitterParameter(String repairType, String key, String value); + + public void setRepairByKeyspace(String repairType, boolean repairByKeyspace); + } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index b887fc97fe73..d1f3e27d2573 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2262,6 +2262,11 @@ public Set getAutoRepairOnGoingRepairHostIds(String repairType) { return autoRepairProxy.getOnGoingRepairHostIds(repairType); } + + public void setAutoRepairRepairByKeyspace(String repairType, boolean enabled) + { + autoRepairProxy.setRepairByKeyspace(repairType, enabled); + } } class ColumnFamilyStoreMBeanIterator implements Iterator> diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index 9f22a43c4eb9..37f809943fb6 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -48,7 +48,8 @@ public class SetAutoRepairConfig extends NodeToolCmd "|enabled|table_max_repair_time|priority_hosts|forcerepair_hosts|ignore_dcs" + "|history_clear_delete_hosts_buffer_interval|repair_primary_token_range_only" + "|parallel_repair_count|parallel_repair_percentage|materialized_view_repair_enabled|repair_max_retries" + - "|repair_retry_backoff|repair_session_timeout|min_repair_task_duration|token_range_splitter.]", + "|repair_retry_backoff|repair_session_timeout|min_repair_task_duration" + + "|repair_by_keyspace|token_range_splitter.]", required = true) protected List args = new ArrayList<>(); @@ -159,6 +160,9 @@ public void execute(NodeProbe probe) case "repair_session_timeout": probe.setAutoRepairSessionTimeout(repairTypeStr, paramVal); break; + case "repair_by_keyspace": + probe.setAutoRepairRepairByKeyspace(repairTypeStr, Boolean.parseBoolean(paramVal)); + break; default: throw new IllegalArgumentException("Unknown parameter: " + paramType); } From 3aa18ab496108ab7374ebf8368c034e6b51cb643 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 9 Mar 2025 20:31:05 -0700 Subject: [PATCH 169/199] Use common setup code for FixedSplitTokenRangeSplitterTest*.java --- .../FixedSplitTokenRangeSplitterHelper.java | 26 +++++++++++++++++ ...edSplitTokenRangeSplitterNoVNodesTest.java | 28 +------------------ ...ixedSplitTokenRangeSplitterVNodesTest.java | 28 +------------------ 3 files changed, 28 insertions(+), 54 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java index 925fa8e7482c..07f0bf751b1f 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java @@ -30,11 +30,20 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.apache.cassandra.ServerTestUtils; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.dht.BootStrapper; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.index.sai.disk.format.Version; import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.tcm.ClusterMetadata; +import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; +import static org.apache.cassandra.cql3.CQLTester.Fuzzed.setupSeed; +import static org.apache.cassandra.cql3.CQLTester.Fuzzed.updateConfigs; import static org.apache.cassandra.repair.autorepair.FixedSplitTokenRangeSplitter.DEFAULT_NUMBER_OF_SUBRANGES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -52,6 +61,23 @@ public class FixedSplitTokenRangeSplitterHelper private static final String TABLE3 = "tbl3"; public static final String KEYSPACE = "ks"; + public static void setupClass(int numTokens) throws Exception + { + setupSeed(); + updateConfigs(); + DatabaseDescriptor.setPartitioner("org.apache.cassandra.dht.Murmur3Partitioner"); + ServerTestUtils.prepareServerNoRegister(); + + Set tokens = BootStrapper.getRandomTokens(ClusterMetadata.current(), numTokens); + ServerTestUtils.registerLocal(tokens); + // Ensure that the on-disk format statics are loaded before the test run + Version.LATEST.onDiskFormat(); + StorageService.instance.doAutoRepairSetup(); + + SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); + QueryProcessor.executeInternal(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", FixedSplitTokenRangeSplitterHelper.KEYSPACE)); + } + public static void testTokenRangesSplitByTable(int numTokens, int numberOfSubRanges, AutoRepairConfig.RepairType repairType) { int numberOfSplits = calcSplits(numTokens, numberOfSubRanges); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java index a92c7d2bc69b..5b86728e574d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java @@ -22,26 +22,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.Set; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.apache.cassandra.ServerTestUtils; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.cql3.QueryProcessor; -import org.apache.cassandra.dht.BootStrapper; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.index.sai.disk.format.Version; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.tcm.ClusterMetadata; - -import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; -import static org.apache.cassandra.cql3.CQLTester.Fuzzed.setupSeed; -import static org.apache.cassandra.cql3.CQLTester.Fuzzed.updateConfigs; - /** * Unit tests for a setup that does not have v-nodes {@link FixedSplitTokenRangeSplitter} */ @@ -73,19 +59,7 @@ public static Collection parameters() @BeforeClass public static void setupClass() throws Exception { - setupSeed(); - updateConfigs(); - DatabaseDescriptor.setPartitioner("org.apache.cassandra.dht.Murmur3Partitioner"); - ServerTestUtils.prepareServerNoRegister(); - - Set tokens = BootStrapper.getRandomTokens(ClusterMetadata.current(), numTokens); - ServerTestUtils.registerLocal(tokens); - // Ensure that the on-disk format statics are loaded before the test run - Version.LATEST.onDiskFormat(); - StorageService.instance.doAutoRepairSetup(); - - SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); - QueryProcessor.executeInternal(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", FixedSplitTokenRangeSplitterHelper.KEYSPACE)); + FixedSplitTokenRangeSplitterHelper.setupClass(numTokens); } @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java index 1b48537968d4..8ca30bd2017c 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java @@ -22,26 +22,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.Set; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.apache.cassandra.ServerTestUtils; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.cql3.QueryProcessor; -import org.apache.cassandra.dht.BootStrapper; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.index.sai.disk.format.Version; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.tcm.ClusterMetadata; - -import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; -import static org.apache.cassandra.cql3.CQLTester.Fuzzed.setupSeed; -import static org.apache.cassandra.cql3.CQLTester.Fuzzed.updateConfigs; - /** * Unit tests for a setup that has v-nodes {@link FixedSplitTokenRangeSplitter} */ @@ -73,19 +59,7 @@ public static Collection parameters() @BeforeClass public static void setupClass() throws Exception { - setupSeed(); - updateConfigs(); - DatabaseDescriptor.setPartitioner("org.apache.cassandra.dht.Murmur3Partitioner"); - ServerTestUtils.prepareServerNoRegister(); - - Set tokens = BootStrapper.getRandomTokens(ClusterMetadata.current(), numTokens); - ServerTestUtils.registerLocal(tokens); - // Ensure that the on-disk format statics are loaded before the test run - Version.LATEST.onDiskFormat(); - StorageService.instance.doAutoRepairSetup(); - - SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); - QueryProcessor.executeInternal(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", FixedSplitTokenRangeSplitterHelper.KEYSPACE)); + FixedSplitTokenRangeSplitterHelper.setupClass(numTokens); } @Test From 3bf9b441a28d38853bd539bcfc9f087072990240 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 9 Mar 2025 20:39:19 -0700 Subject: [PATCH 170/199] Fix FixedSplitTokenRangeSplitterHelper.java failure --- .../repair/autorepair/FixedSplitTokenRangeSplitterHelper.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java index 07f0bf751b1f..d6c2e8ce70e4 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java @@ -27,9 +27,6 @@ import java.util.Set; import java.util.TreeSet; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryProcessor; @@ -53,7 +50,6 @@ /** * Helper class for for {@link FixedSplitTokenRangeSplitterNoVNodesTest} and {@link FixedSplitTokenRangeSplitterVNodesTest} */ -@RunWith(Parameterized.class) public class FixedSplitTokenRangeSplitterHelper { private static final String TABLE1 = "tbl1"; From 53349556bed996fd9f7520cd0ac4fb6006ec8893 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:25:12 -0500 Subject: [PATCH 171/199] Update num_of_subranges --- doc/modules/cassandra/managing/operating/auto_repair.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/cassandra/managing/operating/auto_repair.adoc b/doc/modules/cassandra/managing/operating/auto_repair.adoc index e82cb95d1223..c456536d905c 100644 --- a/doc/modules/cassandra/managing/operating/auto_repair.adoc +++ b/doc/modules/cassandra/managing/operating/auto_repair.adoc @@ -297,9 +297,9 @@ The following `parameters` apply for `FixedSplitTokenRangeSplitter` configuratio [cols=",,",options="header",] |=== | Name | Default | Description -| number_of_subranges | 64 | Number of evenly split subranges to create for each node that repair runs for. +| number_of_subranges | 32 | Number of evenly split subranges to create for each node that repair runs for. If vnodes are configured using `num_tokens`, attempts to evenly subdivide subranges by each range. For example, for -`num_tokens: 16` and `number_of_subranges: 64`, 4 (64/16) repair assignments will be created for each token range. At +`num_tokens: 16` and `number_of_subranges: 32`, 2 (32/16) repair assignments will be created for each token range. At least one repair assignment will be created for each token range. |=== From f53fbd7e12b1f910c7ae0d45d94b0aabe3d91b2d Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 10 Mar 2025 16:22:11 -0700 Subject: [PATCH 172/199] Address review comments: update comment/remove dead code --- .../repair/autorepair/FixedSplitTokenRangeSplitter.java | 6 +++--- .../autorepair/FixedSplitTokenRangeSplitterHelper.java | 7 +++---- .../FixedSplitTokenRangeSplitterNoVNodesTest.java | 2 +- .../autorepair/FixedSplitTokenRangeSplitterVNodesTest.java | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index 414c924f6250..9d16a48a9107 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -47,10 +47,10 @@ public class FixedSplitTokenRangeSplitter implements IAutoRepairTokenRangeSplitt * Selecting the default value is tricky. If we select a small number, individual repairs would be heavy. * On the other hand, if we select a large number, too many repair sessions would be created. *

        - * The default value of 64 works well for non-vnodes, but with vnodes, it can result in too many repair sessions - * because the total number of repair sessions on a node is calculated as: + * The default value of 64 works well for non-vnodes, but with vnodes, it might result in too many repair sessions + * and to cap the number for vnodes, we use the following formula: *

        - * number_of_subranges * num_token + * Math.max(1, numberOfSubranges / tokens.size()) *

        * To maintain balance, 32 serves as a good default that accommodates both vnodes and non-vnodes effectively. */ diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java index d6c2e8ce70e4..5bd3a5407e4e 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java @@ -48,7 +48,7 @@ import static org.junit.Assert.assertTrue; /** - * Helper class for for {@link FixedSplitTokenRangeSplitterNoVNodesTest} and {@link FixedSplitTokenRangeSplitterVNodesTest} + * Helper class for {@link FixedSplitTokenRangeSplitterNoVNodesTest} and {@link FixedSplitTokenRangeSplitterVNodesTest} */ public class FixedSplitTokenRangeSplitterHelper { @@ -111,7 +111,7 @@ public static void testTokenRangesSplitByTable(int numTokens, int numberOfSubRan List> expectedTokensForATable = new ArrayList<>(); for (int j = 0; j < assignmentsPerTable; j++) { - assertEquals(Arrays.asList(tables.get(i)), assignments.get(i * assignmentsPerTable + j).getTableNames()); + assertEquals(Collections.singletonList(tables.get(i)), assignments.get(i * assignmentsPerTable + j).getTableNames()); assignmentForATable.add(assignments.get(i * assignmentsPerTable + j)); expectedTokensForATable.add(expectedToken.get(i * assignmentsPerTable + j)); } @@ -125,7 +125,6 @@ public static void testTokenRangesSplitByKeyspace(int numTokens, int numberOfSub AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(repairType, true); Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); assertEquals(numTokens, tokens.size()); - List tables = Arrays.asList(TABLE1, TABLE2, TABLE3); List> expectedToken = new ArrayList<>(); for (Range range : tokens) { @@ -151,7 +150,7 @@ public static void testTokenRangesSplitByKeyspace(int numTokens, int numberOfSub compare(numTokens, numberOfSplits, expectedToken, assignments); } - public static void testTokenRangesWithDefaultSplit(int numTokens, int numberOfSubRanges, AutoRepairConfig.RepairType repairType) + public static void testTokenRangesWithDefaultSplit(int numTokens, AutoRepairConfig.RepairType repairType) { int numberOfSplits = calcSplits(numTokens, DEFAULT_NUMBER_OF_SUBRANGES); Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java index 5b86728e574d..a30f3aa76246 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java @@ -77,6 +77,6 @@ public void testTokenRangesSplitByKeyspace() @Test public void testTokenRangesWithDefaultSplit() { - FixedSplitTokenRangeSplitterHelper.testTokenRangesWithDefaultSplit(numTokens, numberOfSubRanges, repairType); + FixedSplitTokenRangeSplitterHelper.testTokenRangesWithDefaultSplit(numTokens, repairType); } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java index 8ca30bd2017c..6839748d1f01 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java @@ -77,6 +77,6 @@ public void testTokenRangesSplitByKeyspace() @Test public void testTokenRangesWithDefaultSplit() { - FixedSplitTokenRangeSplitterHelper.testTokenRangesWithDefaultSplit(numTokens, numberOfSubRanges, repairType); + FixedSplitTokenRangeSplitterHelper.testTokenRangesWithDefaultSplit(numTokens, repairType); } } From be586738f0cbcb95139f7f784b40ac950bfaa065 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Tue, 11 Mar 2025 09:43:55 -0700 Subject: [PATCH 173/199] Adjust comment for DEFAULT_NUMBER_OF_SUBRANGES and NUMBER_OF_SUBRANGES --- .../FixedSplitTokenRangeSplitter.java | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index 9d16a48a9107..3824c63b7e91 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -47,8 +47,8 @@ public class FixedSplitTokenRangeSplitter implements IAutoRepairTokenRangeSplitt * Selecting the default value is tricky. If we select a small number, individual repairs would be heavy. * On the other hand, if we select a large number, too many repair sessions would be created. *

        - * The default value of 64 works well for non-vnodes, but with vnodes, it might result in too many repair sessions - * and to cap the number for vnodes, we use the following formula: + * If vnodes are configured using num_tokens, attempts to evenly subdivide subranges by each range + * using the following formula: *

        * Math.max(1, numberOfSubranges / tokens.size()) *

        @@ -57,19 +57,12 @@ public class FixedSplitTokenRangeSplitter implements IAutoRepairTokenRangeSplitt public static final int DEFAULT_NUMBER_OF_SUBRANGES = 32; /** - * The number of subranges to split each to-be-repaired token range into. Defaults to 1. + * Number of evenly split subranges to create for each node that repair runs for. *

        - * The higher this number, the smaller the repair sessions will be. - *

        - * If you are using vnodes, say 256, then the repair will always go one vnode range at a time. This parameter, - * additionally, will let us further subdivide a given vnode range into subranges. - *

        - * With the value "1" and vnodes of 256, a given table on a node will undergo the repair 256 times. But with a - * value "2", the same table on a node will undergo a repair 512 times because every vnode range will be further - * divided by two. - *

        - * If you do not use vnodes or the number of vnodes is pretty small, say 8, setting this value to a higher number, - * such as 16, will be useful to repair on a smaller range, and the chance of succeeding is higher. + * If vnodes are configured using num_tokens, attempts to evenly subdivide subranges by each range. + * For example, for num_tokens: 16 and number_of_subranges: 32, 2 (32/16) + * repair assignments will be created for each token range. At least one repair assignment will be + * created for each token range. */ static final String NUMBER_OF_SUBRANGES = "number_of_subranges"; From 8fd000e7edb812eed2de0520f6d0bc3b8e467db8 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Wed, 12 Mar 2025 18:34:06 -0700 Subject: [PATCH 174/199] Adjust as per 4.1 branch code --- .../cassandra/pages/operating/repair.adoc | 10 ++++-- .../FixedSplitTokenRangeSplitterHelper.java | 36 +++++++++---------- ...edSplitTokenRangeSplitterNoVNodesTest.java | 4 ++- ...ixedSplitTokenRangeSplitterVNodesTest.java | 4 ++- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/doc/modules/cassandra/pages/operating/repair.adoc b/doc/modules/cassandra/pages/operating/repair.adoc index 01926255554d..3f6acbc58de5 100644 --- a/doc/modules/cassandra/pages/operating/repair.adoc +++ b/doc/modules/cassandra/pages/operating/repair.adoc @@ -73,9 +73,13 @@ Or even on specific tables: nodetool repair [options] ---- -The repair command repairs token ranges only on the node being repaired; it does not repair the whole cluster. -By default, repair operates on all token ranges replicated by the node on which repair is run, causing duplicate work when running it on every node. Avoid duplicate work by using the `-pr` flag to repair only the "primary" ranges on a node. -Do a full cluster repair by running the `nodetool repair -pr` command on each node in each datacenter in the cluster, until all of the nodes and datacenters are repaired. +The repair command only repairs token ranges on the node being repaired, +it doesn't repair the whole cluster. By default, repair will operate on +all token ranges replicated by the node you're running repair on, which +will cause duplicate work if you run it on every node. The `-pr` flag +will only repair the "primary" ranges on a node, so you can repair your +entire cluster by running `nodetool repair -pr` on each node in a single +datacenter. The specific frequency of repair that's right for your cluster, of course, depends on several factors. However, if you're just starting out diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java index 5bd3a5407e4e..e0198c58bde0 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java @@ -27,20 +27,20 @@ import java.util.Set; import java.util.TreeSet; -import org.apache.cassandra.ServerTestUtils; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.dht.BootStrapper; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.index.sai.disk.format.Version; +import org.apache.cassandra.locator.TokenMetadata; +import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.utils.FBUtilities; +import static org.apache.cassandra.Util.setAutoRepairEnabled; import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; -import static org.apache.cassandra.cql3.CQLTester.Fuzzed.setupSeed; -import static org.apache.cassandra.cql3.CQLTester.Fuzzed.updateConfigs; import static org.apache.cassandra.repair.autorepair.FixedSplitTokenRangeSplitter.DEFAULT_NUMBER_OF_SUBRANGES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -50,7 +50,7 @@ /** * Helper class for {@link FixedSplitTokenRangeSplitterNoVNodesTest} and {@link FixedSplitTokenRangeSplitterVNodesTest} */ -public class FixedSplitTokenRangeSplitterHelper +public class FixedSplitTokenRangeSplitterHelper extends CQLTester { private static final String TABLE1 = "tbl1"; private static final String TABLE2 = "tbl2"; @@ -59,19 +59,17 @@ public class FixedSplitTokenRangeSplitterHelper public static void setupClass(int numTokens) throws Exception { - setupSeed(); - updateConfigs(); - DatabaseDescriptor.setPartitioner("org.apache.cassandra.dht.Murmur3Partitioner"); - ServerTestUtils.prepareServerNoRegister(); + AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("0s"); + SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); + setAutoRepairEnabled(true); - Set tokens = BootStrapper.getRandomTokens(ClusterMetadata.current(), numTokens); - ServerTestUtils.registerLocal(tokens); - // Ensure that the on-disk format statics are loaded before the test run - Version.LATEST.onDiskFormat(); - StorageService.instance.doAutoRepairSetup(); + requireNetwork(); + AutoRepairUtils.setup(); + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1)); - SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); - QueryProcessor.executeInternal(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", FixedSplitTokenRangeSplitterHelper.KEYSPACE)); + TokenMetadata metadata = StorageService.instance.getTokenMetadata(); + metadata.updateNormalTokens(BootStrapper.getRandomTokens(metadata, numTokens), FBUtilities.getBroadcastAddressAndPort()); } public static void testTokenRangesSplitByTable(int numTokens, int numberOfSubRanges, AutoRepairConfig.RepairType repairType) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java index a30f3aa76246..a173299f4a73 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterNoVNodesTest.java @@ -28,11 +28,13 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.apache.cassandra.cql3.CQLTester; + /** * Unit tests for a setup that does not have v-nodes {@link FixedSplitTokenRangeSplitter} */ @RunWith(Parameterized.class) -public class FixedSplitTokenRangeSplitterNoVNodesTest +public class FixedSplitTokenRangeSplitterNoVNodesTest extends CQLTester { private static final int numTokens = 1; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java index 6839748d1f01..d7ef654a3b21 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterVNodesTest.java @@ -28,11 +28,13 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.apache.cassandra.cql3.CQLTester; + /** * Unit tests for a setup that has v-nodes {@link FixedSplitTokenRangeSplitter} */ @RunWith(Parameterized.class) -public class FixedSplitTokenRangeSplitterVNodesTest +public class FixedSplitTokenRangeSplitterVNodesTest extends CQLTester { private static final int numTokens = 16; From 7b62e399e5c6b41ee24a8cf2955e64454040232a Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Tue, 11 Mar 2025 17:06:07 -0700 Subject: [PATCH 175/199] Move auto-repair retry policy config to repair-type level --- .../repair/autorepair/AutoRepair.java | 8 ++-- .../repair/autorepair/AutoRepairConfig.java | 48 +++++++++---------- .../cassandra/service/AutoRepairService.java | 28 +++++------ .../service/AutoRepairServiceMBean.java | 7 ++- .../org/apache/cassandra/tools/NodeProbe.java | 20 ++++---- .../tools/nodetool/SetAutoRepairConfig.java | 12 ++--- .../AutoRepairParameterizedTest.java | 12 ++--- .../service/AutoRepairServiceTest.java | 8 ++-- .../nodetool/SetAutoRepairConfigTest.java | 24 ++-------- 9 files changed, 74 insertions(+), 93 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index b793d91e650d..655565d95a3c 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -313,7 +313,7 @@ private void repairKeyspace(AutoRepairConfig.RepairType repairType, boolean prim boolean success = false; int retryCount = 0; Future f = null; - while (retryCount <= config.getRepairMaxRetries()) + while (retryCount <= config.getRepairMaxRetries(repairType)) { RepairRunnable task = repairState.getRepairRunnable(keyspaceName, Lists.newArrayList(curRepairAssignment.getTableNames()), @@ -336,14 +336,14 @@ private void repairKeyspace(AutoRepairConfig.RepairType repairType, boolean prim { break; } - else if (retryCount < config.getRepairMaxRetries()) + else if (retryCount < config.getRepairMaxRetries(repairType)) { boolean cancellationStatus = f.cancel(true); logger.warn("Repair failed for range {}-{} for {} tables {} with cancellationStatus: {} retrying after {} seconds...", tokenRange.left, tokenRange.right, keyspaceName, curRepairAssignment.getTableNames(), - cancellationStatus, config.getRepairRetryBackoff().toSeconds()); - sleepFunc.accept(config.getRepairRetryBackoff().toSeconds(), TimeUnit.SECONDS); + cancellationStatus, config.getRepairRetryBackoff(repairType).toSeconds()); + sleepFunc.accept(config.getRepairRetryBackoff(repairType).toSeconds(), TimeUnit.SECONDS); } retryCount++; } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 1b1596f1a0cf..19f4fb941e52 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -55,10 +55,6 @@ public class AutoRepairConfig implements Serializable // The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata // for a specified duration to ensure they are indeed removed before adjustments are made to the schedule. public volatile DurationSpec.IntSecondsBound history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound("2h"); - // Maximum number of retries for a repair session. - public volatile Integer repair_max_retries = 3; - // Backoff time before retrying a repair session. - public volatile DurationSpec.LongSecondsBound repair_retry_backoff = new DurationSpec.LongSecondsBound("30s"); // Minimum duration for the execution of a single repair task. This prevents the scheduler from overwhelming // the node by scheduling too many repair tasks in a short period of time. public volatile DurationSpec.LongSecondsBound repair_task_min_duration = new DurationSpec.LongSecondsBound("5s"); @@ -168,26 +164,6 @@ public void setAutoRepairHistoryClearDeleteHostsBufferInterval(String duration) history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound(duration); } - public int getRepairMaxRetries() - { - return repair_max_retries; - } - - public void setRepairMaxRetries(int maxRetries) - { - repair_max_retries = maxRetries; - } - - public DurationSpec.LongSecondsBound getRepairRetryBackoff() - { - return repair_retry_backoff; - } - - public void setRepairRetryBackoff(String interval) - { - repair_retry_backoff = new DurationSpec.LongSecondsBound(interval); - } - public DurationSpec.LongSecondsBound getRepairTaskMinDuration() { return repair_task_min_duration; @@ -359,6 +335,26 @@ public void setRepairSessionTimeout(RepairType repairType, String repairSessionT getOptions(repairType).repair_session_timeout = new DurationSpec.IntSecondsBound(repairSessionTimeout); } + public int getRepairMaxRetries(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.repair_max_retries); + } + + public void setRepairMaxRetries(RepairType repairType, int maxRetries) + { + getOptions(repairType).repair_max_retries = maxRetries; + } + + public DurationSpec.LongSecondsBound getRepairRetryBackoff(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.repair_retry_backoff); + } + + public void setRepairRetryBackoff(RepairType repairType, String interval) + { + getOptions(repairType).repair_retry_backoff = new DurationSpec.LongSecondsBound(interval); + } + @VisibleForTesting static IAutoRepairTokenRangeSplitter newAutoRepairTokenRangeSplitter(RepairType repairType, ParameterizedClass parameterizedClass) throws ConfigurationException { @@ -504,6 +500,10 @@ protected static Options getDefaultOptions() public volatile DurationSpec.IntSecondsBound initial_scheduler_delay; // Timeout for resuming stuck repair sessions. public volatile DurationSpec.IntSecondsBound repair_session_timeout; + // Maximum number of retries for a repair session. + public volatile Integer repair_max_retries = 3; + // Backoff time before retrying a repair session. + public volatile DurationSpec.LongSecondsBound repair_retry_backoff = new DurationSpec.LongSecondsBound("30s"); public String toString() { diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 0cf744296e07..9f66a92abc95 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -100,8 +100,6 @@ public String getAutoRepairConfiguration() StringBuilder sb = new StringBuilder(); sb.append("repair scheduler configuration:"); appendConfig(sb, "repair_check_interval", config.getRepairCheckInterval()); - appendConfig(sb, "repair_max_retries", config.getRepairMaxRetries()); - appendConfig(sb, "repair_retry_backoff", config.getRepairRetryBackoff()); appendConfig(sb, "repair_task_min_duration", config.getRepairTaskMinDuration()); appendConfig(sb, "history_clear_delete_hosts_buffer_interval", config.getAutoRepairHistoryClearDeleteHostsBufferInterval()); for (RepairType repairType : RepairType.values()) @@ -162,18 +160,6 @@ public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration) config.setAutoRepairHistoryClearDeleteHostsBufferInterval(duration); } - @Override - public void setAutoRepairMaxRetriesCount(int retries) - { - config.setRepairMaxRetries(retries); - } - - @Override - public void setAutoRepairRetryBackoff(String interval) - { - config.setRepairRetryBackoff(interval); - } - @Override public void setAutoRepairMinRepairTaskDuration(String duration) { @@ -261,6 +247,18 @@ public void setRepairByKeyspace(String repairType, boolean repairByKeyspace) config.setRepairByKeyspace(RepairType.parse(repairType), repairByKeyspace); } + @Override + public void setAutoRepairMaxRetriesCount(String repairType, int retries) + { + config.setRepairMaxRetries(RepairType.parse(repairType), retries); + } + + @Override + public void setAutoRepairRetryBackoff(String repairType, String interval) + { + config.setRepairRetryBackoff(RepairType.parse(repairType), interval); + } + private String formatRepairTypeConfig(RepairType repairType, AutoRepairConfig config) { StringBuilder sb = new StringBuilder(); @@ -288,6 +286,8 @@ private String formatRepairTypeConfig(RepairType repairType, AutoRepairConfig co appendConfig(sb , "initial_scheduler_delay", config.getInitialSchedulerDelay(repairType)); appendConfig(sb , "repair_session_timeout", config.getRepairSessionTimeout(repairType)); appendConfig(sb , "force_repair_new_node", config.getForceRepairNewNode(repairType)); + appendConfig(sb , "repair_max_retries", config.getRepairMaxRetries(repairType)); + appendConfig(sb , "repair_retry_backoff", config.getRepairRetryBackoff(repairType)); final ParameterizedClass splitterClass = config.getTokenRangeSplitter(repairType); final String splitterClassName = splitterClass.class_name != null ? splitterClass.class_name : AutoRepairConfig.DEFAULT_SPLITTER.getName(); diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index d57329e86795..72d9da7c5a0c 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -39,10 +39,6 @@ public interface AutoRepairServiceMBean public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration); - public void setAutoRepairMaxRetriesCount(int retries); - - public void setAutoRepairRetryBackoff(String interval); - public void setAutoRepairMinRepairTaskDuration(String duration); public void setRepairSSTableCountHigherThreshold(String repairType, int ssTableHigherThreshold); @@ -71,4 +67,7 @@ public interface AutoRepairServiceMBean public void setRepairByKeyspace(String repairType, boolean repairByKeyspace); + public void setAutoRepairMaxRetriesCount(String repairType, int retries); + + public void setAutoRepairRetryBackoff(String repairType, String interval); } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index d1f3e27d2573..816c658144af 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2193,16 +2193,6 @@ public void startAutoRepairScheduler() autoRepairProxy.startScheduler(); } - public void setAutoRepairMaxRetriesCount(int retries) - { - autoRepairProxy.setAutoRepairMaxRetriesCount(retries); - } - - public void setAutoRepairRetryBackoff(String interval) - { - autoRepairProxy.setAutoRepairRetryBackoff(interval); - } - public void setAutoRepairMinRepairTaskDuration(String duration) { autoRepairProxy.setAutoRepairMinRepairTaskDuration(duration); @@ -2267,6 +2257,16 @@ public void setAutoRepairRepairByKeyspace(String repairType, boolean enabled) { autoRepairProxy.setRepairByKeyspace(repairType, enabled); } + + public void setAutoRepairMaxRetriesCount(String repairType, int retries) + { + autoRepairProxy.setAutoRepairMaxRetriesCount(repairType, retries); + } + + public void setAutoRepairRetryBackoff(String repairType, String interval) + { + autoRepairProxy.setAutoRepairRetryBackoff(repairType, interval); + } } class ColumnFamilyStoreMBeanIterator implements Iterator> diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index 37f809943fb6..5a9a1411bb59 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -87,12 +87,6 @@ public void execute(NodeProbe probe) case "history_clear_delete_hosts_buffer_interval": probe.setAutoRepairHistoryClearDeleteHostsBufferDuration(paramVal); return; - case "repair_max_retries": - probe.setAutoRepairMaxRetriesCount(Integer.parseInt(paramVal)); - return; - case "repair_retry_backoff": - probe.setAutoRepairRetryBackoff(paramVal); - return; case "min_repair_task_duration": probe.setAutoRepairMinRepairTaskDuration(paramVal); return; @@ -163,6 +157,12 @@ public void execute(NodeProbe probe) case "repair_by_keyspace": probe.setAutoRepairRepairByKeyspace(repairTypeStr, Boolean.parseBoolean(paramVal)); break; + case "repair_max_retries": + probe.setAutoRepairMaxRetriesCount(repairTypeStr, Integer.parseInt(paramVal)); + break; + case "repair_retry_backoff": + probe.setAutoRepairRetryBackoff(repairTypeStr, paramVal); + break; default: throw new IllegalArgumentException("Unknown parameter: " + paramType); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 5a4be0d800ec..2b8c1d51e3be 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -519,7 +519,7 @@ public void testMetrics() AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setMaterializedViewRepairEnabled(repairType, true); config.setRepairMinInterval(repairType, "0s"); - config.setRepairRetryBackoff("0s"); + config.setRepairRetryBackoff(repairType, "0s"); config.setAutoRepairTableMaxRepairTime(repairType, "0s"); AutoRepair.timeFunc = () -> { timeFuncCalls++; @@ -683,7 +683,7 @@ public void testRepairMaxRetries() AutoRepair.sleepFunc = (Long duration, TimeUnit unit) -> { sleepCalls.getAndIncrement(); assertEquals(TimeUnit.SECONDS, unit); - assertEquals(config.getRepairRetryBackoff().toSeconds(), (long) duration); + assertEquals(config.getRepairRetryBackoff(repairType).toSeconds(), (long) duration); }; config.setRepairMinInterval(repairType, "0s"); AutoRepair.instance.repairStates.put(repairType, autoRepairState); @@ -691,7 +691,7 @@ public void testRepairMaxRetries() AutoRepair.instance.repair(repairType); // Expect configured retries for each keyspace expected to be repaired - assertEquals(config.getRepairMaxRetries()*expectedRepairAssignments, sleepCalls.get()); + assertEquals(config.getRepairMaxRetries(repairType)*expectedRepairAssignments, sleepCalls.get()); verify(autoRepairState, times(1)).setSucceededTokenRangesCount(0); verify(autoRepairState, times(1)).setSkippedTokenRangesCount(0); verify(autoRepairState, times(1)).setFailedTokenRangesCount(expectedRepairAssignments); @@ -707,7 +707,7 @@ public void testRepairSuccessAfterRetry() AutoRepair.sleepFunc = (Long duration, TimeUnit unit) -> { sleepCalls.getAndIncrement(); assertEquals(TimeUnit.SECONDS, unit); - assertEquals(config.getRepairRetryBackoff().toSeconds(), (long) duration); + assertEquals(config.getRepairRetryBackoff(repairType).toSeconds(), (long) duration); }; doAnswer(invocation -> { if (sleepCalls.get() == 0) @@ -722,7 +722,7 @@ public void testRepairSuccessAfterRetry() return null; }).when(repairRunnable).addProgressListener(any()); config.setRepairMinInterval(repairType, "0s"); - config.setRepairMaxRetries(1); + config.setRepairMaxRetries(repairType, 1); AutoRepair.instance.repairStates.put(repairType, autoRepairState); AutoRepair.instance.repair(repairType); @@ -896,7 +896,7 @@ public void testSchedulerIgnoresErrorsFromUnrelatedRepairRunables() return runnable; }).when(spyState).getRepairRunnable(any(), any(), any(), anyBoolean()); when(spyState.getLastRepairTime()).thenReturn((long) 0); - AutoRepairService.instance.getAutoRepairConfig().setRepairMaxRetries(0); + AutoRepairService.instance.getAutoRepairConfig().setRepairMaxRetries(repairType, 0); AutoRepair.instance.repairStates.put(repairType, spyState); AutoRepair.instance.repair(repairType); diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index a7d3a8a7bf9b..7617fddbe830 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -103,17 +103,17 @@ public void testsetAutoRepairHistoryClearDeleteHostsBufferInSecV2() @Test public void testsetAutoRepairMaxRetriesCount() { - autoRepairService.setAutoRepairMaxRetriesCount(101); + autoRepairService.setAutoRepairMaxRetriesCount(AutoRepairConfig.RepairType.INCREMENTAL.name(), 101); - assertEquals(101, config.getRepairMaxRetries()); + assertEquals(101, config.getRepairMaxRetries(AutoRepairConfig.RepairType.INCREMENTAL.name(), )); } @Test public void testsetAutoRepairRetryBackoffInSec() { - autoRepairService.setAutoRepairRetryBackoff("102s"); + autoRepairService.setAutoRepairRetryBackoff(AutoRepairConfig.RepairType.INCREMENTAL.name(), "102s"); - assertEquals(102, config.getRepairRetryBackoff().toSeconds()); + assertEquals(102, config.getRepairRetryBackoff(AutoRepairConfig.RepairType.INCREMENTAL).toSeconds()); } @Test(expected = ConfigurationException.class) diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index 8027eadb6232..7bc9d28d8278 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -95,25 +95,6 @@ public void testHistoryDeleteHostsClearBufferInSec() verify(probe, times(1)).setAutoRepairHistoryClearDeleteHostsBufferDuration("1s"); } - @Test - public void testRepairMaxRetries() - { - cmd.args = ImmutableList.of("repair_max_retries", "2"); - - cmd.execute(probe); - - verify(probe, times(1)).setAutoRepairMaxRetriesCount(2); - } - - @Test - public void testRetryBackoffInSec() - { - cmd.args = ImmutableList.of("repair_retry_backoff", "3s"); - - cmd.execute(probe); - - verify(probe, times(1)).setAutoRepairRetryBackoff("3s"); - } @Test public void testMinRepairDuration() @@ -125,7 +106,6 @@ public void testMinRepairDuration() verify(probe, times(1)).setAutoRepairMinRepairTaskDuration("4s"); } - @Test public void testStartScheduler() { cmd.args = ImmutableList.of("start_scheduler", "false"); @@ -286,7 +266,9 @@ public static Collection testCases() forEachRepairType("parallel_repair_percentage", "7", (type) -> verify(probe, times(1)).setAutoRepairParallelRepairPercentage(type.name(), 7)), forEachRepairType("materialized_view_repair_enabled", "true", (type) -> verify(probe, times(1)).setAutoRepairMaterializedViewRepairEnabled(type.name(), true)), forEachRepairType("ignore_dcs", "dc1,dc2", (type) -> verify(probe, times(1)).setAutoRepairIgnoreDCs(type.name(), ImmutableSet.of("dc1", "dc2"))), - forEachRepairType("token_range_splitter.max_bytes_per_schedule", "500GiB", (type) -> verify(probe, times(1)).setAutoRepairTokenRangeSplitterParameter(type.name(), "max_bytes_per_schedule", "500GiB")) + forEachRepairType("token_range_splitter.max_bytes_per_schedule", "500GiB", (type) -> verify(probe, times(1)).setAutoRepairTokenRangeSplitterParameter(type.name(), "max_bytes_per_schedule", "500GiB")), + forEachRepairType("repair_max_retries", "3", (type) -> verify(probe, times(1)).setAutoRepairMaxRetriesCount(type.name(), 3)), + forEachRepairType("repair_retry_backoff", "60s", (type) -> verify(probe, times(1)).setAutoRepairRetryBackoff(type.name(), "60s")) ).flatMap(Function.identity()).collect(Collectors.toList()); } From 847673854f13709d028bc609f6124af76174cb8d Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Wed, 12 Mar 2025 20:32:30 -0700 Subject: [PATCH 176/199] Rebase From 91e2e2efba30e483759bd3f8df7501b7162bd56a Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Wed, 12 Mar 2025 20:37:50 -0700 Subject: [PATCH 177/199] Update docs --- conf/cassandra.yaml | 8 ++++---- .../cassandra/managing/operating/auto_repair.adoc | 12 +++++++----- .../cassandra/service/AutoRepairServiceTest.java | 2 +- .../tools/nodetool/SetAutoRepairConfigTest.java | 1 + 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index fb07effd11ae..4c9aec53d583 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1960,10 +1960,6 @@ drop_compact_storage_enabled: false # # Time interval between successive checks to see if ongoing repairs are complete or if it is time to schedule # # repairs. # repair_check_interval: 5m -# # Maximum number of retries for a repair session. -# repair_max_retries: 3 -# # Backoff time before retrying a repair session. -# repair_retry_backoff: 30s # # Minimum duration for the execution of a single repair task. This prevents the scheduler from overwhelming # # the node by scheduling too many repair tasks in a short period of time. # repair_task_min_duration: 5s @@ -2005,6 +2001,10 @@ drop_compact_storage_enabled: false # # Repair only the primary ranges owned by a node. Equivalent to the -pr option in nodetool repair. Defaults # # to true. General advice is to keep this true. # repair_primary_token_range_only: true +# # Maximum number of retries for a repair session. +# repair_max_retries: 3 +# # Backoff time before retrying a repair session. +# repair_retry_backoff: 30s # token_range_splitter: # # Splitter implementation to generate repair assignments. Defaults to RepairTokenRangeSplitter. # class_name: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter diff --git a/doc/modules/cassandra/managing/operating/auto_repair.adoc b/doc/modules/cassandra/managing/operating/auto_repair.adoc index c456536d905c..a1bf57acf158 100644 --- a/doc/modules/cassandra/managing/operating/auto_repair.adoc +++ b/doc/modules/cassandra/managing/operating/auto_repair.adoc @@ -164,9 +164,6 @@ for each repair type (full, incremental, and preview_repaired), and based on tha | repair_check_interval | 5m | Time interval between successive checks to see if ongoing repairs are complete or if it is time to schedule repairs. | repair_max_retries | 3 | Maximum number of retries for a repair session. -| repair_retry_backoff | 30s | Backoff time before retrying a repair session. -| repair_task_min_duration | 5s | Minimum duration for the execution of a single repair task. This prevents the -scheduler from overwhelming the node by scheduling too many repair tasks in a short period of time. | history_clear_delete_hosts_buffer_interval | 2h | The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata for a specified duration to ensure they are indeed removed before adjustments are made to the schedule. @@ -212,6 +209,9 @@ centers. Useful if you have keyspaces that are not replicated in certain data ce schedule in certain data centers. | repair_primary_token_range_only | true | Repair only the primary ranges owned by a node. Equivalent to the -pr option in nodetool repair. General advice is to keep this true. +| repair_retry_backoff | 30s | Backoff time before retrying a repair session. +| repair_task_min_duration | 5s | Minimum duration for the execution of a single repair task. This prevents the +scheduler from overwhelming the node by scheduling too many repair tasks in a short period of time. |=== === `RepairTokenRangeSplitter` configuration @@ -352,8 +352,6 @@ $> nodetool getautorepairconfig repair scheduler configuration: repair_check_interval: 5m repair_max_retries: 3 - repair_retry_backoff: 30s - repair_task_min_duration: 5s history_clear_delete_hosts_buffer_interval: 2h configuration for repair_type: full enabled: true @@ -370,6 +368,8 @@ configuration for repair_type: full initial_scheduler_delay: 5m repair_session_timeout: 3h force_repair_new_node: false + repair_retry_backoff: 30s + repair_task_min_duration: 5s token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter token_range_splitter.bytes_per_assignment: 50GiB token_range_splitter.partitions_per_assignment: 1048576 @@ -390,6 +390,8 @@ configuration for repair_type: incremental initial_scheduler_delay: 5m repair_session_timeout: 3h force_repair_new_node: false + repair_retry_backoff: 30s + repair_task_min_duration: 5s token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter token_range_splitter.bytes_per_assignment: 50GiB token_range_splitter.partitions_per_assignment: 1048576 diff --git a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java index 7617fddbe830..4d041e8bbb84 100644 --- a/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AutoRepairServiceTest.java @@ -105,7 +105,7 @@ public void testsetAutoRepairMaxRetriesCount() { autoRepairService.setAutoRepairMaxRetriesCount(AutoRepairConfig.RepairType.INCREMENTAL.name(), 101); - assertEquals(101, config.getRepairMaxRetries(AutoRepairConfig.RepairType.INCREMENTAL.name(), )); + assertEquals(101, config.getRepairMaxRetries(AutoRepairConfig.RepairType.INCREMENTAL)); } @Test diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index 7bc9d28d8278..28686365a3cb 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -106,6 +106,7 @@ public void testMinRepairDuration() verify(probe, times(1)).setAutoRepairMinRepairTaskDuration("4s"); } + @Test public void testStartScheduler() { cmd.args = ImmutableList.of("start_scheduler", "false"); From 9b20caa15da36cb87c62afdc97e19ae6ddc739c6 Mon Sep 17 00:00:00 2001 From: Andy Tolbert <6889771+tolbertam@users.noreply.github.com> Date: Wed, 12 Mar 2025 20:34:43 -0500 Subject: [PATCH 178/199] Avoid returning MY_TURN when replicas are actively taking their turn Previously AutoRepairUtils.myTurnToRunRepair would return MY_TURN if there is capacity to select a new node for repair and the current node has the oldest lastRepairFinishTime. This change adjusts this method to make use of a newly added method getMostEligibleHostToRepair which will identify the node with the oldest lastRepairFinishTime of which none of the node's replicas are actively repairing. This should prevent replicas from actively repairing concurrently, which would increase load on replica sets and also create anticompaction conflicts for incremental repair. Behavior is configurable by allow_parallel_replica_repair and allow_parallel_replica_repair_across_schedules configuration. patch by Andy Tolbert, reviewed by Jaydeepkumar Chovatia and Francisco Guerrero for CASSANDRA-20180 --- conf/cassandra.yaml | 9 + .../managing/operating/auto_repair.adoc | 7 + .../cassandra/pages/operating/metrics.adoc | 11 +- .../cassandra/metrics/AutoRepairMetrics.java | 39 ++ .../repair/autorepair/AutoRepair.java | 22 +- .../repair/autorepair/AutoRepairConfig.java | 33 ++ .../repair/autorepair/AutoRepairUtils.java | 379 ++++++++++++++++-- .../cassandra/service/AutoRepairService.java | 46 ++- .../service/AutoRepairServiceMBean.java | 4 + .../cassandra/service/StorageService.java | 15 +- .../org/apache/cassandra/tools/NodeProbe.java | 10 + .../tools/nodetool/SetAutoRepairConfig.java | 10 +- ...allelReplicaRepairAcrossSchedulesTest.java | 139 +++++++ .../test/repair/AutoRepairSchedulerTest.java | 153 ++++--- .../autorepair/AutoRepairConfigTest.java | 69 +++- .../autorepair/AutoRepairMetricsTest.java | 92 +++++ .../AutoRepairParameterizedTest.java | 2 + .../autorepair/AutoRepairUtilsTest.java | 3 +- 18 files changed, 925 insertions(+), 118 deletions(-) create mode 100644 test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairMetricsTest.java diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 4c9aec53d583..b28e225bb061 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1979,6 +1979,15 @@ drop_compact_storage_enabled: false # # Percentage of nodes in the cluster running repair in parallel. If parallel_repair_count is set, the larger value # # is used. # parallel_repair_percentage: 3 +# # Whether to allow a node to take its turn running repair while one or more of its replicas are running repair. +# # Defaults to false, as running repairs concurrently on replicas can increase load and also cause anticompaction +# # conflicts while running incremental repair. +# allow_parallel_replica_repair: false +# # An addition to allow_parallel_replica_repair that also blocks repairs when replicas (including this node itself) +# # are repairing in any schedule. For example, if a replica is executing full repairs, a value of false will +# # prevent starting incremental repairs for this node. Defaults to true and is only evaluated when +# # allow_parallel_replica_repair is false. +# allow_parallel_replica_repair_across_schedules: true # # Repairs materialized views if true. # materialized_view_repair_enabled: false # # Delay before starting repairs after a node restarts to avoid repairs starting immediately after a restart. diff --git a/doc/modules/cassandra/managing/operating/auto_repair.adoc b/doc/modules/cassandra/managing/operating/auto_repair.adoc index a1bf57acf158..06c0db456753 100644 --- a/doc/modules/cassandra/managing/operating/auto_repair.adoc +++ b/doc/modules/cassandra/managing/operating/auto_repair.adoc @@ -195,6 +195,13 @@ the -j option in nodetool repair. larger value is used. | parallel_repair_percentage | 3 | Percentage of nodes in the cluster running repair in parallel. If `parallel_repair_count is set`, the larger value is used. +| allow_parallel_replica_repair | false | Whether to allow a node to take its turn running repair while one or more of +its replicas are running repair. Defaults to false, as running repairs concurrently on replicas can increase load and +also cause anticompaction conflicts while running incremental repair. +| allow_parallel_replica_repair_across_schedules | true | An addition to allow_parallel_repair that also blocks repairs +when replicas (including this node itself) are repairing in any schedule. +For example, if a replica is executing full repairs, a value of false will prevent starting incremental repairs for this +node. Defaults to true and is only evaluated when allow_parallel_replica_repair is false. | materialized_view_repair_enabled | false | Repairs materialized views if true. | initial_scheduler_delay | 5m | Delay before starting repairs after a node restarts to avoid repairs starting immediately after a restart. diff --git a/doc/modules/cassandra/pages/operating/metrics.adoc b/doc/modules/cassandra/pages/operating/metrics.adoc index 671ea3ae3ecb..451b9b245bfa 100644 --- a/doc/modules/cassandra/pages/operating/metrics.adoc +++ b/doc/modules/cassandra/pages/operating/metrics.adoc @@ -978,6 +978,9 @@ the entire Cassandra cluster in seconds |LongestUnrepairedSec |Gauge |Time since the last repair ran on the node in seconds +|RepairStartLagSec|Gauge |If a repair has not run within min_repair_interval, how long past this value since +repairs last completed. Useful for determining if repairs are behind schedule. + |SucceededTokenRangesCount |Gauge |Number of token ranges successfully repaired on the node |FailedTokenRangesCount |Gauge |Number of token ranges failed to repair on the node @@ -999,9 +1002,13 @@ the automated repair has been disabled on the node |RepairTurnMyTurnDueToPriority |Counter |Represents the node's turn to repair due to priority set in the automated repair -|RepairTurnMyTurnForceRepair |Counter |Represents the node's turn to repair -due to force repair set in the automated repair +|RepairDelayedByReplica |Counter |Represents occurrences of a node's turn being +delayed because a replica was currently taking its turn. Only revelent if +`allow_parallel_replica_repair` is false. +|RepairDelayedBySchedule |Counter |Represents occurrences of a node's turn being +delayed because it was already being repaired in another schedule. Only relevent +if `allow_parallel_replica_repair_across_schedules` is false. |=== diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java index b83d2d8a512c..8efc8d679ab2 100644 --- a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java @@ -23,7 +23,9 @@ import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; import org.apache.cassandra.repair.autorepair.AutoRepairUtils; import org.apache.cassandra.repair.autorepair.AutoRepair; +import org.apache.cassandra.service.AutoRepairService; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; /** @@ -35,6 +37,7 @@ public class AutoRepairMetrics public final Gauge nodeRepairTimeInSec; public final Gauge clusterRepairTimeInSec; public final Gauge longestUnrepairedSec; + public final Gauge repairStartLagSec; public final Gauge succeededTokenRangesCount; public final Gauge failedTokenRangesCount; public final Gauge skippedTokenRangesCount; @@ -44,8 +47,16 @@ public class AutoRepairMetrics public Counter repairTurnMyTurn; public Counter repairTurnMyTurnDueToPriority; public Counter repairTurnMyTurnForceRepair; + public Counter repairDelayedByReplica; + public Counter repairDelayedBySchedule; + + private final RepairType repairType; + + private volatile int repairStartLagSecVal; + public AutoRepairMetrics(RepairType repairType) { + this.repairType = repairType; AutoRepairMetricsFactory factory = new AutoRepairMetricsFactory(repairType); repairsInProgress = Metrics.register(factory.createMetricName("RepairsInProgress"), new Gauge() @@ -96,6 +107,14 @@ public Integer getValue() } }); + repairStartLagSec = Metrics.register(factory.createMetricName("RepairStartLagSec"), new Gauge() + { + public Integer getValue() + { + return repairStartLagSecVal; + } + }); + succeededTokenRangesCount = Metrics.register(factory.createMetricName("SucceededTokenRangesCount"), new Gauge() { public Integer getValue() @@ -116,6 +135,9 @@ public Integer getValue() repairTurnMyTurnDueToPriority = Metrics.counter(factory.createMetricName("RepairTurnMyTurnDueToPriority")); repairTurnMyTurnForceRepair = Metrics.counter(factory.createMetricName("RepairTurnMyTurnForceRepair")); + repairDelayedByReplica = Metrics.counter(factory.createMetricName("RepairDelayedByReplica")); + repairDelayedBySchedule = Metrics.counter(factory.createMetricName("RepairDelayedBySchedule")); + totalMVTablesConsideredForRepair = Metrics.register(factory.createMetricName("TotalMVTablesConsideredForRepair"), new Gauge() { public Integer getValue() @@ -149,6 +171,23 @@ public void recordTurn(AutoRepairUtils.RepairTurn turn) default: throw new RuntimeException(String.format("Unrecoginized turn: %s", turn.name())); } + this.repairStartLagSecVal = 0; + } + + /** + * Record perceived lag in scheduling repair. + *

        + * Takes the current time and subtracts it from the given last repair finish time. It then compares the difference + * with the min repair interval for this repair type, and if that value is greater than 0, records it. + */ + public void recordRepairStartLag(long lastFinishTimeInMs) + { + long now = AutoRepair.instance.currentTimeMs(); + long deltaFinish = now - lastFinishTimeInMs; + long deltaMinRepairInterval = deltaFinish - AutoRepairService.instance + .getAutoRepairConfig().getRepairMinInterval(repairType) + .toMilliseconds(); + this.repairStartLagSecVal = deltaMinRepairInterval > 0 ? (int) MILLISECONDS.toSeconds(deltaMinRepairInterval) : 0; } @VisibleForTesting diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 655565d95a3c..f92f7d984d27 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -154,6 +154,14 @@ public void repairAsync(AutoRepairConfig.RepairType repairType) repairExecutors.get(repairType).submit(() -> repair(repairType)); } + /** + * @return The current observed system time in ms. + */ + public long currentTimeMs() + { + return timeFunc.get(); + } + // repair runs a repair session of the given type synchronously. public void repair(AutoRepairConfig.RepairType repairType) { @@ -179,6 +187,13 @@ public void repair(AutoRepairConfig.RepairType repairType) //consistency level to use for local query UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); + + // If it's too soon to run repair, don't bother checking if it's our turn. + if (tooSoonToRunRepair(repairType, repairState, config, myId)) + { + return; + } + RepairTurn turn = AutoRepairUtils.myTurnToRunRepair(repairType, myId); if (turn == MY_TURN || turn == MY_TURN_DUE_TO_PRIORITY || turn == MY_TURN_FORCE_REPAIR) { @@ -189,10 +204,6 @@ public void repair(AutoRepairConfig.RepairType repairType) // When doing force repair, we want to repair without -pr. boolean primaryRangeOnly = config.getRepairPrimaryTokenRangeOnly(repairType) && turn != MY_TURN_FORCE_REPAIR; - if (tooSoonToRunRepair(repairType, repairState, config, myId)) - { - return; - } long startTime = timeFunc.get(); logger.info("My host id: {}, my turn to run repair...repair primary-ranges only? {}", myId, @@ -387,7 +398,8 @@ private boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoR // we should check for the node's repair history in the DB repairState.setLastRepairTime(AutoRepairUtils.getLastRepairTimeForNode(repairType, myId)); } - /** check if it is too soon to run repair. one of the reason we + /* + * check if it is too soon to run repair. one of the reason we * should not run frequent repair is that repair triggers * memtable flush */ diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 19f4fb941e52..22a1a3c78482 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -274,6 +274,26 @@ public void setParallelRepairCount(RepairType repairType, int count) getOptions(repairType).parallel_repair_count = count; } + public boolean getAllowParallelReplicaRepair(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.allow_parallel_replica_repair); + } + + public void setAllowParallelReplicaRepair(RepairType repairType, boolean enabled) + { + getOptions(repairType).allow_parallel_replica_repair = enabled; + } + + public boolean getAllowParallelReplicaRepairAcrossSchedules(RepairType repairType) + { + return applyOverrides(repairType, opt -> opt.allow_parallel_replica_repair_across_schedules); + } + + public void setAllowParallelReplicaRepairAcrossSchedules(RepairType repairType, boolean enabled) + { + getOptions(repairType).allow_parallel_replica_repair_across_schedules = enabled; + } + public boolean getMaterializedViewRepairEnabled(RepairType repairType) { return applyOverrides(repairType, opt -> opt.materialized_view_repair_enabled); @@ -439,6 +459,8 @@ protected static Options getDefaultOptions() opts.number_of_repair_threads = 1; opts.parallel_repair_count = 3; opts.parallel_repair_percentage = 3; + opts.allow_parallel_replica_repair = false; + opts.allow_parallel_replica_repair_across_schedules = true; opts.sstable_upper_threshold = 10000; opts.ignore_dcs = new HashSet<>(); opts.repair_primary_token_range_only = true; @@ -466,6 +488,15 @@ protected static Options getDefaultOptions() // Percentage of nodes in the cluster running repair in parallel. If parallel_repair_count is set, the larger value // is used. Recommendation is that the repair cycle on the cluster should finish within gc_grace_seconds. public volatile Integer parallel_repair_percentage; + // Whether to allow a node to take its turn running repair while one or more of its replicas are running repair. + // Defaults to false, as running repairs concurrently on replicas can increase load and also cause + // anticompaction conflicts while running incremental repair. + public volatile Boolean allow_parallel_replica_repair; + // An addition to allow_parallel_replica_repair that also blocks repairs when replicas (including this node itself) + // are repairing in any schedule. For example, if a replica is executing full repairs, a value of false will + // prevent starting incremental repairs for this node. Defaults to true and is only evaluated when + // allow_parallel_replica_repair is false. + public volatile Boolean allow_parallel_replica_repair_across_schedules; // Threshold to skip repairing tables with too many SSTables. Defaults to 10,000 SSTables to avoid penalizing good // tables. public volatile Integer sstable_upper_threshold; @@ -513,6 +544,8 @@ public String toString() ", number_of_repair_threads=" + number_of_repair_threads + ", parallel_repair_count=" + parallel_repair_count + ", parallel_repair_percentage=" + parallel_repair_percentage + + ", allow_parallel_replica_repair=" + allow_parallel_replica_repair + + ", allow_parallel_replica_repair_across_schedules=" + allow_parallel_replica_repair_across_schedules + ", sstable_upper_threshold=" + sstable_upper_threshold + ", min_repair_interval=" + min_repair_interval + ", ignore_dcs=" + ignore_dcs + diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index fe4a49e9b89f..5c8fcb49479c 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -21,22 +21,33 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Splitter; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.EndpointsByRange; +import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.LocalStrategy; import org.slf4j.Logger; @@ -56,6 +67,9 @@ import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.NetworkTopologyStrategy; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.metrics.AutoRepairMetricsManager; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SystemDistributedKeyspace; @@ -257,9 +271,10 @@ public static class CurrentRepairStatus public Set hostIdsWithOnGoingRepair; // hosts that is running repair public Set hostIdsWithOnGoingForceRepair; // hosts that is running repair because of force repair Set priority; + public AutoRepairHistory myRepairHistory; List historiesWithoutOnGoingRepair; // hosts that is NOT running repair - public CurrentRepairStatus(List repairHistories, Set priority) + public CurrentRepairStatus(List repairHistories, Set priority, UUID myId) { hostIdsWithOnGoingRepair = new HashSet<>(); hostIdsWithOnGoingForceRepair = new HashSet<>(); @@ -282,10 +297,19 @@ public CurrentRepairStatus(List repairHistories, Set pr { historiesWithoutOnGoingRepair.add(history); } + if (history.hostId.equals(myId)) + { + myRepairHistory = history; + } } this.priority = priority; } + public Set getAllHostsWithOngoingRepair() + { + return Sets.union(hostIdsWithOnGoingRepair, hostIdsWithOnGoingForceRepair); + } + public String toString() { return MoreObjects.toStringHelper(this). @@ -293,6 +317,7 @@ public String toString() add("hostIdsWithOnGoingForceRepair", hostIdsWithOnGoingForceRepair). add("historiesWithoutOnGoingRepair", historiesWithoutOnGoingRepair). add("priority", priority). + add("myRepairHistory", myRepairHistory). toString(); } } @@ -307,7 +332,7 @@ public static List getAutoRepairHistory(RepairType repairType repairHistoryResult = UntypedResultSet.create(repairStatusRows.result); List repairHistories = new ArrayList<>(); - if (repairHistoryResult.size() > 0) + if (!repairHistoryResult.isEmpty()) { for (UntypedResultSet.Row row : repairHistoryResult) { @@ -319,7 +344,7 @@ public static List getAutoRepairHistory(RepairType repairType long lastRepairFinishTime = row.getLong(COL_REPAIR_FINISH_TS, 0); Set deleteHosts = row.getSet(COL_DELETE_HOSTS, UUIDType.instance); long deleteHostsUpdateTime = row.getLong(COL_DELETE_HOSTS_UPDATE_TIME, 0); - Boolean forceRepair = row.has(COL_FORCE_REPAIR) ? row.getBoolean(COL_FORCE_REPAIR) : false; + boolean forceRepair = row.has(COL_FORCE_REPAIR) && row.getBoolean(COL_FORCE_REPAIR); repairHistories.add(new AutoRepairHistory(hostId, repairTurn, lastRepairStartTime, lastRepairFinishTime, deleteHosts, deleteHostsUpdateTime, forceRepair)); } @@ -369,12 +394,6 @@ public static void setForceRepair(RepairType repairType, UUID hostId) logger.info("Set force repair repair type: {}, node: {}", repairType, hostId); } - public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType) - { - List autoRepairHistories = getAutoRepairHistory(repairType); - return getCurrentRepairStatus(repairType, autoRepairHistories); - } - public static long getLastRepairTimeForNode(RepairType repairType, UUID hostId) { ResultMessage.Rows rows = selectLastRepairTimeForNode.execute(QueryState.forInternalCalls(), @@ -391,13 +410,12 @@ public static long getLastRepairTimeForNode(RepairType repairType, UUID hostId) return repairTime.one().getLong(COL_REPAIR_FINISH_TS); } - public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType, List autoRepairHistories) + @VisibleForTesting + public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType, List autoRepairHistories, UUID myId) { if (autoRepairHistories != null) { - CurrentRepairStatus status = new CurrentRepairStatus(autoRepairHistories, getPriorityHostIds(repairType)); - - return status; + return new CurrentRepairStatus(autoRepairHistories, getPriorityHostIds(repairType), myId); } return null; } @@ -414,7 +432,8 @@ protected static TreeSet getHostIdsInCurrentRing(RepairType repairType, Se logger.info("Ignore node {} because its datacenter is {}", node, nodeDC); continue; } - /** Check if endpoint state exists in gossip or not. If it + /* + * Check if endpoint state exists in gossip or not. If it * does not then this maybe a ghost node so ignore it */ if (Gossiper.instance.isAlive(node)) @@ -443,6 +462,252 @@ public static AutoRepairHistory getHostWithLongestUnrepairTime(RepairType repair return getHostWithLongestUnrepairTime(autoRepairHistories); } + /** + * Convenience method to resolve the broadcast address of a host id + * + * @return broadcast address if it exists, otherwise null. + */ + @Nullable + private static InetAddressAndPort getBroadcastAddress(UUID hostId) + { + return StorageService.instance.getEndpointForHostId(hostId); + } + + /** + * @return Map of broadcast address to host id, if a broadcast address cannot be found for a host, it is + * not included in the map. + */ + private static Map getBroadcastAddressToHostIdMap(Set hosts) + { + // Get a mapping of endpoint : host id + Map broadcastAddressMap = new HashMap<>(hosts.size()); + for (UUID hostId : hosts) + { + InetAddressAndPort broadcastAddress = getBroadcastAddress(hostId); + if (broadcastAddress == null) + { + logger.warn("Could not resolve broadcast address from host id {} in ClusterMetadata can't accurately " + + "determine if this node is a replica of the local node.", hostId); + } + else + { + broadcastAddressMap.put(broadcastAddress, hostId); + } + } + return broadcastAddressMap; + } + + /** + * @return Mapping of unique replication strategy to keyspaces using that strategy that we care about repairing. + */ + private static Map> getReplicationStrategies() + { + // Collect all unique replication strategies among all keyspaces. + Map> replicationStrategies = new HashMap<>(); + for (Keyspace keyspace : Keyspace.all()) + { + if (AutoRepairUtils.shouldConsiderKeyspace(keyspace)) + { + replicationStrategies.computeIfAbsent(keyspace.getReplicationStrategy(), k -> new ArrayList<>()) + .add(keyspace.getName()); + } + } + return replicationStrategies; + } + + /** + * Collects all hosts being repaired among all active repair schedules and their schedule if + * {@link AutoRepairConfig#getAllowParallelReplicaRepairAcrossSchedules(RepairType)} is true for this repairType. + * Accepts the currently evaluated repairType's schedule as an optimization to avoid grabbing its repair status an + * additional time. + * + * @param myRepairType The repair type schedule being evaluated. + * @param myRepairStatus The repair status for that repair type. + * @return All hosts among active schedules currently being repaired. + */ + private static Map getHostsBeingRepaired(RepairType myRepairType, CurrentRepairStatus myRepairStatus) + { + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + + Map hostsBeingRepaired = myRepairStatus.getAllHostsWithOngoingRepair().stream() + .collect(Collectors.toMap((h) -> h, (v) -> myRepairType)); + + // If we don't allow repairing across schedules, iterate over other enabled schedules and include hosts + // actively being repaired. + if (!config.getAllowParallelReplicaRepairAcrossSchedules(myRepairType)) + { + for (RepairType repairType : RepairType.values()) + { + if (myRepairType == repairType) + continue; + + if (config.isAutoRepairEnabled(repairType)) + { + CurrentRepairStatus repairStatus = getCurrentRepairStatus(repairType, getAutoRepairHistory(repairType), null); + if (repairStatus != null) + { + for (UUID hostId : repairStatus.getAllHostsWithOngoingRepair()) + { + hostsBeingRepaired.putIfAbsent(hostId, repairType); + } + } + } + } + } + return hostsBeingRepaired; + } + + /** + * Identifies the most eligible host to repair for nodes preceding or equal to this nodes' lastRepairFinishTime. + * The criteria for this is to find the node with the oldest last repair finish time of which none of its replicas + * are currently under repair. + * + * @return The most eligible host to repair or null if no candidates before and including this nodes' current repair status. + */ + @VisibleForTesting + public static AutoRepairHistory getMostEligibleHostToRepair(RepairType repairType, CurrentRepairStatus currentRepairStatus, UUID myId) + { + // 0. If this repairType allows parallel replica repair, short circuit and return the host with the longest unrepair time + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + if (config.getAllowParallelReplicaRepair(repairType)) + { + return getHostWithLongestUnrepairTime(currentRepairStatus.historiesWithoutOnGoingRepair); + } + + // 1. Sort repair histories from oldest completed to newest + Stream finishedRepairHistories = currentRepairStatus.historiesWithoutOnGoingRepair + .stream() + .sorted(Comparator.comparingLong(h -> h.lastRepairFinishTime)); + + // 2. Optimization: Truncate repair histories after myId so we don't evaluate anything more recent as if we + // aren't interested in anything that isn't this node. + final AtomicBoolean myHistoryFound = new AtomicBoolean(false); + finishedRepairHistories = finishedRepairHistories + .filter(history -> { + if (myHistoryFound.get()) return false; // Stop processing after finding myId + myHistoryFound.set(history.hostId.equals(myId)); + return true; + }); + + // If there are any hosts with ongoing repair, filter the repair histories to not include nodes whose replicas + // are ongoing repair. + Map hostsBeingRepairedToRepairType = getHostsBeingRepaired(repairType, currentRepairStatus); + + // 3. If I am already actively being repaired in another schedule, defer submitting repairs; if already + // repairing for this type, return node so it can take its turn. + RepairType alreadyRepairingType = hostsBeingRepairedToRepairType.get(myId); + if (alreadyRepairingType != null) + { + if (repairType != alreadyRepairingType) + { + logger.info("Deferring repair because I am already actively repairing in schedule {}", hostsBeingRepairedToRepairType.get(myId)); + AutoRepairMetricsManager.getMetrics(repairType).repairDelayedBySchedule.inc(); + return null; + } + else if (currentRepairStatus.myRepairHistory != null) + { + // if the repair type matches this repair, assume the node was restarted while repairing, return node + // so it can take its turn. + logAlreadyMyTurn(); + return currentRepairStatus.myRepairHistory; + } + } + + if (!hostsBeingRepairedToRepairType.isEmpty()) + { + // 4. Extract InetAddresses for each UUID as replicas are identified by their address. + Map hostsBeingRepaired = getBroadcastAddressToHostIdMap(hostsBeingRepairedToRepairType.keySet()); + + // 5. Collect unique replication strategies and group them up with their keyspaces. + Map> replicationStrategies = getReplicationStrategies(); + + // 6. Filter out repair histories who have a replica being repaired, note that this is lazy, given the stream + // is completed using findFirst, it should stop as soon as the matching criteria is met. + finishedRepairHistories = finishedRepairHistories.filter((history) -> !hasReplicaWithOngoingRepair(history, + myId, + repairType, + hostsBeingRepaired, + hostsBeingRepairedToRepairType, + replicationStrategies)); + } + + // 7. Select the first (oldest lastRepairFinishTime) repair history without replicas being repaired + return finishedRepairHistories.findFirst().orElse(null); + } + + + /** + * @param eligibleHistory History of node to check + * @param myId Host id of this node, if the repair history is for this node, additional logging will take place. + * @param myRepairType repair type being evaluated + * @param hostsBeingRepaired Hosts being repaired. + * @param hostIdToRepairType mapping of hosts being repaired to the repair type its being repaired for. + * @param replicationStrategies Mapping of unique replication strategies to keyspaces having that strategy. + * @return Whether the host for the given eligibleRepairHistory has any replicas in hostsBeingRepaired. + */ + private static boolean hasReplicaWithOngoingRepair(AutoRepairHistory eligibleHistory, + UUID myId, + RepairType myRepairType, + Map hostsBeingRepaired, + Map hostIdToRepairType, + Map> replicationStrategies) + { + // If no broadcast address found for this host id in cluster metadata, just skip it, a node should always + // see itself in cluster metadata. + InetAddressAndPort eligibleBroadcastAddress = getBroadcastAddress(eligibleHistory.hostId); + if (eligibleBroadcastAddress == null) + { + return true; + } + + // For each replication strategy, determine if host being repaired is a replica of the local node. + for (Map.Entry> entry : replicationStrategies.entrySet()) + { + AbstractReplicationStrategy replicationStrategy = entry.getKey(); + EndpointsByRange endpointsByRange = replicationStrategy.getRangeAddresses(StorageService.instance.getTokenMetadata()); + + // get ranges of the eligible address for the given replication strategy. + RangesAtEndpoint rangesAtEndpoint = StorageService.instance.getReplicas(replicationStrategy, eligibleBroadcastAddress); + for (Replica replica : rangesAtEndpoint) + { + // get the endpoints involved in this range. + EndpointsForRange endpointsForRange = endpointsByRange.get(replica.range()); + // For each host in this range... + for (InetAddressAndPort inetAddressAndPort : endpointsForRange.endpoints()) + { + // If the address of the node in the range belongs to a host being repaired, return true. + UUID hostId = hostsBeingRepaired.get(inetAddressAndPort); + if (hostId != null) + { + // log if the repair history matches the current running node. + InetAddressAndPort myBroadcastAddress = getBroadcastAddress(myId); + if (myBroadcastAddress != null && myBroadcastAddress.equals(eligibleBroadcastAddress)) + { + logger.info("Deferring repair because replica {} ({}) with shared ranges for " + + "{} keyspace(s) (e.g. {}) is currently taking its turn for schedule {}", + hostId, inetAddressAndPort, entry.getValue().size(), entry.getValue().get(0), + hostIdToRepairType.get(hostId)); + AutoRepairMetricsManager.getMetrics(myRepairType).repairDelayedByReplica.inc(); + } + else if (logger.isDebugEnabled()) + { + logger.debug("Not considering node {} ({}) for repair as it has replica {} ({}) with " + + "shared ranges for {} keyspace(s) (e.g. {}) which is currently taking its " + + "turn for schedule {}", + eligibleHistory.hostId, eligibleBroadcastAddress, + hostId, inetAddressAndPort, entry.getValue().size(), entry.getValue().get(0), + hostIdToRepairType.get(hostId)); + } + return true; + } + } + } + } + + // No replicas found of eligible host. + return false; + } + private static AutoRepairHistory getHostWithLongestUnrepairTime(List autoRepairHistories) { if (autoRepairHistories == null) @@ -476,6 +741,12 @@ public static int getMaxNumberOfNodeRunAutoRepair(RepairType repairType, int gro return Math.max(1, value); } + private static void logAlreadyMyTurn() + { + logger.warn("This node already was considered to having an ongoing repair for this repair type, must have " + + "been restarted, taking my turn back"); + } + @VisibleForTesting public static RepairTurn myTurnToRunRepair(RepairType repairType, UUID myId) { @@ -489,7 +760,7 @@ public static RepairTurn myTurnToRunRepair(RepairType repairType, UUID myId) List autoRepairHistories = getAutoRepairHistory(repairType); Set autoRepairHistoryIds = new HashSet<>(); - // 1. Remove any node that is not part of group based on goissip info + // 1. Remove any node that is not part of group based on gossip info if (autoRepairHistories != null) { for (AutoRepairHistory nodeHistory : autoRepairHistories) @@ -497,7 +768,7 @@ public static RepairTurn myTurnToRunRepair(RepairType repairType, UUID myId) autoRepairHistoryIds.add(nodeHistory.hostId); // clear delete_hosts if the node's delete hosts is not growing for more than two hours AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); - if (nodeHistory.deleteHosts.size() > 0 + if (!nodeHistory.deleteHosts.isEmpty() && config.getAutoRepairHistoryClearDeleteHostsBufferInterval().toSeconds() < TimeUnit.MILLISECONDS.toSeconds( currentTimeMillis() - nodeHistory.deleteHostsUpdateTime )) @@ -534,11 +805,14 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) } } - //get current repair status - CurrentRepairStatus currentRepairStatus = getCurrentRepairStatus(repairType, autoRepairHistories); + // get updated current repair status + CurrentRepairStatus currentRepairStatus = getCurrentRepairStatus(repairType, getAutoRepairHistory(repairType), myId); if (currentRepairStatus != null) { - logger.info("Latest repair status {}", currentRepairStatus); + if (logger.isDebugEnabled()) + { + logger.debug("Latest repair status {}", currentRepairStatus); + } //check if I am forced to run repair for (AutoRepairHistory history : currentRepairStatus.historiesWithoutOnGoingRepair) { @@ -549,8 +823,25 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) } } + // check if node was already indicated as having an ongoing repair, this may happen when a node restarts + // before finishing repairing. + if (currentRepairStatus != null && currentRepairStatus.getAllHostsWithOngoingRepair().contains(myId)) + { + logAlreadyMyTurn(); + + // use the previously chosen turn. + if (currentRepairStatus.myRepairHistory != null && currentRepairStatus.myRepairHistory.repairTurn != null) + { + return RepairTurn.valueOf(currentRepairStatus.myRepairHistory.repairTurn); + } + else + { + return MY_TURN; + } + } + int parallelRepairNumber = getMaxNumberOfNodeRunAutoRepair(repairType, - autoRepairHistories == null ? 0 : autoRepairHistories.size()); + autoRepairHistories == null ? 0 : autoRepairHistories.size()); logger.info("Will run repairs concurrently on {} node(s)", parallelRepairNumber); if (currentRepairStatus == null || parallelRepairNumber > currentRepairStatus.hostIdsWithOnGoingRepair.size()) { @@ -563,16 +854,17 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) { // try to fetch again autoRepairHistories = getAutoRepairHistory(repairType); - currentRepairStatus = getCurrentRepairStatus(repairType, autoRepairHistories); - if (autoRepairHistories == null || currentRepairStatus == null) + if (autoRepairHistories == null) { logger.error("No record found"); return NOT_MY_TURN; } + + currentRepairStatus = getCurrentRepairStatus(repairType, autoRepairHistories, myId); } UUID priorityHostId = null; - if (currentRepairStatus != null && currentRepairStatus.priority != null) + if (currentRepairStatus.priority != null) { for (UUID priorityID : currentRepairStatus.priority) { @@ -593,21 +885,37 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) if (priorityHostId != null && !myId.equals(priorityHostId)) { logger.info("Priority list is not empty and I'm not the first node in the list, not my turn." + - "First node in priority list is {}", StorageService.instance.getTokenMetadata().getEndpointForHostId(priorityHostId)); + "First node in priority list is {}", getBroadcastAddress(priorityHostId)); return NOT_MY_TURN; } + if (myId.equals(priorityHostId)) { //I have a priority for repair hence its my turn now return MY_TURN_DUE_TO_PRIORITY; } - // get the longest unrepaired node from the nodes which are not running repair - AutoRepairHistory defaultNodeToBeRepaired = getHostWithLongestUnrepairTime(currentRepairStatus.historiesWithoutOnGoingRepair); - //check who is next, which is helpful for debugging - logger.info("Next node to be repaired for repair type {} by default: {}", repairType, defaultNodeToBeRepaired); - if (defaultNodeToBeRepaired != null && defaultNodeToBeRepaired.hostId.equals(myId)) + + // Determine if this node is the most eligible host to repair. + AutoRepairHistory nodeToBeRepaired = getMostEligibleHostToRepair(repairType, currentRepairStatus, myId); + if (nodeToBeRepaired != null) { - return MY_TURN; + if (nodeToBeRepaired.hostId.equals(myId)) + { + logger.info("This node is selected to be repaired for repair type {}", repairType); + return MY_TURN; + } + + // log which node is next, which is helpful for debugging + logger.info("Next node to be repaired for repair type {}: {} ({})", repairType, + getBroadcastAddress(nodeToBeRepaired.hostId), + nodeToBeRepaired); + } + + // If this node is not identified as most eligible, set the repair lag time. + if (currentRepairStatus.myRepairHistory != null) + { + AutoRepairMetricsManager.getMetrics(repairType) + .recordRepairStartLag(currentRepairStatus.myRepairHistory.lastRepairFinishTime); } } else if (currentRepairStatus.hostIdsWithOnGoingForceRepair.contains(myId)) @@ -652,7 +960,6 @@ static void updateFinishAutoRepairHistory(RepairType repairType, UUID myId, long ByteBufferUtil.bytes(repairType.toString()), ByteBufferUtil.bytes(myId) )), Dispatcher.RequestTime.forImmediateExecution()); - // Do not remove beblow log, the log is used by dtest logger.info("Auto repair finished for {}", myId); } @@ -752,7 +1059,7 @@ public static Set getPriorityHostIds(RepairType repairType) repairPriorityResult = UntypedResultSet.create(repairPriorityRows.result); Set priorities = null; - if (repairPriorityResult.size() > 0) + if (!repairPriorityResult.isEmpty()) { // there should be only one row UntypedResultSet.Row row = repairPriorityResult.one(); @@ -770,7 +1077,13 @@ public static Set getPriorityHosts(RepairType repairType) Set hosts = new HashSet<>(); for (UUID hostId : getPriorityHostIds(repairType)) { - hosts.add(StorageService.instance.getTokenMetadata().getEndpointForHostId(hostId)); + InetAddressAndPort broadcastAddress = getBroadcastAddress(hostId); + if (broadcastAddress == null) + { + logger.warn("Could not resolve broadcastAddress for {}, skipping considering it as a priority host", hostId); + continue; + } + hosts.add(broadcastAddress); } return hosts; } diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 9f66a92abc95..4d34a53268f1 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -202,6 +202,18 @@ public void setParallelRepairCount(String repairType, int count) config.setParallelRepairCount(RepairType.parse(repairType), count); } + @Override + public void setAllowParallelReplicaRepair(String repairType, boolean enabled) + { + config.setAllowParallelReplicaRepair(RepairType.parse(repairType), enabled); + } + + @Override + public void setAllowParallelReplicaRepairAcrossSchedules(String repairType, boolean enabled) + { + config.setAllowParallelReplicaRepairAcrossSchedules(RepairType.parse(repairType), enabled); + } + @Override public void setMVRepairEnabled(String repairType, boolean enabled) { @@ -223,7 +235,7 @@ public Set getOnGoingRepairHostIds(String repairType) return Collections.emptySet(); } Set hostIds = new HashSet<>(); - AutoRepairUtils.CurrentRepairStatus currentRepairStatus = new AutoRepairUtils.CurrentRepairStatus(histories, AutoRepairUtils.getPriorityHostIds(RepairType.parse(repairType))); + AutoRepairUtils.CurrentRepairStatus currentRepairStatus = new AutoRepairUtils.CurrentRepairStatus(histories, AutoRepairUtils.getPriorityHostIds(RepairType.parse(repairType)), null); for (UUID id : currentRepairStatus.hostIdsWithOnGoingRepair) { hostIds.add(id.toString()); @@ -273,21 +285,23 @@ private String formatRepairTypeConfig(RepairType repairType, AutoRepairConfig co appendConfig(sb, "priority_hosts", Joiner.on(',').skipNulls().join(priorityHosts)); } - appendConfig(sb , "min_repair_interval", config.getRepairMinInterval(repairType)); - appendConfig(sb , "repair_by_keyspace", config.getRepairByKeyspace(repairType)); - appendConfig(sb , "number_of_repair_threads", config.getRepairThreads(repairType)); - appendConfig(sb , "sstable_upper_threshold", config.getRepairSSTableCountHigherThreshold(repairType)); - appendConfig(sb , "table_max_repair_time", config.getAutoRepairTableMaxRepairTime(repairType)); - appendConfig(sb , "ignore_dcs", config.getIgnoreDCs(repairType)); - appendConfig(sb , "repair_primary_token_range_only", config.getRepairPrimaryTokenRangeOnly(repairType)); - appendConfig(sb , "parallel_repair_count", config.getParallelRepairCount(repairType)); - appendConfig(sb , "parallel_repair_percentage", config.getParallelRepairPercentage(repairType)); - appendConfig(sb , "materialized_view_repair_enabled", config.getMaterializedViewRepairEnabled(repairType)); - appendConfig(sb , "initial_scheduler_delay", config.getInitialSchedulerDelay(repairType)); - appendConfig(sb , "repair_session_timeout", config.getRepairSessionTimeout(repairType)); - appendConfig(sb , "force_repair_new_node", config.getForceRepairNewNode(repairType)); - appendConfig(sb , "repair_max_retries", config.getRepairMaxRetries(repairType)); - appendConfig(sb , "repair_retry_backoff", config.getRepairRetryBackoff(repairType)); + appendConfig(sb, "min_repair_interval", config.getRepairMinInterval(repairType)); + appendConfig(sb, "repair_by_keyspace", config.getRepairByKeyspace(repairType)); + appendConfig(sb, "number_of_repair_threads", config.getRepairThreads(repairType)); + appendConfig(sb, "sstable_upper_threshold", config.getRepairSSTableCountHigherThreshold(repairType)); + appendConfig(sb, "table_max_repair_time", config.getAutoRepairTableMaxRepairTime(repairType)); + appendConfig(sb, "ignore_dcs", config.getIgnoreDCs(repairType)); + appendConfig(sb, "repair_primary_token_range_only", config.getRepairPrimaryTokenRangeOnly(repairType)); + appendConfig(sb, "parallel_repair_count", config.getParallelRepairCount(repairType)); + appendConfig(sb, "parallel_repair_percentage", config.getParallelRepairPercentage(repairType)); + appendConfig(sb, "allow_parallel_replica_repair", config.getAllowParallelReplicaRepair(repairType)); + appendConfig(sb, "allow_parallel_replica_repair_across_schedules", config.getAllowParallelReplicaRepairAcrossSchedules(repairType)); + appendConfig(sb, "materialized_view_repair_enabled", config.getMaterializedViewRepairEnabled(repairType)); + appendConfig(sb, "initial_scheduler_delay", config.getInitialSchedulerDelay(repairType)); + appendConfig(sb, "repair_session_timeout", config.getRepairSessionTimeout(repairType)); + appendConfig(sb, "force_repair_new_node", config.getForceRepairNewNode(repairType)); + appendConfig(sb, "repair_max_retries", config.getRepairMaxRetries(repairType)); + appendConfig(sb, "repair_retry_backoff", config.getRepairRetryBackoff(repairType)); final ParameterizedClass splitterClass = config.getTokenRangeSplitter(repairType); final String splitterClassName = splitterClass.class_name != null ? splitterClass.class_name : AutoRepairConfig.DEFAULT_SPLITTER.getName(); diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index 72d9da7c5a0c..181c6008f533 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -53,6 +53,10 @@ public interface AutoRepairServiceMBean public void setParallelRepairCount(String repairType, int count); + public void setAllowParallelReplicaRepair(String repairType, boolean enabled); + + public void setAllowParallelReplicaRepairAcrossSchedules(String repairType, boolean enabled); + public void setMVRepairEnabled(String repairType, boolean enabled); public boolean isAutoRepairDisabled(); diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index a684a9adac01..32861afbfb2f 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -333,8 +333,17 @@ public void setIsShutdownUnsafeForTests(boolean isShutdown) public RangesAtEndpoint getLocalReplicas(String keyspaceName) { - return Keyspace.open(keyspaceName).getReplicationStrategy() - .getAddressReplicas(FBUtilities.getBroadcastAddressAndPort()); + return getReplicas(keyspaceName, FBUtilities.getBroadcastAddressAndPort()); + } + + public RangesAtEndpoint getReplicas(String keyspaceName, InetAddressAndPort endpoint) + { + return getReplicas(Keyspace.open(keyspaceName).getReplicationStrategy(), endpoint); + } + + public RangesAtEndpoint getReplicas(AbstractReplicationStrategy replicationStrategy, InetAddressAndPort endpoint) + { + return replicationStrategy.getAddressReplicas(tokenMetadata.cloneOnlyTokenMap(), endpoint); } public List> getLocalRanges(String ks) @@ -1347,7 +1356,7 @@ public void doAuthSetup(boolean setUpSchema) } } - private void doAutoRepairSetup() + public void doAutoRepairSetup() { AutoRepairService.setup(); if (DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled()) diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 816c658144af..300bbc6c4efc 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2223,6 +2223,16 @@ public void setAutoRepairParallelRepairCount(String repairType, int count) autoRepairProxy.setParallelRepairCount(repairType, count); } + public void setAutoRepairAllowParallelReplicaRepair(String repairType, boolean enabled) + { + autoRepairProxy.setAllowParallelReplicaRepair(repairType, enabled); + } + + public void setAutoRepairAllowParallelReplicaRepairAcrossSchedules(String repairType, boolean enabled) + { + autoRepairProxy.setAllowParallelReplicaRepairAcrossSchedules(repairType, enabled); + } + public void setAutoRepairPrimaryTokenRangeOnly(String repairType, boolean primaryTokenRangeOnly) { autoRepairProxy.setPrimaryTokenRangeOnly(repairType, primaryTokenRangeOnly); diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index 5a9a1411bb59..cc00cabd0633 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -47,7 +47,9 @@ public class SetAutoRepairConfig extends NodeToolCmd "[start_scheduler|number_of_repair_threads|min_repair_interval|sstable_upper_threshold" + "|enabled|table_max_repair_time|priority_hosts|forcerepair_hosts|ignore_dcs" + "|history_clear_delete_hosts_buffer_interval|repair_primary_token_range_only" + - "|parallel_repair_count|parallel_repair_percentage|materialized_view_repair_enabled|repair_max_retries" + + "|parallel_repair_count|parallel_repair_percentage" + + "|allow_parallel_replica_repair|allow_parallel_repair_across_schedules" + + "|materialized_view_repair_enabled|repair_max_retries" + "|repair_retry_backoff|repair_session_timeout|min_repair_task_duration" + "|repair_by_keyspace|token_range_splitter.]", required = true) @@ -148,6 +150,12 @@ public void execute(NodeProbe probe) case "parallel_repair_percentage": probe.setAutoRepairParallelRepairPercentage(repairTypeStr, Integer.parseInt(paramVal)); break; + case "allow_parallel_replica_repair": + probe.setAutoRepairAllowParallelReplicaRepair(repairTypeStr, Boolean.parseBoolean(paramVal)); + break; + case "allow_parallel_replica_repair_across_schedules": + probe.setAutoRepairAllowParallelReplicaRepairAcrossSchedules(repairTypeStr, Boolean.parseBoolean(paramVal)); + break; case "materialized_view_repair_enabled": probe.setAutoRepairMaterializedViewRepairEnabled(repairTypeStr, Boolean.parseBoolean(paramVal)); break; diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java new file mode 100644 index 000000000000..0a2238ac4024 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.repair; + +import java.io.IOException; + +import com.google.common.collect.ImmutableMap; + +import org.apache.cassandra.metrics.AutoRepairMetrics; +import org.apache.cassandra.metrics.AutoRepairMetricsManager; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.repair.autorepair.AutoRepair; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.service.AutoRepairService; + +import static org.junit.Assert.assertEquals; + +/** + * Distributed tests for {@link org.apache.cassandra.repair.autorepair.AutoRepair} scheduler's + * allow_parallel_replica_repair_across_schedules feature. + */ +public class AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest extends TestBaseImpl +{ + private static Cluster cluster; + + @BeforeClass + public static void init() throws IOException + { + // Configure a cluster with preview and incremental repair enabled in a way that preview repair can be + // run on all three nodes concurrently, but incremental repair can only be run when there are no parallel + // repairs. We should detect contention in the incremental repair scheduler but not preview repaired + // scheduler as a result. + cluster = Cluster.build(3) + .withConfig(config -> config + .set("auto_repair", + ImmutableMap.of( + "repair_type_overrides", + ImmutableMap.>builder() + .put(AutoRepairConfig.RepairType.PREVIEW_REPAIRED.getConfigName(), + ImmutableMap.builder() + // Configure preview repair to run frequently to provoke contention + .put("initial_scheduler_delay", "5s") + .put("enabled", "true") + .put("parallel_repair_count", "3") + .put("allow_parallel_replica_repair", "true") + .put("min_repair_interval", "5s") + .build() + ) + .put(AutoRepairConfig.RepairType.INCREMENTAL.getConfigName(), + ImmutableMap.builder() + .put("initial_scheduler_delay", "5s") + .put("enabled", "true") + .put("parallel_repair_count", "3") + // Don't allow parallel replica repair across schedules + .put("allow_parallel_replica_repair", "false") + .put("allow_parallel_replica_repair_across_schedules", "false") + .put("min_repair_interval", "5s") + .build() + ) + .build() + ) + ) + .set("auto_repair.enabled", "true") + .set("auto_repair.global_settings.repair_retry_backoff", "5s") + .set("auto_repair.repair_task_min_duration", "0s") + .set("auto_repair.repair_check_interval", "5s")) + .start(); + + cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck text, v1 int, v2 int, PRIMARY KEY (pk, ck)) WITH read_repair='NONE'")); + } + + @AfterClass + public static void tearDown() + { + cluster.close(); + } + + @Test + public void testScheduler() + { + cluster.forEach(i -> i.runOnInstance(() -> { + try + { + AutoRepairService.setup(); + AutoRepair.instance.setup(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + })); + + // validate that the repair ran on all nodes + cluster.forEach(i -> i.runOnInstance(() -> { + // Expect contention on incremental repair across schedules + AutoRepairMetrics incrementalMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.INCREMENTAL); + + while (incrementalMetrics.repairDelayedByReplica.getCount() <= 0) + { + try + { + Thread.sleep(1000); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + + // No repair contention should be observed for preview repaired since allow_parallel_replica_repair was true + AutoRepairMetrics previewMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.PREVIEW_REPAIRED); + assertEquals(0L, previewMetrics.repairDelayedByReplica.getCount()); + assertEquals(0L, previewMetrics.repairDelayedBySchedule.getCount()); + })); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index 89ef40640317..49892741c9a2 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -25,13 +25,20 @@ import com.google.common.collect.ImmutableMap; -import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.distributed.api.TokenSupplier; +import org.apache.cassandra.metrics.AutoRepairMetrics; +import org.apache.cassandra.metrics.AutoRepairMetricsManager; import org.apache.cassandra.schema.SystemDistributedKeyspace; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.test.TestBaseImpl; @@ -43,10 +50,11 @@ import static org.junit.Assert.assertEquals; /** - * Unit tests for {@link org.apache.cassandra.repair.autorepair.AutoRepair} + * Distributed tests for {@link org.apache.cassandra.repair.autorepair.AutoRepair} scheduler */ public class AutoRepairSchedulerTest extends TestBaseImpl { + static final Logger logger = LoggerFactory.getLogger(AutoRepairSchedulerTest.class); private static Cluster cluster; static SimpleDateFormat sdf; @@ -58,33 +66,57 @@ public static void init() throws IOException // Create SimpleDateFormat object with the given pattern sdf = new SimpleDateFormat(pattern); sdf.setLenient(false); - cluster = Cluster.build(3).withConfig(config -> config - .set("auto_repair", - ImmutableMap.of( - "repair_type_overrides", - ImmutableMap.of(AutoRepairConfig.RepairType.FULL.getConfigName(), - ImmutableMap.of( - "initial_scheduler_delay", "5s", - "enabled", "true", - "parallel_repair_count", "1", - "parallel_repair_percentage", "0", - "min_repair_interval", "1s"), - AutoRepairConfig.RepairType.INCREMENTAL.getConfigName(), - ImmutableMap.of( - "initial_scheduler_delay", "5s", - "enabled", "true", - "parallel_repair_count", "1", - "parallel_repair_percentage", "0", - "min_repair_interval", "1s")))) - .set("auto_repair.enabled", "true") - .set("auto_repair.global_settings.repair_by_keyspace", "true") - .set("auto_repair.repair_task_min_duration", "0s") - .set("auto_repair.repair_check_interval", "10s")).start(); + // Configure a 3-node cluster with num_tokens: 4 and auto_repair enabled + cluster = Cluster.build(3) + .withTokenCount(4) + .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(3, 4)) + .withConfig(config -> config + .set("num_tokens", 4) + .set("auto_repair", + ImmutableMap.of( + "repair_type_overrides", + ImmutableMap.of(AutoRepairConfig.RepairType.FULL.getConfigName(), + ImmutableMap.of( + "initial_scheduler_delay", "5s", + "enabled", "true", + "parallel_repair_count", "2", + // Allow parallel replica repair to allow replicas + // to execute full repair at same time. + "allow_parallel_replica_repair", "true", + "min_repair_interval", "15s"), + AutoRepairConfig.RepairType.INCREMENTAL.getConfigName(), + ImmutableMap.of( + "initial_scheduler_delay", "5s", + "enabled", "true", + // Set parallel repair count to 3 to provoke + // contention between replicas when scheduling. + "parallel_repair_count", "3", + // Disallow parallel replica repair to prevent + // replicas from issuing incremental repair at + // same time. + "allow_parallel_replica_repair", "false", + // Run more aggressively since full repair is + // less restrictive about when it can run repair, + // so need to check more frequently to allow + // incremental to get an attempt in. + "min_repair_interval", "5s")))) + .set("auto_repair.enabled", "true") + .set("auto_repair.global_settings.repair_by_keyspace", "true") + .set("auto_repair.global_settings.repair_retry_backoff", "5s") + .set("auto_repair.repair_task_min_duration", "0s") + .set("auto_repair.repair_check_interval", "5s")) + .start(); cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck text, v1 int, v2 int, PRIMARY KEY (pk, ck)) WITH read_repair='NONE'")); } + @AfterClass + public static void tearDown() + { + cluster.close(); + } + @Test public void testScheduler() throws ParseException { @@ -95,9 +127,7 @@ public void testScheduler() throws ParseException cluster.forEach(i -> i.runOnInstance(() -> { try { - DatabaseDescriptor.setCDCOnRepairEnabled(false); - DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(false); - AutoRepairService.instance.setup(); + AutoRepairService.setup(); AutoRepair.instance.setup(); } catch (Exception e) @@ -106,33 +136,64 @@ public void testScheduler() throws ParseException } })); + logger.info("Repair setup done"); // validate that the repair ran on all nodes cluster.forEach(i -> i.runOnInstance(() -> { - try + // Reduce sleeping if repair finishes quickly to speed up test but make it non-zero to provoke some + // contention. + AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("1s"); + + AutoRepairMetrics incrementalMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.INCREMENTAL); + while (incrementalMetrics.nodeRepairTimeInSec.getValue().longValue() <= 0) { - while(true) + try { - if (AutoRepair.instance.repairStates.get(AutoRepairConfig.RepairType.FULL).getLastRepairTime() == 0 - || AutoRepair.instance.repairStates.get(AutoRepairConfig.RepairType.INCREMENTAL).getLastRepairTime() == 0) - { - try - { - Thread.sleep(5000); - } - catch (InterruptedException e) - { - throw new RuntimeException(e); - } - continue; - } - break; + Thread.sleep(1000); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); } } - catch (Exception e) + logger.info("AutoRepair has completed one INCREMENTAL repair cycle"); + + // Expect some contention on incremental repair. + while (incrementalMetrics.repairDelayedByReplica.getCount() <= 0) { - throw new RuntimeException(e); + try + { + Thread.sleep(1000); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + logger.info("AutoRepair has observed replica contention in INCREMENTAL repair"); + + // Do not expect any contention across schedules since allow_parallel_replica_repairs across schedules + // was not configured. + assertEquals(0L, incrementalMetrics.repairDelayedBySchedule.getCount()); + + AutoRepairMetrics fullMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.FULL); + while (fullMetrics.nodeRepairTimeInSec.getValue().longValue() <= 0) + { + try + { + Thread.sleep(1000); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } } + logger.info("AutoRepair has completed one FULL repair cycle"); + + // No repair contention should be observed for full repair since allow_parallel_replica_repair was true + assertEquals(0L, fullMetrics.repairDelayedByReplica.getCount()); + assertEquals(0L, fullMetrics.repairDelayedBySchedule.getCount()); })); + validate(AutoRepairConfig.RepairType.FULL.toString()); validate(AutoRepairConfig.RepairType.INCREMENTAL.toString()); } @@ -147,7 +208,7 @@ private void validate(String repairType) throws ParseException // repair_type Assert.assertEquals(repairType, row[0].toString()); // host_id - UUID.fromString(row[1].toString()); + Assert.assertNotNull(UUID.fromString(row[1].toString())); // ensure there is a legit repair_start_ts and repair_finish_ts sdf.parse(row[2].toString()); sdf.parse(row[3].toString()); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 2c3e4401ed9f..0cf51ca40884 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -20,7 +20,6 @@ import java.util.HashSet; import java.util.Map; -import java.util.Objects; import java.util.Collections; import java.util.Set; @@ -155,7 +154,7 @@ public void testSetRepairThreads() { config.setRepairThreads(repairType, 5); - assert config.getOptions(repairType).number_of_repair_threads == 5; + assertEquals(5, config.getOptions(repairType).number_of_repair_threads.intValue()); } @Test @@ -183,7 +182,7 @@ public void testSetRepairMinFrequencyInHours() { config.setRepairMinInterval(repairType, "5s"); - assert config.getOptions(repairType).min_repair_interval.toSeconds() == 5; + assertEquals(5, config.getOptions(repairType).min_repair_interval.toSeconds()); } @Test @@ -201,7 +200,7 @@ public void testSetAutoRepairHistoryClearDeleteHostsBufferInSec() { config.setAutoRepairHistoryClearDeleteHostsBufferInterval("5s"); - assert Objects.equals(config.history_clear_delete_hosts_buffer_interval, new DurationSpec.IntSecondsBound("5s")); + assertEquals(new DurationSpec.IntSecondsBound("5s"), config.history_clear_delete_hosts_buffer_interval); } @Test @@ -219,7 +218,7 @@ public void testSetRepairSSTableCountHigherThreshold() { config.setRepairSSTableCountHigherThreshold(repairType, 5); - assert config.getOptions(repairType).sstable_upper_threshold == 5; + assertEquals(5, config.getOptions(repairType).sstable_upper_threshold.intValue()); } @Test @@ -237,8 +236,8 @@ public void testSetAutoRepairTableMaxRepairTimeInSec() { config.setAutoRepairTableMaxRepairTime(repairType, "5s"); - assert config.getOptions(repairType).table_max_repair_time.toSeconds() == 5; - } + assertEquals(5, config.getOptions(repairType).table_max_repair_time.toSeconds()); +} @Test public void testGetIgnoreDCs() @@ -291,7 +290,7 @@ public void testSetParallelRepairPercentageInGroup() { config.setParallelRepairPercentage(repairType, 5); - assert config.getOptions(repairType).parallel_repair_percentage == 5; + assertEquals(5, config.getOptions(repairType).parallel_repair_percentage.intValue()); } @Test @@ -309,7 +308,55 @@ public void testSetParallelRepairCountInGroup() { config.setParallelRepairCount(repairType, 5); - assert config.getOptions(repairType).parallel_repair_count == 5; + assertEquals(5, config.getOptions(repairType).parallel_repair_count.intValue()); + } + + @Test + public void testGetAllowParallelReplicaRepair() + { + // should default to false + assertFalse(config.global_settings.allow_parallel_replica_repair); + assertFalse(config.getAllowParallelReplicaRepair(repairType)); + + // setting global to true should also cause repair type config to inherit. + config.global_settings.allow_parallel_replica_repair = true; + assertTrue(config.getAllowParallelReplicaRepair(repairType)); + + } + + @Test + public void testSetAllowParallelReplicaRepair() + { + // should default to false + assertFalse(config.getAllowParallelReplicaRepair(repairType)); + + // setting explicitly for repair type should update it + config.setAllowParallelReplicaRepair(repairType, true); + assertTrue(config.getAllowParallelReplicaRepair(repairType)); + } + + @Test + public void testGetAllowParallelReplicaRepairAcrossSchedules() + { + // should default to true + assertTrue(config.global_settings.allow_parallel_replica_repair_across_schedules); + assertTrue(config.getAllowParallelReplicaRepairAcrossSchedules(repairType)); + + // setting global to true should also cause repair type config to inherit. + config.global_settings.allow_parallel_replica_repair_across_schedules = false; + assertFalse(config.getAllowParallelReplicaRepairAcrossSchedules(repairType)); + + } + + @Test + public void testSetAllowParallelReplicaRepairAcrossSchedules() + { + // should default to true + assertTrue(config.getAllowParallelReplicaRepairAcrossSchedules(repairType)); + + // setting explicitly for repair type should update it + config.setAllowParallelReplicaRepairAcrossSchedules(repairType, false); + assertFalse(config.getAllowParallelReplicaRepairAcrossSchedules(repairType)); } @Test @@ -396,7 +443,7 @@ public void testSetInitialSchedulerDelay() { config.setInitialSchedulerDelay(repairType, "5s"); - assert config.getOptions(repairType).initial_scheduler_delay.toSeconds() == 5; + assertEquals(5, config.getOptions(repairType).initial_scheduler_delay.toSeconds()); } @Test @@ -414,7 +461,7 @@ public void testSetRepairSessionTimeout() { config.setRepairSessionTimeout(repairType, "1h"); - assert config.getOptions(repairType).repair_session_timeout.toSeconds() == 3600; + assertEquals(3600, config.getOptions(repairType).repair_session_timeout.toSeconds()); } @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairMetricsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairMetricsTest.java new file mode 100644 index 000000000000..d0b053a58778 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairMetricsTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.metrics.AutoRepairMetrics; +import org.apache.cassandra.metrics.AutoRepairMetricsManager; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.service.StorageService; + +import static org.apache.cassandra.Util.setAutoRepairEnabled; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class AutoRepairMetricsTest extends CQLTester +{ + + private AutoRepairMetrics metrics; + + @BeforeClass + public static void setupClass() throws Exception + { + setAutoRepairEnabled(true); + requireNetwork(); + AutoRepairUtils.setup(); + StorageService.instance.doAutoRepairSetup(); + + // Set min repair interval to an hour. + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setRepairMinInterval(RepairType.FULL, "1h"); + } + + @Before + public void setup() + { + metrics = AutoRepairMetricsManager.getMetrics(RepairType.FULL); + } + + @Test + public void testShouldRecordRepairStartLagAndResetOnMyTurn() + { + // record a last finish repair time of one day. + long oneDayAgo = AutoRepair.instance.currentTimeMs() - 86_400_000; + metrics.recordRepairStartLag(oneDayAgo); + + // expect a recorded lag time of approximately 1 day (last repair finish time) - 1 hour (min repair interval) + long expectedLag = 86400 - 3600; + long recordedLag = metrics.repairStartLagSec.getValue(); + assertTrue(String.format("Expected at last 23h of lag (%d) but got (%d)", expectedLag, recordedLag), + recordedLag >= expectedLag); + // Given timing, allow at most 5 seconds of skew. + assertTrue(String.format("Expected 23h of lag (%d) but got a larger value (%d)", expectedLag, recordedLag), + recordedLag <= expectedLag + 5); + + // expect lag time to be restarted when recording a turn. + metrics.recordTurn(RepairTurn.MY_TURN); + assertEquals(0, metrics.repairStartLagSec.getValue().intValue()); + } + + @Test + public void testShouldRecordRepairStartLagOfZeroWhenFinishTimeIsWithinMinRepairInterval() + { + // record a last finish repair time of one 30 minutes + long thirtyMinutesAgo = AutoRepair.instance.currentTimeMs() - 1_800_000; + metrics.recordRepairStartLag(thirtyMinutesAgo); + + // expect 0 lag because last repair finish time was less than min repair interval + assertEquals(0, metrics.repairStartLagSec.getValue().intValue()); + } +} diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 2b8c1d51e3be..93857dc4b4cd 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -306,6 +306,8 @@ public void testRepair() //if repair was done then lastRepairTime should be non-zero Assert.assertTrue(String.format("Expected lastRepairTime > 0, actual value lastRepairTime %d", lastRepairTime), lastRepairTime > 0); + // repair start lag sec should be reset on a successful repair + assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).repairStartLagSec.getValue().intValue()); } @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index 0ff78b08479d..ee841a62c3b1 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -212,7 +212,7 @@ public void testGetCurrentRepairStatus() SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, repairType.toString(), regularRepair)); - CurrentRepairStatus status = AutoRepairUtils.getCurrentRepairStatus(repairType); + CurrentRepairStatus status = AutoRepairUtils.getCurrentRepairStatus(repairType, AutoRepairUtils.getAutoRepairHistory(repairType), hostId); assertNotNull(status); assertEquals(1, status.historiesWithoutOnGoingRepair.size()); @@ -223,6 +223,7 @@ public void testGetCurrentRepairStatus() assertTrue(status.hostIdsWithOnGoingForceRepair.contains(forceRepair)); assertEquals(1, status.priority.size()); assertTrue(status.priority.contains(regularRepair)); + assertEquals(hostId, status.myRepairHistory.hostId); } @Test From 642cde489c9194bcd478cb3566c08006ccf3ef8a Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 30 Mar 2025 13:25:02 -0700 Subject: [PATCH 179/199] Add toString to AutoRepairConfig.java --- .../repair/autorepair/AutoRepairConfig.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 22a1a3c78482..09a96d359b4b 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -593,4 +593,16 @@ protected T applyOverrides(RepairType repairType, Function optio // Otherwise check defaults return getOverride(Options.getDefaultOptionsMap().get(repairType), optionSupplier); } + + public String toString() + { + return "AutoRepairConfig{" + + "enabled=" + enabled + + ", repair_check_interval=" + repair_check_interval + + ", history_clear_delete_hosts_buffer_interval=" + history_clear_delete_hosts_buffer_interval + + ", repair_task_min_duration=" + repair_task_min_duration + + ", global_settings=" + global_settings + + ", repair_type_overrides=" + repair_type_overrides + + "}"; + } } From a4eabcf61839f421745d1671e35b2c971bc69117 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 6 Apr 2025 14:15:36 -0700 Subject: [PATCH 180/199] Create AutoRepair executors only if enabled and a few other nits --- conf/cassandra.yaml | 2 +- .../managing/operating/auto_repair.adoc | 2 +- .../repair/autorepair/AutoRepair.java | 45 ++++++++----------- .../repair/autorepair/AutoRepairConfig.java | 12 +---- .../cassandra/service/AutoRepairService.java | 2 +- .../cassandra/service/StorageService.java | 2 +- .../AutoRepairParameterizedTest.java | 22 +-------- .../repair/autorepair/AutoRepairTest.java | 39 +++++++--------- 8 files changed, 42 insertions(+), 84 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index b28e225bb061..470d05b0def8 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1992,7 +1992,7 @@ drop_compact_storage_enabled: false # materialized_view_repair_enabled: false # # Delay before starting repairs after a node restarts to avoid repairs starting immediately after a restart. # initial_scheduler_delay: 5m -# # Timeout for resuming stuck repair sessions. +# # Timeout for retrying stuck repair sessions. # repair_session_timeout: 3h # # Force immediate repair on new nodes after they join the ring. # force_repair_new_node: false diff --git a/doc/modules/cassandra/managing/operating/auto_repair.adoc b/doc/modules/cassandra/managing/operating/auto_repair.adoc index 06c0db456753..3928d0afccef 100644 --- a/doc/modules/cassandra/managing/operating/auto_repair.adoc +++ b/doc/modules/cassandra/managing/operating/auto_repair.adoc @@ -205,7 +205,7 @@ node. Defaults to true and is only evaluated when allow_parallel_replica_repair | materialized_view_repair_enabled | false | Repairs materialized views if true. | initial_scheduler_delay | 5m | Delay before starting repairs after a node restarts to avoid repairs starting immediately after a restart. -| repair_session_timeout | 3h | Timeout for resuming stuck repair sessions. +| repair_session_timeout | 3h | Timeout for retrying stuck repair sessions. | force_repair_new_node | false | Force immediate repair on new nodes after they join the ring. | sstable_upper_threshold | 10000 | Threshold to skip repairing tables with too many SSTables. | table_max_repair_time | 6h | Maximum time allowed for repairing one table on a given node. If exceeded, the repair diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index f92f7d984d27..93e371e3b5d9 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -38,7 +38,6 @@ import com.google.common.util.concurrent.Uninterruptibles; import org.apache.cassandra.config.DurationSpec; -import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.utils.Clock; @@ -84,12 +83,12 @@ public class AutoRepair public static DurationSpec.IntSecondsBound SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("5s"); @VisibleForTesting - public final Map repairStates; + public Map repairStates; @VisibleForTesting - protected final Map repairExecutors; + protected Map repairExecutors; - protected final Map repairRunnableExecutors; + protected Map repairRunnableExecutors; @VisibleForTesting // Auto-repair is likely to be run on multiple nodes independently, we want to avoid running multiple repair @@ -99,21 +98,13 @@ public class AutoRepair @VisibleForTesting protected static BiConsumer sleepFunc = Uninterruptibles::sleepUninterruptibly; - private boolean isSetupDone = false; + @VisibleForTesting + public boolean isSetupDone = false; public static AutoRepair instance = new AutoRepair(); - @VisibleForTesting - protected AutoRepair() + private AutoRepair() { - repairExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class); - repairRunnableExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class); - repairStates = new EnumMap<>(AutoRepairConfig.RepairType.class); - for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) - { - repairExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-Repair-" + repairType.getConfigName(), Thread.NORM_PRIORITY)); - repairRunnableExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-RepairRunnable-" + repairType.getConfigName(), Thread.NORM_PRIORITY)); - repairStates.put(repairType, AutoRepairConfig.RepairType.getAutoRepairState(repairType)); - } + // Private constructor to prevent instantiation } public void setup() @@ -126,6 +117,16 @@ public void setup() { return; } + repairExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class); + repairRunnableExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class); + repairStates = new EnumMap<>(AutoRepairConfig.RepairType.class); + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + repairExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-Repair-" + repairType.getConfigName(), Thread.NORM_PRIORITY)); + repairRunnableExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-RepairRunnable-" + repairType.getConfigName(), Thread.NORM_PRIORITY)); + repairStates.put(repairType, AutoRepairConfig.RepairType.getAutoRepairState(repairType)); + } + AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); AutoRepairUtils.setup(); @@ -144,16 +145,6 @@ public void setup() } } - // repairAsync runs a repair session of the given type asynchronously. - public void repairAsync(AutoRepairConfig.RepairType repairType) - { - if (!AutoRepairService.instance.getAutoRepairConfig().isAutoRepairEnabled(repairType)) - { - throw new ConfigurationException("Auto-repair is disabled for repair type " + repairType); - } - repairExecutors.get(repairType).submit(() -> repair(repairType)); - } - /** * @return The current observed system time in ms. */ @@ -554,7 +545,7 @@ public boolean isSuccess() public void progress(String tag, ProgressEvent event) { ProgressEventType type = event.getType(); - String message = String.format("[%s] %s", format.format(Clock.Global.currentTimeMillis()), event.getMessage()); + String message = String.format("[%s] %s", format.format(timeFunc.get()), event.getMessage()); if (type == ProgressEventType.ERROR) { logger.error("Repair failure for repair {}: {}", repairType.toString(), message); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 09a96d359b4b..1fa1ac4f8c4b 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -319,16 +319,6 @@ public ParameterizedClass getTokenRangeSplitter(RepairType repairType) return applyOverrides(repairType, opt -> opt.token_range_splitter); } - /** - * Set a new token range splitter, this is not meant to be used other than for testing. - */ - @VisibleForTesting - void setTokenRangeSplitter(RepairType repairType, ParameterizedClass tokenRangeSplitter) - { - getOptions(repairType).token_range_splitter = tokenRangeSplitter; - tokenRangeSplitters.remove(repairType); - } - public IAutoRepairTokenRangeSplitter getTokenRangeSplitterInstance(RepairType repairType) { return tokenRangeSplitters.computeIfAbsent(repairType, @@ -529,7 +519,7 @@ protected static Options getDefaultOptions() public volatile ParameterizedClass token_range_splitter; // After a node restart, wait for this much delay before scheduler starts running repair; this is to avoid starting repair immediately after a node restart. public volatile DurationSpec.IntSecondsBound initial_scheduler_delay; - // Timeout for resuming stuck repair sessions. + // Timeout for retrying stuck repair sessions. public volatile DurationSpec.IntSecondsBound repair_session_timeout; // Maximum number of retries for a repair session. public volatile Integer repair_max_retries = 3; diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index 4d34a53268f1..db1b29c38968 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -71,7 +71,7 @@ public void checkCanRun(String repairType) public void checkCanRun(RepairType repairType) { if (!config.isAutoRepairSchedulingEnabled()) - throw new ConfigurationException("Auto-repair scheduller is disabled."); + throw new ConfigurationException("Auto-repair scheduler is disabled."); if (repairType != RepairType.INCREMENTAL) return; diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 32861afbfb2f..c689875c79ef 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -1363,8 +1363,8 @@ public void doAutoRepairSetup() { logger.info("Enabling auto-repair scheduling"); AutoRepair.instance.setup(); + logger.info("AutoRepair setup complete!"); } - logger.info("AutoRepair setup complete!"); } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index 93857dc4b4cd..bd56b1e60ce8 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -214,8 +214,8 @@ public void setup() Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY).truncateBlocking(); Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(SystemDistributedKeyspace.AUTO_REPAIR_HISTORY).truncateBlocking(); - - AutoRepair.instance = new AutoRepair(); + AutoRepair.instance.isSetupDone = false; + AutoRepair.instance.setup(); executeCQL(); timeFuncCalls = 0; @@ -270,24 +270,6 @@ private void executeCQL() .forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); } - @Test(expected = ConfigurationException.class) - public void testRepairAsyncWithRepairTypeDisabled() - { - AutoRepairService.instance.getAutoRepairConfig().setAutoRepairEnabled(repairType, false); - - AutoRepair.instance.repairAsync(repairType); - } - - @Test - public void testRepairAsync() - { - AutoRepair.instance.repairExecutors.put(repairType, mockExecutor); - - AutoRepair.instance.repairAsync(repairType); - - verify(mockExecutor, times(1)).submit(any(Runnable.class)); - } - @Test public void testRepairTurn() { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index 27a6da7cc478..53eaca86dcbc 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -66,14 +66,12 @@ public void setup() @Test public void testSetup() { - AutoRepair instance = new AutoRepair(); - instance.setup(); - - assertEquals(RepairType.values().length, instance.repairExecutors.size()); - for (RepairType repairType : instance.repairExecutors.keySet()) + AutoRepair.instance.setup(); + assertEquals(RepairType.values().length, AutoRepair.instance.repairExecutors.size()); + for (RepairType repairType : AutoRepair.instance.repairExecutors.keySet()) { - int expectedTasks = instance.repairExecutors.get(repairType).getPendingTaskCount() - + instance.repairExecutors.get(repairType).getActiveTaskCount(); + int expectedTasks = AutoRepair.instance.repairExecutors.get(repairType).getPendingTaskCount() + + AutoRepair.instance.repairExecutors.get(repairType).getActiveTaskCount(); assertTrue(String.format("Expected > 0 task in queue for %s but was %s", repairType, expectedTasks), expectedTasks > 0); } @@ -82,18 +80,16 @@ public void testSetup() @Test public void testSafeGuardSetupCall() { - AutoRepair instance = new AutoRepair(); - // only one should be setup, and rest should be ignored - instance.setup(); - instance.setup(); - instance.setup(); + AutoRepair.instance.setup(); + AutoRepair.instance.setup(); + AutoRepair.instance.setup(); - assertEquals(RepairType.values().length, instance.repairExecutors.size()); - for (RepairType repairType : instance.repairExecutors.keySet()) + assertEquals(RepairType.values().length, AutoRepair.instance.repairExecutors.size()); + for (RepairType repairType : AutoRepair.instance.repairExecutors.keySet()) { - int expectedTasks = instance.repairExecutors.get(repairType).getPendingTaskCount() - + instance.repairExecutors.get(repairType).getActiveTaskCount(); + int expectedTasks = AutoRepair.instance.repairExecutors.get(repairType).getPendingTaskCount() + + AutoRepair.instance.repairExecutors.get(repairType).getActiveTaskCount(); assertTrue(String.format("Expected > 0 task in queue for %s but was %s", repairType, expectedTasks), expectedTasks > 0); } @@ -106,8 +102,8 @@ public void testSetupFailsWhenIREnabledWithCDCReplay() DatabaseDescriptor.setCDCOnRepairEnabled(true); DatabaseDescriptor.setCDCEnabled(true); - AutoRepair instance = new AutoRepair(); - instance.setup(); + AutoRepair.instance.isSetupDone = false; + AutoRepair.instance.setup(); } @Test @@ -117,8 +113,7 @@ public void testNoFailureIfMVRepairOnButConfigIsOff() DatabaseDescriptor.getAutoRepairConfig().setMaterializedViewRepairEnabled(RepairType.INCREMENTAL, false); DatabaseDescriptor.setCDCOnRepairEnabled(false); DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); - AutoRepair instance = new AutoRepair(); - instance.setup(); + AutoRepair.instance.setup(); } @Test(expected = ConfigurationException.class) @@ -128,8 +123,8 @@ public void testSetupFailsWhenIREnabledWithMVReplay() DatabaseDescriptor.getAutoRepairConfig().setMaterializedViewRepairEnabled(RepairType.INCREMENTAL, true); DatabaseDescriptor.setCDCOnRepairEnabled(false); DatabaseDescriptor.setMaterializedViewsOnRepairEnabled(true); - AutoRepair instance = new AutoRepair(); - instance.setup(); + AutoRepair.instance.isSetupDone = false; + AutoRepair.instance.setup(); } @Test From 42f751f09214c2096e7bcdf9d56045ca147f3476 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 6 Apr 2025 20:12:54 -0700 Subject: [PATCH 181/199] parens in next line --- .../autorepair/AutoRepairStateTest.java | 8 +++-- .../nodetool/SSTableRepairedSetTest.java | 29 ++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java index fc4ac2f03c79..422ebdff5192 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -185,7 +185,9 @@ public void testGetLongestUnrepairedSecNull() try { assertEquals(0, state.getLongestUnrepairedSec()); - } catch (Exception e) { + } + catch (Exception e) + { assertNull(e); } } @@ -201,7 +203,9 @@ public void testGetLongestUnrepairedSec() try { assertEquals(1, state.getLongestUnrepairedSec()); - } catch (Exception e) { + } + catch (Exception e) + { assertNull(e); } } diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java index edd57447ad13..5d23d22253ad 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SSTableRepairedSetTest.java @@ -50,15 +50,23 @@ public class SSTableRepairedSetTest private SSTableRepairedSet cmd; @Before - public void setUp() { + public void setUp() + { MockitoAnnotations.initMocks(this); - PrintStream noopStream = new PrintStream(new OutputStream() {@Override public void write(int b) {}}); + PrintStream noopStream = new PrintStream(new OutputStream() + { + @Override + public void write(int b) + { + } + }); when(probe.output()).thenReturn(new Output(noopStream, noopStream)); cmd = new SSTableRepairedSet(); } @Test - public void testNoKeyspace() { + public void testNoKeyspace() + { when(probe.getNonLocalStrategyKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("ks1", "ks2"))); when(probe.getKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("ks1", "ks2"))); when(probe.getAutoRepairTablesForKeyspace("ks1")).thenReturn(new ArrayList<>(Arrays.asList("table1", "table2"))); @@ -73,7 +81,8 @@ public void testNoKeyspace() { } @Test - public void testBothRepairedAndUnrepaired() { + public void testBothRepairedAndUnrepaired() + { cmd.args = Arrays.asList("keyspace"); cmd.isRepaired = true; cmd.isUnrepaired = true; @@ -82,14 +91,16 @@ public void testBothRepairedAndUnrepaired() { } @Test - public void testNeitherRepairedNorUnrepaired() { + public void testNeitherRepairedNorUnrepaired() + { cmd.args = Arrays.asList("keyspace"); cmd.execute(probe); verify(probe, never()).mutateSSTableRepairedState(anyBoolean(), anyBoolean(), anyString(), anyList()); } @Test - public void testRepairedPreview() { + public void testRepairedPreview() + { cmd.args = Arrays.asList("keyspace"); when(probe.getKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("keyspace"))); cmd.isRepaired = true; @@ -98,7 +109,8 @@ public void testRepairedPreview() { } @Test - public void testUnrepairedReallySet() { + public void testUnrepairedReallySet() + { cmd.args = Arrays.asList("keyspace"); when(probe.getKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("keyspace"))); cmd.isUnrepaired = true; @@ -108,7 +120,8 @@ public void testUnrepairedReallySet() { } @Test - public void testExecuteWithTableNames() { + public void testExecuteWithTableNames() + { cmd.args = Arrays.asList("keyspace", "table1", "table2"); when(probe.getKeyspaces()).thenReturn(new ArrayList<>(Arrays.asList("keyspace"))); cmd.isRepaired = true; From 5bcb2e5220fd392c7c961bb8ae119334d685c7b3 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 21 Apr 2025 15:03:26 -0700 Subject: [PATCH 182/199] Update doc/modules/cassandra/pages/managing/operating/metrics.adoc Co-authored-by: Francisco Guerrero --- doc/modules/cassandra/pages/operating/metrics.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/cassandra/pages/operating/metrics.adoc b/doc/modules/cassandra/pages/operating/metrics.adoc index 451b9b245bfa..4ea2c2b7c715 100644 --- a/doc/modules/cassandra/pages/operating/metrics.adoc +++ b/doc/modules/cassandra/pages/operating/metrics.adoc @@ -1007,7 +1007,7 @@ delayed because a replica was currently taking its turn. Only revelent if `allow_parallel_replica_repair` is false. |RepairDelayedBySchedule |Counter |Represents occurrences of a node's turn being -delayed because it was already being repaired in another schedule. Only relevent +delayed because it was already being repaired in another schedule. Only relevant if `allow_parallel_replica_repair_across_schedules` is false. |=== From 3b9e193062166b76635cd87a95c5e80ed2f72298 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 21 Apr 2025 15:03:45 -0700 Subject: [PATCH 183/199] Update src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java Co-authored-by: Francisco Guerrero --- .../cassandra/repair/autorepair/RepairTokenRangeSplitter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index 99c3e3add9ec..a1511db342d4 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -92,7 +92,7 @@ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter private static final Logger logger = LoggerFactory.getLogger(RepairTokenRangeSplitter.class); // Default max bytes to 100TiB, which is much more readable than Long.MAX_VALUE - private static final DataStorageSpec.LongBytesBound MAX_BYTES = new DataStorageSpec.LongBytesBound(100_000, DataStorageSpec.DataStorageUnit.GIBIBYTES); + private static final DataStorageSpec.LongBytesBound MAX_BYTES = new DataStorageSpec.LongBytesBound(102_400, DataStorageSpec.DataStorageUnit.GIBIBYTES); /** * The target bytes that should be included in a repair assignment From 2fbd54f709fc9497093969acdf05eb0dca6c10e4 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 21 Apr 2025 15:04:30 -0700 Subject: [PATCH 184/199] Update src/java/org/apache/cassandra/schema/AutoRepairParams.java Co-authored-by: Francisco Guerrero --- src/java/org/apache/cassandra/schema/AutoRepairParams.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index 8e82df1cbd52..1bde7e603c33 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -69,11 +69,7 @@ public String toString() public static AutoRepairParams create(Map options) { - Map optionsMap = new TreeMap<>(); - for (Map.Entry entry : DEFAULT_OPTIONS.entrySet()) - { - optionsMap.put(entry.getKey(), entry.getValue()); - } + Map optionsMap = new TreeMap<>(DEFAULT_OPTIONS); if (options != null) { for (Map.Entry entry : options.entrySet()) From be32a2e63b66d1275decfd09f11e481f2f456df7 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 21 Apr 2025 15:04:52 -0700 Subject: [PATCH 185/199] Update src/java/org/apache/cassandra/schema/AutoRepairParams.java Co-authored-by: Francisco Guerrero --- src/java/org/apache/cassandra/schema/AutoRepairParams.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index 1bde7e603c33..3214b2dc661c 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -95,10 +95,8 @@ public boolean repairEnabled(AutoRepairConfig.RepairType type) public int priority() { - String priority = options.get(Option.PRIORITY.toString()); - return priority == null - ? Integer.parseInt(DEFAULT_OPTIONS.get(Option.PRIORITY.toString())) - : Integer.parseInt(priority); + String priority = options.getOrDefault(Option.PRIORITY.toString(), DEFAULT_OPTIONS.get(Option.PRIORITY.toString())); + return Integer.parseInt(priority); } public void validate() From 15242bd8ec2a225035327e400dd35a7640816be2 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 21 Apr 2025 15:06:14 -0700 Subject: [PATCH 186/199] Update src/java/org/apache/cassandra/schema/AutoRepairParams.java Co-authored-by: Francisco Guerrero --- src/java/org/apache/cassandra/schema/AutoRepairParams.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java index 3214b2dc661c..5f05edab9dd8 100644 --- a/src/java/org/apache/cassandra/schema/AutoRepairParams.java +++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java @@ -87,10 +87,8 @@ public static AutoRepairParams create(Map options) public boolean repairEnabled(AutoRepairConfig.RepairType type) { String option = type.toString().toLowerCase() + "_enabled"; - String enabled = options.get(option); - return enabled == null - ? Boolean.parseBoolean(DEFAULT_OPTIONS.get(option)) - : Boolean.parseBoolean(enabled); + String enabled = options.getOrDefault(option, DEFAULT_OPTIONS.get(option)); + return Boolean.parseBoolean(enabled); } public int priority() From db25cbae72b081a38277a8e22d507c2a3b4dff4e Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 21 Apr 2025 15:37:42 -0700 Subject: [PATCH 187/199] Validate disk headroom for IR and a couple of comment updates --- .../cassandra/config/DatabaseDescriptor.java | 3 +++ .../autorepair/RepairTokenRangeSplitter.java | 17 +++++++++++++++++ .../schema/SystemDistributedKeyspace.java | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 12964456e2e8..9ea9a833c257 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -4549,6 +4549,9 @@ public static double getIncrementalRepairDiskHeadroomRejectRatio() public static void setIncrementalRepairDiskHeadroomRejectRatio(double value) { + if (value < 0.0 || value > 1.0) { + throw new IllegalArgumentException("Value must be >= 0 and <= 1 for incremental_repair_disk_headroom_reject_ratio"); + } conf.incremental_repair_disk_headroom_reject_ratio = value; } diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index a1511db342d4..2922f7f4044a 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -86,6 +86,23 @@ * shall be sensible for all {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType}'s. The following * configuration parameters are offered. *

        + * + *

        Configuration parameters:

        + *
          + *
        • bytes_per_assignment – Target size (in compressed bytes) for each repair. Throttles incremental repair + * and anticompaction per schedule after incremental repairs are enabled.
          • + * + *
        • max_bytes_per_schedule – Maximum data (in compressed bytes) to cover in a single schedule. Acts as a + * throttle for the repair cycle workload. Tune this up if writes are outpacing repair, or down if repairs are too + * disruptive. Alternatively, adjust {@code min_repair_interval}.
          • + * + *
        • partitions_per_assignment – Maximum number of partitions per repair assignment. Limits the number of + * partitions in Merkle tree leaves to prevent overstreaming.
          • + * + *
        • max_tables_per_assignment – Maximum number of tables to include in a single repair assignment. + * Especially useful for keyspaces with many tables. Prevents excessive batching of tables that exceed other + * parameters like {@code bytes_per_assignment} or {@code partitions_per_assignment}.
          • + *
        */ public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter { diff --git a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java index 94e33d2b69f9..0bd9080eae02 100644 --- a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java @@ -82,7 +82,7 @@ private SystemDistributedKeyspace() * gen 4: compression chunk length reduced to 16KiB, memtable_flush_period_in_ms now unset on all tables in 4.0 * gen 5: add ttl and TWCS to repair_history tables * gen 6: add denylist table - * gen 7: add AutoRepair related tables + * gen 7: add auto_repair_history and auto_repair_priority tables for AutoRepair feature */ public static final long GENERATION = 7; From 22b204f1fc372bf9ea799743d1302eb74cd5ad04 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 21 Apr 2025 15:43:35 -0700 Subject: [PATCH 188/199] Format { on the new line --- src/java/org/apache/cassandra/config/DatabaseDescriptor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 9ea9a833c257..a43a03b90db2 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -4549,7 +4549,8 @@ public static double getIncrementalRepairDiskHeadroomRejectRatio() public static void setIncrementalRepairDiskHeadroomRejectRatio(double value) { - if (value < 0.0 || value > 1.0) { + if (value < 0.0 || value > 1.0) + { throw new IllegalArgumentException("Value must be >= 0 and <= 1 for incremental_repair_disk_headroom_reject_ratio"); } conf.incremental_repair_disk_headroom_reject_ratio = value; From a48a40df8ffe5f467428d2227e2695cf9f456b0d Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 21 Apr 2025 18:59:34 -0700 Subject: [PATCH 189/199] Update doc/modules/cassandra/pages/managing/operating/metrics.adoc Co-authored-by: Francisco Guerrero --- doc/modules/cassandra/pages/operating/metrics.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/cassandra/pages/operating/metrics.adoc b/doc/modules/cassandra/pages/operating/metrics.adoc index 4ea2c2b7c715..f39f1a503016 100644 --- a/doc/modules/cassandra/pages/operating/metrics.adoc +++ b/doc/modules/cassandra/pages/operating/metrics.adoc @@ -1003,7 +1003,7 @@ the automated repair has been disabled on the node due to priority set in the automated repair |RepairDelayedByReplica |Counter |Represents occurrences of a node's turn being -delayed because a replica was currently taking its turn. Only revelent if +delayed because a replica was currently taking its turn. Only relevant if `allow_parallel_replica_repair` is false. |RepairDelayedBySchedule |Counter |Represents occurrences of a node's turn being From 5e3737346c05bb7a52838cac8a33822eebf3a6a9 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Thu, 24 Apr 2025 12:04:22 -0700 Subject: [PATCH 190/199] Increasing default for auto_repair.sstable_upper_threshold considering large Cassandra tables --- conf/cassandra.yaml | 2 +- doc/modules/cassandra/managing/operating/auto_repair.adoc | 6 +++--- .../cassandra/repair/autorepair/AutoRepairConfig.java | 2 +- .../cassandra/repair/autorepair/AutoRepairConfigTest.java | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 470d05b0def8..ef84544b1284 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1998,7 +1998,7 @@ drop_compact_storage_enabled: false # force_repair_new_node: false # # Threshold to skip repairing tables with too many SSTables. Defaults to 10,000 SSTables to avoid penalizing good # # tables. -# sstable_upper_threshold: 10000 +# sstable_upper_threshold: 50000 # # Maximum time allowed for repairing one table on a given node. If exceeded, the repair proceeds to the # # next table. # table_max_repair_time: 6h diff --git a/doc/modules/cassandra/managing/operating/auto_repair.adoc b/doc/modules/cassandra/managing/operating/auto_repair.adoc index 3928d0afccef..d5701895d74f 100644 --- a/doc/modules/cassandra/managing/operating/auto_repair.adoc +++ b/doc/modules/cassandra/managing/operating/auto_repair.adoc @@ -207,7 +207,7 @@ node. Defaults to true and is only evaluated when allow_parallel_replica_repair immediately after a restart. | repair_session_timeout | 3h | Timeout for retrying stuck repair sessions. | force_repair_new_node | false | Force immediate repair on new nodes after they join the ring. -| sstable_upper_threshold | 10000 | Threshold to skip repairing tables with too many SSTables. +| sstable_upper_threshold | 50000 | Threshold to skip repairing tables with too many SSTables. | table_max_repair_time | 6h | Maximum time allowed for repairing one table on a given node. If exceeded, the repair proceeds to the next table. | ignore_dcs | [] | Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify @@ -365,7 +365,7 @@ configuration for repair_type: full min_repair_interval: 24h repair_by_keyspace: true number_of_repair_threads: 1 - sstable_upper_threshold: 10000 + sstable_upper_threshold: 50000 table_max_repair_time: 6h ignore_dcs: [] repair_primary_token_range_only: true @@ -387,7 +387,7 @@ configuration for repair_type: incremental min_repair_interval: 1h repair_by_keyspace: true number_of_repair_threads: 1 - sstable_upper_threshold: 10000 + sstable_upper_threshold: 50000 table_max_repair_time: 6h ignore_dcs: [] repair_primary_token_range_only: true diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 1fa1ac4f8c4b..f3f4b888010a 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -451,7 +451,7 @@ protected static Options getDefaultOptions() opts.parallel_repair_percentage = 3; opts.allow_parallel_replica_repair = false; opts.allow_parallel_replica_repair_across_schedules = true; - opts.sstable_upper_threshold = 10000; + opts.sstable_upper_threshold = 50000; opts.ignore_dcs = new HashSet<>(); opts.repair_primary_token_range_only = true; opts.force_repair_new_node = false; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java index 0cf51ca40884..93ef906a287d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairConfigTest.java @@ -474,7 +474,7 @@ public void testDefaultOptions() assertEquals(Integer.valueOf(1), options.number_of_repair_threads); assertEquals(Integer.valueOf(3), options.parallel_repair_count); assertEquals(Integer.valueOf(3), options.parallel_repair_percentage); - assertEquals(Integer.valueOf(10000), options.sstable_upper_threshold); + assertEquals(Integer.valueOf(50000), options.sstable_upper_threshold); assertEquals(new HashSet<>(), options.ignore_dcs); assertTrue(options.repair_primary_token_range_only); assertFalse(options.force_repair_new_node); @@ -495,7 +495,7 @@ public void testGlobalOptions() assertEquals(Integer.valueOf(1), config.global_settings.number_of_repair_threads); assertEquals(Integer.valueOf(3), config.global_settings.parallel_repair_count); assertEquals(Integer.valueOf(3), config.global_settings.parallel_repair_percentage); - assertEquals(Integer.valueOf(10000), config.global_settings.sstable_upper_threshold); + assertEquals(Integer.valueOf(50000), config.global_settings.sstable_upper_threshold); assertEquals(new HashSet<>(), config.global_settings.ignore_dcs); assertTrue(config.global_settings.repair_primary_token_range_only); assertFalse(config.global_settings.force_repair_new_node); From 512a9d4ac2b93182d4290dd1c35bd280f05d906a Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 5 May 2025 11:23:26 -0700 Subject: [PATCH 191/199] Fix AutoRepair Flaky InJvm dtest Patch by Jaydeepkumar Chovatia; Reviewed by Andy Tolbert, Chris Lohfink for CASSANDRA-20620 --- .../repair/autorepair/AutoRepair.java | 25 ++++++++++--------- ...allelReplicaRepairAcrossSchedulesTest.java | 2 +- .../test/repair/AutoRepairSchedulerTest.java | 9 ++++--- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 93e371e3b5d9..c60119900704 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -196,7 +196,7 @@ public void repair(AutoRepairConfig.RepairType repairType) boolean primaryRangeOnly = config.getRepairPrimaryTokenRangeOnly(repairType) && turn != MY_TURN_FORCE_REPAIR; - long startTime = timeFunc.get(); + long startTimeInMillis = timeFunc.get(); logger.info("My host id: {}, my turn to run repair...repair primary-ranges only? {}", myId, config.getRepairPrimaryTokenRangeOnly(repairType)); AutoRepairUtils.updateStartAutoRepairHistory(repairType, myId, timeFunc.get(), turn); @@ -242,7 +242,7 @@ public void repair(AutoRepairConfig.RepairType repairType) repairKeyspace(repairType, primaryRangeOnly, repairAssignments.getKeyspaceName(), repairAssignments.getRepairAssignments(), collectedRepairStats); } - cleanupAndUpdateStats(turn, repairType, repairState, myId, startTime, collectedRepairStats); + cleanupAndUpdateStats(turn, repairType, repairState, myId, startTimeInMillis, collectedRepairStats); } else { @@ -449,7 +449,7 @@ private List retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairCon } private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, UUID myId, - long startTime, CollectedRepairStats collectedRepairStats) throws InterruptedException + long startTimeInMillis, CollectedRepairStats collectedRepairStats) throws InterruptedException { //if it was due to priority then remove it now if (turn == MY_TURN_DUE_TO_PRIORITY) @@ -457,12 +457,19 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType logger.info("Remove current host from priority list"); AutoRepairUtils.removePriorityStatus(repairType, myId); } - + long repairScheduleElapsedInMillis = timeFunc.get() - startTimeInMillis; + if (repairScheduleElapsedInMillis < SLEEP_IF_REPAIR_FINISHES_QUICKLY.toMilliseconds()) + { + //If repair finished quickly, happens for Cassndra cluster with empty (or tiny) data, in such cases, + //wait for some duration so that the JMX metrics can detect the repairInProgress + logger.info("Wait for {}ms for repair type {}.", SLEEP_IF_REPAIR_FINISHES_QUICKLY.toMilliseconds() - repairScheduleElapsedInMillis, repairType); + Thread.sleep(SLEEP_IF_REPAIR_FINISHES_QUICKLY.toMilliseconds() - repairScheduleElapsedInMillis); + } repairState.setFailedTokenRangesCount(collectedRepairStats.failedTokenRanges); repairState.setSucceededTokenRangesCount(collectedRepairStats.succeededTokenRanges); repairState.setSkippedTokenRangesCount(collectedRepairStats.skippedTokenRanges); repairState.setSkippedTablesCount(collectedRepairStats.skippedTables); - repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - startTime)); + repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(repairScheduleElapsedInMillis)); long timeInHours = TimeUnit.SECONDS.toHours(repairState.getNodeRepairTimeInSec()); logger.info("Local {} repair time {} hour(s), stats: repairKeyspaceCount {}, " + "repairTokenRangesSuccessCount {}, repairTokenRangesFailureCount {}, " + @@ -477,13 +484,7 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType TimeUnit.SECONDS.toDays(repairState.getClusterRepairTimeInSec())); } repairState.setLastRepairTime(timeFunc.get()); - if (timeInHours == 0 && SLEEP_IF_REPAIR_FINISHES_QUICKLY.toSeconds() > 0) - { - //If repair finished quickly, happens for an empty instance, in such case - //wait for some duration so that the JMX metrics can detect the repairInProgress - logger.info("Wait for {} for repair type {}.", SLEEP_IF_REPAIR_FINISHES_QUICKLY, repairType); - Thread.sleep(SLEEP_IF_REPAIR_FINISHES_QUICKLY.toMilliseconds()); - } + repairState.setRepairInProgress(false); AutoRepairUtils.updateFinishAutoRepairHistory(repairType, myId, timeFunc.get()); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java index 0a2238ac4024..f7fa023e5b47 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java @@ -34,6 +34,7 @@ import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; @@ -117,7 +118,6 @@ public void testScheduler() cluster.forEach(i -> i.runOnInstance(() -> { // Expect contention on incremental repair across schedules AutoRepairMetrics incrementalMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.INCREMENTAL); - while (incrementalMetrics.repairDelayedByReplica.getCount() <= 0) { try diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index 49892741c9a2..642adb502ccf 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -45,6 +45,7 @@ import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.schema.SchemaConstants.DISTRIBUTED_KEYSPACE_NAME; import static org.junit.Assert.assertEquals; @@ -79,11 +80,11 @@ public static void init() throws IOException ImmutableMap.of( "initial_scheduler_delay", "5s", "enabled", "true", - "parallel_repair_count", "2", + "parallel_repair_count", "3", // Allow parallel replica repair to allow replicas // to execute full repair at same time. "allow_parallel_replica_repair", "true", - "min_repair_interval", "15s"), + "min_repair_interval", "5s"), AutoRepairConfig.RepairType.INCREMENTAL.getConfigName(), ImmutableMap.of( "initial_scheduler_delay", "5s", @@ -139,9 +140,11 @@ public void testScheduler() throws ParseException logger.info("Repair setup done"); // validate that the repair ran on all nodes cluster.forEach(i -> i.runOnInstance(() -> { + String broadcastAddress = FBUtilities.getJustBroadcastAddress().toString(); + // Reduce sleeping if repair finishes quickly to speed up test but make it non-zero to provoke some // contention. - AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("1s"); + AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("2s"); AutoRepairMetrics incrementalMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.INCREMENTAL); while (incrementalMetrics.nodeRepairTimeInSec.getValue().longValue() <= 0) From 44e960b516bb778d5f2f0e6f54a9d8ddd94db949 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Mon, 5 May 2025 18:39:07 -0700 Subject: [PATCH 192/199] Fix a bug in AutoRepair duration metric calculation if schedule finishes quickly patch by Jaydeepkumar Chovatia; reviewed by Andy Tolbert for CASSANDRA-20622 --- .../cassandra/repair/autorepair/AutoRepair.java | 2 +- ...lowParallelReplicaRepairAcrossSchedulesTest.java | 2 +- .../test/repair/AutoRepairSchedulerTest.java | 13 ++++++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index c60119900704..6d3ace58fd4c 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -469,7 +469,7 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType repairState.setSucceededTokenRangesCount(collectedRepairStats.succeededTokenRanges); repairState.setSkippedTokenRangesCount(collectedRepairStats.skippedTokenRanges); repairState.setSkippedTablesCount(collectedRepairStats.skippedTables); - repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(repairScheduleElapsedInMillis)); + repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - startTimeInMillis)); long timeInHours = TimeUnit.SECONDS.toHours(repairState.getNodeRepairTimeInSec()); logger.info("Local {} repair time {} hour(s), stats: repairKeyspaceCount {}, " + "repairTokenRangesSuccessCount {}, repairTokenRangesFailureCount {}, " + diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java index f7fa023e5b47..0a2238ac4024 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java @@ -34,7 +34,6 @@ import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.service.AutoRepairService; -import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; @@ -118,6 +117,7 @@ public void testScheduler() cluster.forEach(i -> i.runOnInstance(() -> { // Expect contention on incremental repair across schedules AutoRepairMetrics incrementalMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.INCREMENTAL); + while (incrementalMetrics.repairDelayedByReplica.getCount() <= 0) { try diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java index 642adb502ccf..e5bd83674b3e 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java @@ -45,7 +45,6 @@ import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.cassandra.repair.autorepair.AutoRepairConfig; import org.apache.cassandra.service.AutoRepairService; -import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.schema.SchemaConstants.DISTRIBUTED_KEYSPACE_NAME; import static org.junit.Assert.assertEquals; @@ -140,14 +139,15 @@ public void testScheduler() throws ParseException logger.info("Repair setup done"); // validate that the repair ran on all nodes cluster.forEach(i -> i.runOnInstance(() -> { - String broadcastAddress = FBUtilities.getJustBroadcastAddress().toString(); - // Reduce sleeping if repair finishes quickly to speed up test but make it non-zero to provoke some // contention. AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("2s"); AutoRepairMetrics incrementalMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.INCREMENTAL); - while (incrementalMetrics.nodeRepairTimeInSec.getValue().longValue() <= 0) + // Since the AutoRepair sleeps up to SLEEP_IF_REPAIR_FINISHES_QUICKLY if the repair finishes quickly, + // so the "nodeRepairTimeInSec" metric should at least be greater than or equal to + // SLEEP_IF_REPAIR_FINISHES_QUICKLY + while (incrementalMetrics.nodeRepairTimeInSec.getValue().longValue() <= 1) { try { @@ -179,7 +179,10 @@ public void testScheduler() throws ParseException assertEquals(0L, incrementalMetrics.repairDelayedBySchedule.getCount()); AutoRepairMetrics fullMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.FULL); - while (fullMetrics.nodeRepairTimeInSec.getValue().longValue() <= 0) + // Since the AutoRepair sleeps up to SLEEP_IF_REPAIR_FINISHES_QUICKLY if the repair finishes quickly, + // so the "nodeRepairTimeInSec" metric should at least be greater than or equal to + // SLEEP_IF_REPAIR_FINISHES_QUICKLY + while (fullMetrics.nodeRepairTimeInSec.getValue().longValue() <= 1) { try { From afdffb48a4860d29f5cde3f0fcf122da89f58b65 Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Fri, 16 May 2025 14:45:08 -0700 Subject: [PATCH 193/199] Stop AutoRepair monitoring thread upon Cassandra shutdown patch by Jaydeepkumar Chovatia; reviewed by Bernardo Botella, Andrew Tolbert for CASSANDRA-20623 --- CHANGES.txt | 1 + .../repair/autorepair/AutoRepair.java | 34 +++++++++ .../cassandra/service/StorageService.java | 2 + .../autorepair/AutoRepairShutdownTest.java | 75 +++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 test/unit/org/apache/cassandra/repair/autorepair/AutoRepairShutdownTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 3ac558b08e9b..ee9a4ca3aeca 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1.6 + * Stop AutoRepair monitoring thread upon Cassandra shutdown (CASSANDRA-20623) * Fix race condition in auto-repair scheduler (CASSANDRA-20265) * Implement minimum repair task duration setting for auto-repair scheduler (CASSANDRA-20160) * Implement preview_repaired auto-repair type (CASSANDRA-20046) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 6d3ace58fd4c..bdf6bfdc176b 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -102,6 +103,8 @@ public class AutoRepair public boolean isSetupDone = false; public static AutoRepair instance = new AutoRepair(); + public volatile boolean isShutDown = false; + private AutoRepair() { // Private constructor to prevent instantiation @@ -566,4 +569,35 @@ public void progress(String tag, ProgressEvent event) } } } + + public synchronized void shutdownBlocking() throws ExecutionException, InterruptedException + { + if (!isSetupDone) + { + // By default, executors within AutoRepair are not initialized as the feature is opt-in. + // If the AutoRepair has not been set up, then there is no need to worry about shutting it down + return; + } + if (isShutDown) + { + throw new IllegalStateException("AutoRepair has already been shut down"); + } + isShutDown = true; + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) + { + repairRunnableExecutors.get(repairType).shutdown(); + repairExecutors.get(repairType).shutdown(); + } + logger.info("Paused AutoRepair"); + } + + public Map getRepairExecutors() + { + return repairExecutors; + } + + public Map getRepairRunnableExecutors() + { + return repairRunnableExecutors; + } } diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index c689875c79ef..04095a6c40fd 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -5579,6 +5579,8 @@ protected synchronized void drain(boolean isFinalShutdown) throws IOException, I CommitLog.instance.shutdownBlocking(); + AutoRepair.instance.shutdownBlocking(); + // wait for miscellaneous tasks like sstable and commitlog segment deletion ColumnFamilyStore.shutdownPostFlushExecutor(); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairShutdownTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairShutdownTest.java new file mode 100644 index 000000000000..c7de17e0e455 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairShutdownTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; + +import static org.apache.cassandra.Util.setAutoRepairEnabled; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Unit tests to validate the executor shutdown inside {@link AutoRepair} + */ +public class AutoRepairShutdownTest extends CQLTester +{ + @BeforeClass + public static void setupClass() throws Exception + { + setAutoRepairEnabled(true); + requireNetwork(); + } + + @Test + public void testAutoRepairShutdown() throws Exception + { + AutoRepair.instance.setup(); + + for (RepairType type : RepairType.values()) + { + assertFalse("RepairRunnableExecutor should not have been shut down", AutoRepair.instance.getRepairRunnableExecutors().get(type).isShutdown()); + assertFalse("RepairExecutor should not have been shut down", AutoRepair.instance.getRepairExecutors().get(type).isShutdown()); + } + assertFalse("AutoRepair should not be marked as shut down", AutoRepair.instance.isShutDown); + + AutoRepair.instance.shutdownBlocking(); + + for (RepairType type : RepairType.values()) + { + assertTrue("RepairRunnableExecutor should be shut down", AutoRepair.instance.getRepairRunnableExecutors().get(type).isShutdown()); + assertTrue("RepairExecutor should be shut down", AutoRepair.instance.getRepairExecutors().get(type).isShutdown()); + } + assertTrue("AutoRepair should be marked as shut down", AutoRepair.instance.isShutDown); + + try + { + AutoRepair.instance.shutdownBlocking(); + fail("A second call to shutdown should have thrown an exception"); + } + catch (IllegalStateException e) + { + // expected + } + } +} From ae294091ce7c55dac27d7277b492ac32e1ece92f Mon Sep 17 00:00:00 2001 From: Himanshu Jindal Date: Tue, 12 Aug 2025 09:59:36 -0700 Subject: [PATCH 194/199] Safeguard Full repair against disk protection As per CASSANDRA-20045, we want to prevent full repair against disk full scenarios. Current protection exists only for incremental repair. This change updates the config name to not be incremental repair specific, using the Replace annotation. patch by Himanshu Jindal; reviewed by David Capwell, Jaydeepkumar Chovatia for CASSANDRA-20045 --- CHANGES.txt | 1 + conf/cassandra.yaml | 8 ++++++-- .../cassandra/managing/operating/auto_repair.adoc | 4 ++-- src/java/org/apache/cassandra/config/Config.java | 7 ++++++- .../apache/cassandra/config/DatabaseDescriptor.java | 10 +++++----- .../cassandra/repair/RepairMessageVerbHandler.java | 2 +- .../cassandra/service/ActiveRepairService.java | 11 ++++------- .../cassandra/service/ActiveRepairServiceTest.java | 12 ++++++------ 8 files changed, 31 insertions(+), 24 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ee9a4ca3aeca..a44b38996ef6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1.6 + * AutoRepair: Safeguard Full repair against disk protection(CASSANDRA-20045) * Stop AutoRepair monitoring thread upon Cassandra shutdown (CASSANDRA-20623) * Fix race condition in auto-repair scheduler (CASSANDRA-20265) * Implement minimum repair task duration setting for auto-repair scheduler (CASSANDRA-20160) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index ef84544b1284..42264145de81 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1871,10 +1871,14 @@ drop_compact_storage_enabled: false # the given value. Defaults to disabled. # reject_repair_compaction_threshold: 1024 -# At least 20% of disk must be unused to run incremental repair. It is useful to avoid disks filling up during -# incremental repair as anti-compaction during incremental repair may contribute to additional space temporarily. +# At least 20% of disk must be unused to run repair. It is useful to avoid disks filling up during +# repair as anti-compaction during repair may contribute to additional space temporarily. # if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason) # then set the ratio to 0.0 +# repair_disk_headroom_reject_ratio: 0.2; + +# This is the deprecated config which was used to safeguard incremental repairs. Use repair_disk_headroom_reject_ratio +# instead as it safeguards against all repairs. # incremental_repair_disk_headroom_reject_ratio: 0.2; # Configuration for Auto Repair Scheduler. diff --git a/doc/modules/cassandra/managing/operating/auto_repair.adoc b/doc/modules/cassandra/managing/operating/auto_repair.adoc index d5701895d74f..bd9dad5aa716 100644 --- a/doc/modules/cassandra/managing/operating/auto_repair.adoc +++ b/doc/modules/cassandra/managing/operating/auto_repair.adoc @@ -318,9 +318,9 @@ When enabling auto_repair, it is advisable to configure the top level `reject_re configuration in cassandra.yaml as a backpressure mechanism to reject new repairs on instances that have many pending compactions. -==== Tune `incremental_repair_disk_headroom_reject_ratio` +==== Tune `repair_disk_headroom_reject_ratio` -By default, incremental repairs will be rejected if less than 20% of disk is available. If one wishes to be +By default, repairs will be rejected if less than 20% of disk is available. If one wishes to be conservative this top level configuration could be increased to a larger value to prevent filling your data directories. == Table configuration diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 841493e7a5c7..c8e3c69898c7 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -331,8 +331,13 @@ public MemtableOptions() public volatile int concurrent_materialized_view_builders = 1; public volatile int reject_repair_compaction_threshold = Integer.MAX_VALUE; - public volatile double incremental_repair_disk_headroom_reject_ratio = 0.2; // at least 20% of disk must be unused to run incremental repair + // The number of executors to use for building secondary indexes + public volatile int concurrent_index_builders = 2; + // at least 20% of disk must be unused to run repair + // if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason) then set the ratio to 0.0 + @Replaces(oldName = "incremental_repair_disk_headroom_reject_ratio") + public volatile double repair_disk_headroom_reject_ratio = 0.2; /** * @deprecated retry support removed on CASSANDRA-10992 */ diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index a43a03b90db2..960817866e26 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -4542,18 +4542,18 @@ public static AutoRepairConfig getAutoRepairConfig() return conf.auto_repair; } - public static double getIncrementalRepairDiskHeadroomRejectRatio() + public static double getRepairDiskHeadroomRejectRatio() { - return conf.incremental_repair_disk_headroom_reject_ratio; + return conf.repair_disk_headroom_reject_ratio; } - public static void setIncrementalRepairDiskHeadroomRejectRatio(double value) + public static void setRepairDiskHeadroomRejectRatio(double value) { if (value < 0.0 || value > 1.0) { - throw new IllegalArgumentException("Value must be >= 0 and <= 1 for incremental_repair_disk_headroom_reject_ratio"); + throw new IllegalArgumentException("Value must be >= 0 and <= 1 for repair_disk_headroom_reject_ratio"); } - conf.incremental_repair_disk_headroom_reject_ratio = value; + conf.repair_disk_headroom_reject_ratio = value; } @VisibleForTesting diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java index 734cb67f95d7..ed04a0fe6ca9 100644 --- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java +++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java @@ -77,7 +77,7 @@ public void doVerb(final Message message) logger.debug("Duplicate prepare message found for {}", state.id); return; } - if (!ActiveRepairService.verifyDiskHeadroomThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind, prepareMessage.isIncremental)) + if (!ActiveRepairService.verifyDiskHeadroomThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind)) { // error is logged in verifyDiskHeadroomThreshold state.phase.fail("Not enough disk headroom to perform incremental repair"); diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index 451487f3b254..1cec79a83432 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -601,11 +601,8 @@ public static boolean verifyCompactionsPendingThreshold(TimeUUID parentRepairSes return true; } - public static boolean verifyDiskHeadroomThreshold(TimeUUID parentRepairSession, PreviewKind previewKind, boolean isIncremental) + public static boolean verifyDiskHeadroomThreshold(TimeUUID parentRepairSession, PreviewKind previewKind) { - if (!isIncremental) // disk headroom is required for anti-compaction which is only performed by incremental repair - return true; - double diskUsage = DiskUsageMonitor.instance.getDiskUsage(); double rejectRatio = ActiveRepairService.instance.getIncrementalRepairDiskHeadroomRejectRatio(); @@ -621,7 +618,7 @@ public static boolean verifyDiskHeadroomThreshold(TimeUUID parentRepairSession, public TimeUUID prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPort coordinator, Set endpoints, RepairOption options, boolean isForcedRepair, List columnFamilyStores) { - if (!verifyDiskHeadroomThreshold(parentRepairSession, options.getPreviewKind(), options.isIncremental())) + if (!verifyDiskHeadroomThreshold(parentRepairSession, options.getPreviewKind())) failRepair(parentRepairSession, "Rejecting incoming repair, disk usage above threshold"); // failRepair throws exception if (!verifyCompactionsPendingThreshold(parentRepairSession, options.getPreviewKind())) @@ -1008,12 +1005,12 @@ public void setRepairPendingCompactionRejectThreshold(int value) public double getIncrementalRepairDiskHeadroomRejectRatio() { - return DatabaseDescriptor.getIncrementalRepairDiskHeadroomRejectRatio(); + return DatabaseDescriptor.getRepairDiskHeadroomRejectRatio(); } public void setIncrementalRepairDiskHeadroomRejectRatio(double value) { - DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(value); + DatabaseDescriptor.setRepairDiskHeadroomRejectRatio(value); } /** diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java index 669fd335d50c..ac84bbb6e0d8 100644 --- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java @@ -477,9 +477,9 @@ public void testQueueWhenPoolFullStrategy() throws InterruptedException } @Test - public void testVerifyDiskHeadroomThresholdFullRepair() + public void testVerifyDefaultDiskHeadroomThreshold() { - Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, false)); + Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE)); } @Test @@ -487,9 +487,9 @@ public void testVerifyDiskHeadroomThresholdDiskFull() { DiskUsageMonitor.instance = diskUsageMonitor; when(diskUsageMonitor.getDiskUsage()).thenReturn(1.0); - DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(1.0); + DatabaseDescriptor.setRepairDiskHeadroomRejectRatio(1.0); - Assert.assertFalse(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, true)); + Assert.assertFalse(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE)); } @Test @@ -497,9 +497,9 @@ public void testVerifyDiskHeadroomThresholdSufficientDisk() { DiskUsageMonitor.instance = diskUsageMonitor; when(diskUsageMonitor.getDiskUsage()).thenReturn(0.0); - DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(0.0); + DatabaseDescriptor.setRepairDiskHeadroomRejectRatio(0.0); - Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, true)); + Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE)); } @Test(expected = RuntimeException.class) From 3229f58168da6ccfd2e16eae21490c7be9a5406a Mon Sep 17 00:00:00 2001 From: Himanshu Jindal Date: Tue, 12 Aug 2025 15:37:15 -0700 Subject: [PATCH 195/199] Stop repair scheduler if two major versions are detected patch by Himanshu Jindal; reviewed by Jaydeepkumar Chovatia, Andy Tolbert for CASSANDRA-20048 --- CHANGES.txt | 1 + conf/cassandra.yaml | 3 + .../managing/operating/auto_repair.adoc | 4 + .../org/apache/cassandra/gms/Gossiper.java | 2 +- .../repair/autorepair/AutoRepair.java | 5 ++ .../repair/autorepair/AutoRepairConfig.java | 17 ++++ .../repair/autorepair/AutoRepairUtils.java | 23 ++++++ .../cassandra/service/AutoRepairService.java | 7 ++ .../service/AutoRepairServiceMBean.java | 2 + .../org/apache/cassandra/tools/NodeProbe.java | 5 ++ .../tools/nodetool/SetAutoRepairConfig.java | 5 +- .../autorepair/AutoRepairUtilsTest.java | 80 ++++++++++++++++++- .../nodetool/SetAutoRepairConfigTest.java | 3 +- 13 files changed, 153 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index a44b38996ef6..41951243a718 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1.6 + * Stop repair scheduler if two major versions are detected (CASSANDRA-20048) * AutoRepair: Safeguard Full repair against disk protection(CASSANDRA-20045) * Stop AutoRepair monitoring thread upon Cassandra shutdown (CASSANDRA-20623) * Fix race condition in auto-repair scheduler (CASSANDRA-20265) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 42264145de81..dcb89937881c 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1970,6 +1970,9 @@ drop_compact_storage_enabled: false # # The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata # # for a specified duration to ensure they are indeed removed before adjustments are made to the schedule. # history_clear_delete_hosts_buffer_interval: 2h +# # By default repair is disabled if there are mixed major versions detected - which would happen +# # if a major version upgrade is being performed on the cluster, but a user can enable it using this flag +# mixed_major_version_repair_enabled: false # # NOTE: Each of the below settings can be overridden per repair type under repair_type_overrides # global_settings: # # If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired diff --git a/doc/modules/cassandra/managing/operating/auto_repair.adoc b/doc/modules/cassandra/managing/operating/auto_repair.adoc index bd9dad5aa716..e989c49d2a9e 100644 --- a/doc/modules/cassandra/managing/operating/auto_repair.adoc +++ b/doc/modules/cassandra/managing/operating/auto_repair.adoc @@ -167,6 +167,10 @@ is time to schedule repairs. | history_clear_delete_hosts_buffer_interval | 2h | The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata for a specified duration to ensure they are indeed removed before adjustments are made to the schedule. +| mixed_major_version_repair_enabled | false | Enable/Disable running repairs on the cluster when there are mixed +major versions detected, which usually occurs when the cluster is being upgraded. Repairs between nodes of +different major versions is not something that is tested, so this may lead to data compatibility issues. +It is strongly discouraged to set this to true without doing extensive testing beforehand. |=== diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java index ff1264d6fa03..5f6863e0b750 100644 --- a/src/java/org/apache/cassandra/gms/Gossiper.java +++ b/src/java/org/apache/cassandra/gms/Gossiper.java @@ -154,7 +154,7 @@ public static class Props /* live member set */ @VisibleForTesting - final Set liveEndpoints = new ConcurrentSkipListSet<>(); + public final Set liveEndpoints = new ConcurrentSkipListSet<>(); /* unreachable member set */ private final Map unreachableEndpoints = new ConcurrentHashMap<>(); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index bdf6bfdc176b..13804dbd1c59 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -165,6 +165,11 @@ public void repair(AutoRepairConfig.RepairType repairType) logger.debug("Auto-repair is disabled for repair type {}", repairType); return; } + if (!config.isMixedMajorVersionRepairEnabled() && AutoRepairUtils.hasMultipleLiveMajorVersions()) + { + logger.info("Auto-repair is disabled when nodes in the cluster have different major versions"); + return; + } AutoRepairService.instance.checkCanRun(repairType); AutoRepairState repairState = repairStates.get(repairType); try diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index f3f4b888010a..2221c2f2def0 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -58,6 +58,8 @@ public class AutoRepairConfig implements Serializable // Minimum duration for the execution of a single repair task. This prevents the scheduler from overwhelming // the node by scheduling too many repair tasks in a short period of time. public volatile DurationSpec.LongSecondsBound repair_task_min_duration = new DurationSpec.LongSecondsBound("5s"); + // by default repair is disabled if there are mixed major versions detected, but you can enable it using this flag + public volatile boolean mixed_major_version_repair_enabled = false; // global_settings overides Options.defaultOptions for all repair types public volatile Options global_settings; @@ -148,6 +150,11 @@ public void setAutoRepairSchedulingEnabled(boolean enabled) this.enabled = enabled; } + public boolean isMixedMajorVersionRepairEnabled() + { + return mixed_major_version_repair_enabled; + } + public DurationSpec.IntSecondsBound getAutoRepairHistoryClearDeleteHostsBufferInterval() { return history_clear_delete_hosts_buffer_interval; @@ -365,6 +372,16 @@ public void setRepairRetryBackoff(RepairType repairType, String interval) getOptions(repairType).repair_retry_backoff = new DurationSpec.LongSecondsBound(interval); } + public boolean getMixedMajorVersionRepairEnabled() + { + return this.mixed_major_version_repair_enabled; + } + + public void setMixedMajorVersionRepairEnabled(boolean enabled) + { + this.mixed_major_version_repair_enabled = enabled; + } + @VisibleForTesting static IAutoRepairTokenRangeSplitter newAutoRepairTokenRangeSplitter(RepairType repairType, ParameterizedClass parameterizedClass) throws ConfigurationException { diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index 5c8fcb49479c..fcb2d73d188a 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -50,6 +50,7 @@ import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.LocalStrategy; +import org.apache.cassandra.utils.CassandraVersion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -420,6 +421,28 @@ public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType, return null; } + /** + * Checks whether the cluster has multiple major versions + * @return + * true if more than one major versions are detected + * false if only one major version is detected + * + */ + public static boolean hasMultipleLiveMajorVersions() + { + Set liveEndpoints = Gossiper.instance.getLiveMembers(); + Set majorVersions = new HashSet<>(); + for (InetAddressAndPort endpoint : liveEndpoints) + { + CassandraVersion releaseVersion = Gossiper.instance.getReleaseVersion(endpoint); + if (releaseVersion != null) + { + majorVersions.add(releaseVersion.major); + } + } + return majorVersions.size() > 1; + } + @VisibleForTesting protected static TreeSet getHostIdsInCurrentRing(RepairType repairType, Set allNodesInRing) { diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java index db1b29c38968..463629f05861 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairService.java +++ b/src/java/org/apache/cassandra/service/AutoRepairService.java @@ -102,6 +102,7 @@ public String getAutoRepairConfiguration() appendConfig(sb, "repair_check_interval", config.getRepairCheckInterval()); appendConfig(sb, "repair_task_min_duration", config.getRepairTaskMinDuration()); appendConfig(sb, "history_clear_delete_hosts_buffer_interval", config.getAutoRepairHistoryClearDeleteHostsBufferInterval()); + appendConfig(sb, "mixed_major_version_repair_enabled", config.getMixedMajorVersionRepairEnabled()); for (RepairType repairType : RepairType.values()) { sb.append(formatRepairTypeConfig(repairType, config)); @@ -271,6 +272,12 @@ public void setAutoRepairRetryBackoff(String repairType, String interval) config.setRepairRetryBackoff(RepairType.parse(repairType), interval); } + @Override + public void setMixedMajorVersionRepairEnabled(boolean enabled) + { + config.setMixedMajorVersionRepairEnabled(enabled); + } + private String formatRepairTypeConfig(RepairType repairType, AutoRepairConfig config) { StringBuilder sb = new StringBuilder(); diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java index 181c6008f533..e4d554dd980d 100644 --- a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java @@ -74,4 +74,6 @@ public interface AutoRepairServiceMBean public void setAutoRepairMaxRetriesCount(String repairType, int retries); public void setAutoRepairRetryBackoff(String repairType, String interval); + + public void setMixedMajorVersionRepairEnabled(boolean enabled); } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 300bbc6c4efc..c92604fc256b 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2277,6 +2277,11 @@ public void setAutoRepairRetryBackoff(String repairType, String interval) { autoRepairProxy.setAutoRepairRetryBackoff(repairType, interval); } + + public void setMixedMajorVersionRepairEnabled(boolean enabled) + { + autoRepairProxy.setMixedMajorVersionRepairEnabled(enabled); + } } class ColumnFamilyStoreMBeanIterator implements Iterator> diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java index cc00cabd0633..376023c2d5f6 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java +++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java @@ -51,7 +51,7 @@ public class SetAutoRepairConfig extends NodeToolCmd "|allow_parallel_replica_repair|allow_parallel_repair_across_schedules" + "|materialized_view_repair_enabled|repair_max_retries" + "|repair_retry_backoff|repair_session_timeout|min_repair_task_duration" + - "|repair_by_keyspace|token_range_splitter.]", + "|repair_by_keyspace|mixed_major_version_repair_enabled|token_range_splitter.]", required = true) protected List args = new ArrayList<>(); @@ -92,6 +92,9 @@ public void execute(NodeProbe probe) case "min_repair_task_duration": probe.setAutoRepairMinRepairTaskDuration(paramVal); return; + case "mixed_major_version_repair_enabled": + probe.setMixedMajorVersionRepairEnabled(Boolean.parseBoolean(paramVal)); + return; default: // proceed to options that require --repair-type option break; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index ee841a62c3b1..13c5fd5462b7 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -18,6 +18,7 @@ package org.apache.cassandra.repair.autorepair; +import java.net.UnknownHostException; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -25,6 +26,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; + +import org.apache.cassandra.gms.*; + import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -36,7 +40,6 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.marshal.UUIDType; -import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; @@ -254,6 +257,81 @@ public void testGetHostIdsInCurrentRing_multiple_nodes() assertTrue(hosts.contains(hostId)); } + @Test + public void testHasMultipleLiveMajorVersionsWithSingleNode() + { + boolean result = AutoRepairUtils.hasMultipleLiveMajorVersions(); + assertFalse(result); + } + + @Test + public void testHasMultipleLiveMajorVersionsWithMultipleNodesOfSameVersion() throws UnknownHostException + { + Gossiper.instance.start(0); + Gossiper.instance.expireUpgradeFromVersion(); + + VersionedValue.VersionedValueFactory factory = new VersionedValue.VersionedValueFactory(null); + EndpointState es = new EndpointState((HeartBeatState) null); + es.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion("4.1")); + + Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.100"), es); + Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.100")); + + Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.200"), es); + Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.200")); + + + boolean result = AutoRepairUtils.hasMultipleLiveMajorVersions(); + assertFalse(result); + } + + + @Test + public void testHasMultipleLiveMajorVersionsWithMultipleNodesOfSameMajorVersionDifferentMinorVersions() throws UnknownHostException + { + Gossiper.instance.start(0); + Gossiper.instance.expireUpgradeFromVersion(); + + VersionedValue.VersionedValueFactory factory = new VersionedValue.VersionedValueFactory(null); + EndpointState es = new EndpointState((HeartBeatState) null); + es.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion("4.1")); + + Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.100"), es); + Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.100")); + + EndpointState es1 = new EndpointState((HeartBeatState) null); + es1.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion("4.2")); + + Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.200"), es1); + Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.200")); + + boolean result = AutoRepairUtils.hasMultipleLiveMajorVersions(); + assertFalse(result); + } + + @Test + public void testHasMultipleLiveMajorVersionsWithMultipleNodesOfDifferentMajorVersions() throws UnknownHostException + { + Gossiper.instance.start(0); + Gossiper.instance.expireUpgradeFromVersion(); + + VersionedValue.VersionedValueFactory factory = new VersionedValue.VersionedValueFactory(null); + EndpointState es = new EndpointState((HeartBeatState) null); + es.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion("4.1")); + + Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.100"), es); + Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.100")); + + EndpointState es1 = new EndpointState((HeartBeatState) null); + es1.addApplicationState(ApplicationState.RELEASE_VERSION, factory.releaseVersion("5.1")); + + Gossiper.instance.endpointStateMap.put(InetAddressAndPort.getByName("127.0.0.200"), es1); + Gossiper.instance.liveEndpoints.add(InetAddressAndPort.getByName("127.0.0.200")); + + boolean result = AutoRepairUtils.hasMultipleLiveMajorVersions(); + assertTrue(result); + } + @Test public void testGetHostWithLongestUnrepairTime() { diff --git a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java index 28686365a3cb..e57ec27acf34 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/SetAutoRepairConfigTest.java @@ -269,7 +269,8 @@ public static Collection testCases() forEachRepairType("ignore_dcs", "dc1,dc2", (type) -> verify(probe, times(1)).setAutoRepairIgnoreDCs(type.name(), ImmutableSet.of("dc1", "dc2"))), forEachRepairType("token_range_splitter.max_bytes_per_schedule", "500GiB", (type) -> verify(probe, times(1)).setAutoRepairTokenRangeSplitterParameter(type.name(), "max_bytes_per_schedule", "500GiB")), forEachRepairType("repair_max_retries", "3", (type) -> verify(probe, times(1)).setAutoRepairMaxRetriesCount(type.name(), 3)), - forEachRepairType("repair_retry_backoff", "60s", (type) -> verify(probe, times(1)).setAutoRepairRetryBackoff(type.name(), "60s")) + forEachRepairType("repair_retry_backoff", "60s", (type) -> verify(probe, times(1)).setAutoRepairRetryBackoff(type.name(), "60s")), + forEachRepairType("mixed_major_version_repair_enabled", "false", (type) -> verify(probe, times(1)).setMixedMajorVersionRepairEnabled(false)) ).flatMap(Function.identity()).collect(Collectors.toList()); } From 7345bed0871a4dd0c5b2f744380ceffef9e4c45f Mon Sep 17 00:00:00 2001 From: jaydeepkumar1984 Date: Sun, 27 Apr 2025 10:47:32 -0700 Subject: [PATCH 196/199] Improved observability in AutoRepair to report both expected vs. actual repair bytes and expected vs. actual keyspaces patch by Jaydeepkumar Chovatia; reviewed by Chris Lohfink for CASSANDRA-20581 --- CHANGES.txt | 1 + .../cassandra/pages/operating/metrics.adoc | 42 ++- .../org/apache/cassandra/config/Config.java | 3 - .../cassandra/metrics/AutoRepairMetrics.java | 33 ++ .../repair/autorepair/AutoRepair.java | 26 +- .../repair/autorepair/AutoRepairConfig.java | 8 +- .../repair/autorepair/AutoRepairState.java | 73 ++++- .../repair/autorepair/AutoRepairUtils.java | 216 +++++++++++++ .../FixedSplitTokenRangeSplitter.java | 20 +- .../repair/autorepair/KeyspaceRepairPlan.java | 40 ++- .../autorepair/PrioritizedRepairPlan.java | 32 +- .../repair/autorepair/RepairAssignment.java | 16 +- .../autorepair/RepairTokenRangeSplitter.java | 298 +++++------------- .../AutoRepairSchedulerStatsHelper.java | 277 ++++++++++++++++ .../AutoRepairSchedulerStatsNoVNodesTest.java | 54 ++++ .../AutoRepairSchedulerStatsVNodesTest.java | 54 ++++ .../AutoRepairStateFactoryTest.java | 8 +- .../autorepair/AutoRepairStateTest.java | 53 ++-- .../autorepair/AutoRepairUtilsTest.java | 6 +- .../FixedSplitTokenRangeSplitterHelper.java | 149 ++++++--- .../autorepair/PrioritizedRepairPlanTest.java | 25 +- .../RepairTokenRangeSplitterTest.java | 201 ++++++++++-- 22 files changed, 1226 insertions(+), 409 deletions(-) create mode 100644 test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsHelper.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsNoVNodesTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsVNodesTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 41951243a718..6e724426bbdb 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1.6 + * Improved observability in AutoRepair to report both expected vs. actual repair bytes and expected vs. actual keyspaces (CASSANDRA-20581) * Stop repair scheduler if two major versions are detected (CASSANDRA-20048) * AutoRepair: Safeguard Full repair against disk protection(CASSANDRA-20045) * Stop AutoRepair monitoring thread upon Cassandra shutdown (CASSANDRA-20623) diff --git a/doc/modules/cassandra/pages/operating/metrics.adoc b/doc/modules/cassandra/pages/operating/metrics.adoc index f39f1a503016..c27aa2df86cc 100644 --- a/doc/modules/cassandra/pages/operating/metrics.adoc +++ b/doc/modules/cassandra/pages/operating/metrics.adoc @@ -967,40 +967,58 @@ Reported name format: |=== |Name |Type |Description |RepairsInProgress |Gauge |Repair is in progress -on the node +on the node. |NodeRepairTimeInSec |Gauge |Time taken to repair -the node in seconds +the node in seconds. |ClusterRepairTimeInSec |Gauge |Time taken to repair -the entire Cassandra cluster in seconds +the entire Cassandra cluster in seconds. |LongestUnrepairedSec |Gauge |Time since the last repair -ran on the node in seconds +ran on the node in seconds. |RepairStartLagSec|Gauge |If a repair has not run within min_repair_interval, how long past this value since repairs last completed. Useful for determining if repairs are behind schedule. -|SucceededTokenRangesCount |Gauge |Number of token ranges successfully repaired on the node +|SucceededTokenRangesCount |Gauge |Number of token ranges successfully repaired on the node. -|FailedTokenRangesCount |Gauge |Number of token ranges failed to repair on the node +|FailedTokenRangesCount |Gauge |Number of token ranges failed to repair on the node. |SkippedTokenRangesCount |Gauge |Number of token ranges skipped -on the node +on the node. |SkippedTablesCount |Gauge |Number of tables skipped -on the node +on the node. |TotalMVTablesConsideredForRepair |Gauge |Number of materialized -views considered on the node +views considered on the node. |TotalDisabledRepairTables |Gauge |Number of tables on which -the automated repair has been disabled on the node +the automated repair has been disabled on the node. + +|TotalBytesToRepair |Gauge |Total bytes to be repaired across all keyspaces and tables involved in the current +repair schedule. + +|BytesAlreadyRepaired |Gauge |Cumulative number of bytes successfully repaired so far in the current +repair schedule. +NOTE: This calculation is the best effort for the FixedSplitTokenRangeSplitter. In practice, this metric +may not give you an accurate view in case of uneven data distribution. + + +|TotalKeyspaceRepairPlansToRepair |Gauge |Represents the total number of keyspace-level repair plans scheduled +for execution. If no table-level repair priorities are configured, this number typically matches the total number +of keyspaces under repair. However, if certain tables have repair priorities set, this number is usually higher than +the number of keyspaces, as multiple repair plans may be generated for different prioritized tables within the +same keyspace. + +|KeyspaceRepairPlansAlreadyRepaired |Gauge |Cumulative number of keyspace-level repair plans successfully +repaired so far in the current repair schedule. -|RepairTurnMyTurn |Counter |Represents the node's turn to repair +|RepairTurnMyTurn |Counter |Represents the node's turn to repair. |RepairTurnMyTurnDueToPriority |Counter |Represents the node's turn to repair -due to priority set in the automated repair +due to priority set in the automated repair. |RepairDelayedByReplica |Counter |Represents occurrences of a node's turn being delayed because a replica was currently taking its turn. Only relevant if diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index c8e3c69898c7..cd4c0fc6797d 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -331,9 +331,6 @@ public MemtableOptions() public volatile int concurrent_materialized_view_builders = 1; public volatile int reject_repair_compaction_threshold = Integer.MAX_VALUE; - // The number of executors to use for building secondary indexes - public volatile int concurrent_index_builders = 2; - // at least 20% of disk must be unused to run repair // if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason) then set the ratio to 0.0 @Replaces(oldName = "incremental_repair_disk_headroom_reject_ratio") diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java index 8efc8d679ab2..e1355dbfa6cd 100644 --- a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java @@ -44,6 +44,11 @@ public class AutoRepairMetrics public final Gauge skippedTablesCount; public final Gauge totalMVTablesConsideredForRepair; public final Gauge totalDisabledRepairTables; + public final Gauge totalBytesToRepair; + public final Gauge bytesAlreadyRepaired; + public final Gauge totalKeyspaceRepairPlansToRepair; + public final Gauge keyspaceRepairPlansAlreadyRepaired; + public Counter repairTurnMyTurn; public Counter repairTurnMyTurnDueToPriority; public Counter repairTurnMyTurnForceRepair; @@ -153,6 +158,34 @@ public Integer getValue() return AutoRepair.instance.getRepairState(repairType).getTotalDisabledTablesRepairCount(); } }); + totalBytesToRepair = Metrics.register(factory.createMetricName("TotalBytesToRepair"), new Gauge() + { + public Long getValue() + { + return AutoRepair.instance.getRepairState(repairType).getTotalBytesToRepair(); + } + }); + bytesAlreadyRepaired = Metrics.register(factory.createMetricName("BytesAlreadyRepaired"), new Gauge() + { + public Long getValue() + { + return AutoRepair.instance.getRepairState(repairType).getBytesAlreadyRepaired(); + } + }); + totalKeyspaceRepairPlansToRepair = Metrics.register(factory.createMetricName("TotalKeyspaceRepairPlansToRepair"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).getTotalKeyspaceRepairPlansToRepair(); + } + }); + keyspaceRepairPlansAlreadyRepaired = Metrics.register(factory.createMetricName("KeyspaceRepairPlansAlreadyRepaired"), new Gauge() + { + public Integer getValue() + { + return AutoRepair.instance.getRepairState(repairType).getKeyspaceRepairPlansAlreadyRepaired(); + } + }); } public void recordTurn(AutoRepairUtils.RepairTurn turn) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index 13804dbd1c59..c8ff80f2678a 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -123,14 +123,15 @@ public void setup() repairExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class); repairRunnableExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class); repairStates = new EnumMap<>(AutoRepairConfig.RepairType.class); + AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); + for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) { repairExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-Repair-" + repairType.getConfigName(), Thread.NORM_PRIORITY)); repairRunnableExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-RepairRunnable-" + repairType.getConfigName(), Thread.NORM_PRIORITY)); - repairStates.put(repairType, AutoRepairConfig.RepairType.getAutoRepairState(repairType)); + repairStates.put(repairType, AutoRepairConfig.RepairType.getAutoRepairState(repairType, config)); } - AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); AutoRepairUtils.setup(); for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values()) @@ -197,6 +198,8 @@ public void repair(AutoRepairConfig.RepairType repairType) if (turn == MY_TURN || turn == MY_TURN_DUE_TO_PRIORITY || turn == MY_TURN_FORCE_REPAIR) { repairState.recordTurn(turn); + repairState.setBytesAlreadyRepaired(0L); + repairState.setKeyspaceRepairPlansAlreadyRepaired(0); // For normal auto repair, we will use primary range only repairs (Repair with -pr option). // For some cases, we may set the auto_repair_primary_token_range_only flag to false then we will do repair // without -pr. We may also do force repair for certain node that we want to repair all the data on one node @@ -231,23 +234,30 @@ public void repair(AutoRepairConfig.RepairType repairType) } // Separate out the keyspaces and tables to repair based on their priority, with each repair plan representing a uniquely occuring priority. - List repairPlans = PrioritizedRepairPlan.build(keyspacesAndTablesToRepair, repairType, shuffleFunc); + List repairPlans = PrioritizedRepairPlan.build(keyspacesAndTablesToRepair, repairType, shuffleFunc, primaryRangeOnly); + repairState.updateRepairScheduleStatistics(repairPlans); // calculate the repair assignments for each priority:keyspace. Iterator repairAssignmentsIterator = config.getTokenRangeSplitterInstance(repairType).getRepairAssignments(primaryRangeOnly, repairPlans); + int keyspaceRepairAssignmentsAlreadyRepaired = 0; while (repairAssignmentsIterator.hasNext()) { KeyspaceRepairAssignments repairAssignments = repairAssignmentsIterator.next(); List assignments = repairAssignments.getRepairAssignments(); if (assignments.isEmpty()) { + keyspaceRepairAssignmentsAlreadyRepaired++; logger.info("Skipping repairs for priorityBucket={} for keyspace={} since it yielded no assignments", repairAssignments.getPriority(), repairAssignments.getKeyspaceName()); continue; } - logger.info("Submitting repairs for priorityBucket={} for keyspace={} with assignmentCount={}", repairAssignments.getPriority(), repairAssignments.getKeyspaceName(), repairAssignments.getRepairAssignments().size()); + logger.info("Submitting repairs for priorityBucket={} for keyspace={} with assignmentCount={} and keyspaceRepairAssignmentsAlreadyRepaired={}/{}", + repairAssignments.getPriority(), repairAssignments.getKeyspaceName(), repairAssignments.getRepairAssignments().size(), + keyspaceRepairAssignmentsAlreadyRepaired, repairState.getTotalKeyspaceRepairPlansToRepair()); repairKeyspace(repairType, primaryRangeOnly, repairAssignments.getKeyspaceName(), repairAssignments.getRepairAssignments(), collectedRepairStats); + keyspaceRepairAssignmentsAlreadyRepaired++; + repairState.setKeyspaceRepairPlansAlreadyRepaired(keyspaceRepairAssignmentsAlreadyRepaired); } cleanupAndUpdateStats(turn, repairType, repairState, myId, startTimeInMillis, collectedRepairStats); @@ -277,6 +287,7 @@ private void repairKeyspace(AutoRepairConfig.RepairType repairType, boolean prim long tableStartTime = timeFunc.get(); int totalProcessedAssignments = 0; Set> ranges = new HashSet<>(); + long bytesAlreadyRepaired = repairState.getBytesAlreadyRepaired(); for (RepairAssignment curRepairAssignment : repairAssignments) { try @@ -380,7 +391,10 @@ else if (retryCount < config.getRepairMaxRetries(repairType)) } ranges.clear(); } - logger.info("Repair completed for {} tables {}, range {}", keyspaceName, curRepairAssignment.getTableNames(), curRepairAssignment.getTokenRange()); + bytesAlreadyRepaired += curRepairAssignment.getEstimatedBytes(); + repairState.setBytesAlreadyRepaired(bytesAlreadyRepaired); + logger.info("Repair completed for {} tables {}, range {}, bytesAlreadyRepaired {}/{}", + keyspaceName, curRepairAssignment.getTableNames(), curRepairAssignment.getTokenRange(), bytesAlreadyRepaired, repairState.getTotalBytesToRepair()); } catch (Exception e) { @@ -492,8 +506,8 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType TimeUnit.SECONDS.toDays(repairState.getClusterRepairTimeInSec())); } repairState.setLastRepairTime(timeFunc.get()); - repairState.setRepairInProgress(false); + AutoRepairUtils.updateFinishAutoRepairHistory(repairType, myId, timeFunc.get()); } diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java index 2221c2f2def0..0998a4a37eb1 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java @@ -91,16 +91,16 @@ public String getConfigName() return configName; } - public static AutoRepairState getAutoRepairState(RepairType repairType) + public static AutoRepairState getAutoRepairState(RepairType repairType, AutoRepairConfig config) { switch (repairType) { case FULL: - return new FullRepairState(); + return new FullRepairState(config); case INCREMENTAL: - return new IncrementalRepairState(); + return new IncrementalRepairState(config); case PREVIEW_REPAIRED: - return new PreviewRepairedState(); + return new PreviewRepairedState(config); } throw new IllegalArgumentException("Invalid repair type: " + repairType); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index 01d2ca7ca38b..f6e33dcae96f 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -60,6 +60,8 @@ public abstract class AutoRepairState @VisibleForTesting protected final RepairType repairType; @VisibleForTesting + protected AutoRepairConfig config; + @VisibleForTesting protected int totalTablesConsideredForRepair = 0; @VisibleForTesting protected long lastRepairTimeInMs; @@ -84,13 +86,22 @@ public abstract class AutoRepairState @VisibleForTesting protected int skippedTablesCount = 0; @VisibleForTesting + protected long totalBytesToRepair = 0; + @VisibleForTesting + protected long bytesAlreadyRepaired = 0; + @VisibleForTesting + protected int totalKeyspaceRepairPlansToRepair = 0; + @VisibleForTesting + protected int keyspaceRepairPlansAlreadyRepaired = 0; + @VisibleForTesting protected AutoRepairHistory longestUnrepairedNode; protected final AutoRepairMetrics metrics; - protected AutoRepairState(RepairType repairType) + protected AutoRepairState(RepairType repairType, AutoRepairConfig config) { metrics = AutoRepairMetricsManager.getMetrics(repairType); this.repairType = repairType; + this.config = config; } public abstract RepairRunnable getRepairRunnable(String keyspace, List tables, Set> ranges, boolean primaryRangeOnly); @@ -101,6 +112,14 @@ protected RepairRunnable getRepairRunnable(String keyspace, RepairOption options options, keyspace); } + public void updateRepairScheduleStatistics(List repairPlans) + { + setTotalBytesToRepair(repairPlans.stream(). + flatMap(repairPlan -> repairPlan.getKeyspaceRepairPlans(). + stream()).mapToLong(KeyspaceRepairPlan::getEstimatedBytes).sum()); + setTotalKeyspaceRepairPlansToRepair(repairPlans.stream().mapToInt(repairPlan -> repairPlan.getKeyspaceRepairPlans().size()).sum()); + } + public long getLastRepairTime() { return lastRepairTimeInMs; @@ -239,13 +258,53 @@ public int getTotalDisabledTablesRepairCount() { return totalDisabledTablesRepairCount; } + + public void setTotalBytesToRepair(long totalBytesToRepair) + { + this.totalBytesToRepair = totalBytesToRepair; + } + + public long getTotalBytesToRepair() + { + return totalBytesToRepair; + } + + public void setBytesAlreadyRepaired(long bytesAlreadyRepaired) + { + this.bytesAlreadyRepaired = bytesAlreadyRepaired; + } + + public long getBytesAlreadyRepaired() + { + return bytesAlreadyRepaired; + } + + public void setTotalKeyspaceRepairPlansToRepair(int totalKeyspaceRepairPlansToRepair) + { + this.totalKeyspaceRepairPlansToRepair = totalKeyspaceRepairPlansToRepair; + } + + public int getTotalKeyspaceRepairPlansToRepair() + { + return totalKeyspaceRepairPlansToRepair; + } + + public void setKeyspaceRepairPlansAlreadyRepaired(int keyspaceRepairPlansAlreadyRepaired) + { + this.keyspaceRepairPlansAlreadyRepaired = keyspaceRepairPlansAlreadyRepaired; + } + + public int getKeyspaceRepairPlansAlreadyRepaired() + { + return keyspaceRepairPlansAlreadyRepaired; + } } class PreviewRepairedState extends AutoRepairState { - public PreviewRepairedState() + public PreviewRepairedState(AutoRepairConfig config) { - super(RepairType.PREVIEW_REPAIRED); + super(RepairType.PREVIEW_REPAIRED, config); } @Override @@ -263,9 +322,9 @@ public RepairRunnable getRepairRunnable(String keyspace, List tables, Se class IncrementalRepairState extends AutoRepairState { - public IncrementalRepairState() + public IncrementalRepairState(AutoRepairConfig config) { - super(RepairType.INCREMENTAL); + super(RepairType.INCREMENTAL, config); } @Override @@ -308,9 +367,9 @@ protected List filterOutUnsafeTables(String keyspaceName, List t class FullRepairState extends AutoRepairState { - public FullRepairState() + public FullRepairState(AutoRepairConfig config) { - super(RepairType.FULL); + super(RepairType.FULL, config); } @Override diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index fcb2d73d188a..8b9c93ab9c4d 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -17,6 +17,7 @@ */ package org.apache.cassandra.repair.autorepair; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -43,9 +44,19 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import com.clearspring.analytics.stream.cardinality.CardinalityMergeException; +import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; +import com.clearspring.analytics.stream.cardinality.ICardinality; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Splitter; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigTableScanner; +import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; +import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.locator.EndpointsByRange; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.LocalStrategy; @@ -89,6 +100,7 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; import org.apache.cassandra.utils.NoSpamLogger; +import org.apache.cassandra.utils.concurrent.Refs; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_DUE_TO_PRIORITY; @@ -1183,4 +1195,208 @@ public static Collection> split(Range tokenRange, int number } return ranges; } + + /** + * Finds a list of SSTables for a given {@code repairType}, + * {@code keyspace}, {@code table}, and {@code tokenRange} and then it internally calls + * another API {@code AutoRepairUtils.getSizesForRangeOfSSTables}, which figures out the estimated data size. + * + * @param repairType the repair type (e.g., FULL, INCREMENTAL) + * @param keyspace the keyspace name + * @param table the table name + * @param tokenRange the token range to evaluate + * @return an estimate representing the number of partitions, size in range, and total size + */ + static SizeEstimate getRangeSizeEstimate(RepairType repairType, String keyspace, String table, Range tokenRange) + { + logger.debug("Calculating size estimate for {}.{} for range {}", keyspace, table, tokenRange); + try (Refs refs = RepairTokenRangeSplitter.getSSTableReaderRefs(repairType, keyspace, table, tokenRange)) + { + SizeEstimate estimate = getSizesForRangeOfSSTables(repairType, keyspace, table, tokenRange, refs); + logger.debug("Generated size estimate {}", estimate); + return estimate; + } + } + /** + * Calculates the size estimation qualified to be repaired for a given {@code repairType}, + * {@code keyspace}, {@code table}, {@code tokenRange}, and {@code refs}. + *

        + * If the compression is enabled, then the size will be an estimate, otherwise it will be accurate. + *

        + * + * @param repairType + * @param keyspace + * @param table + * @param tokenRange + * @param refs + * @return an estimate representing the number of partitions, size in range, and total size + */ + static SizeEstimate getSizesForRangeOfSSTables(RepairType repairType, String keyspace, String table, + Range tokenRange, Refs refs) + { + ICardinality cardinality = new HyperLogLogPlus(13, 25); + long approxBytesInRange = 0L; + long totalBytes = 0L; + + for (SSTableReader reader : refs) + { + try + { + if (reader.openReason == SSTableReader.OpenReason.EARLY) + continue; + CompactionMetadata metadata = (CompactionMetadata) reader.descriptor.getMetadataSerializer().deserialize(reader.descriptor, MetadataType.COMPACTION); + if (metadata != null) + cardinality = cardinality.merge(metadata.cardinalityEstimator); + + long sstableSize = reader.bytesOnDisk(); + totalBytes += sstableSize; + // get the bounds of the sstable for this range using the index file but do not actually read it. + List> bounds = BigTableScanner.makeBounds(reader, Collections.singleton(tokenRange)); + + ISSTableScanner rangeScanner = BigTableScanner.getScanner(reader, Collections.singleton(tokenRange)); + // Type check scanner returned as it may be an EmptySSTableScanner if the range is not covered in the + // SSTable, in this case we will avoid incrementing approxBytesInRange. + if (rangeScanner instanceof BigTableScanner) + { + try (BigTableScanner scanner = (BigTableScanner) rangeScanner) + { + assert bounds.size() == 1; + + AbstractBounds bound = bounds.get(0); + long startPosition = scanner.getDataPosition(bound.left); + long endPosition = scanner.getDataPosition(bound.right); + // If end position is 0 we can assume the sstable ended before that token, bound at size of file + if (endPosition == 0) + { + endPosition = sstableSize; + } + + long approximateRangeBytesInSSTable = Math.max(0, endPosition - startPosition); + approxBytesInRange += Math.min(approximateRangeBytesInSSTable, sstableSize); + } + } + + } + catch (IOException | CardinalityMergeException e) + { + logger.error("Error calculating size estimate for {}.{} for range {} on {}", keyspace, table, tokenRange, reader, e); + } + } + double ratio = approxBytesInRange / (double) totalBytes; + // use the ratio from size to estimate the partitions in the range as well + long partitions = (long) Math.max(1, Math.ceil(cardinality.cardinality() * ratio)); + + return new SizeEstimate(repairType, keyspace, table, tokenRange, partitions, approxBytesInRange, totalBytes); + } + + /** + * Calculates the token ranges owned by this node for a given keyspace. + * + * @param primaryRangeOnly whether to use only primary token ranges or include replicated ones + * @param keyspaceName the name of the keyspace + * @return one or more token ranges owned by this node + */ + public static List> getTokenRanges(boolean primaryRangeOnly, String keyspaceName) + { + // Collect all applicable token ranges + Collection> wrappedRanges; + if (primaryRangeOnly) + { + wrappedRanges = StorageService.instance.getPrimaryRanges(keyspaceName); + } + else + { + wrappedRanges = StorageService.instance.getLocalRanges(keyspaceName); + } + + // Unwrap each range as we need to account for ranges that overlap the ring + List> ranges = new ArrayList<>(); + for (Range wrappedRange : wrappedRanges) + { + ranges.addAll(wrappedRange.unwrap()); + } + return ranges; + } + + /** + * Calculates the total bytes to be repaired for a given keyspace and list of tables. + * + * @param repairType the repair type (e.g., FULL, INCREMENTAL) + * @param keyspaceName the name of the keyspace + * @param tableNames the list of tables + * @return a key-value map where the key is {@code keyspaceName.tableName} and the value is the number of bytes + * to be repaired. + */ + public static Map, SizeEstimate>> calcTotalBytesToBeRepaired(RepairType repairType, String keyspaceName, List tableNames, List> tokenRanges) + { + Map, SizeEstimate>> ksTablesEstimatedBytes = new HashMap<>(); + for (String tableName : tableNames) + { + String ksTable = getKeyspaceTableName(keyspaceName, tableName); + ksTablesEstimatedBytes.computeIfAbsent(ksTable, k -> new HashMap<>()); + Map, SizeEstimate> tokenToSize = ksTablesEstimatedBytes.get(ksTable); + for (Range tokenRange : tokenRanges) + { + SizeEstimate tableAssignments = getRangeSizeEstimate(repairType, keyspaceName, tableName, tokenRange); + tokenToSize.put(tokenRange, tableAssignments); + } + } + return ksTablesEstimatedBytes; + } + + public static String getKeyspaceTableName(String keyspace, String table) + { + return keyspace + "." + table; + } + + /** + * Represents a size estimate by both bytes and partition count for a given keyspace and table for a token range. + */ + @VisibleForTesting + protected static class SizeEstimate + { + public final RepairType repairType; + public final String keyspace; + public final String table; + public final Range tokenRange; + public final long partitions; + public final long sizeInRange; + public final long totalSize; + /** + * Size to consider in the repair. For incremental repair, we want to consider the total size + * of the estimate as we have to factor in anticompacting the entire SSTable. + * For full repair, just use the size containing the range. + */ + public final long sizeForRepair; + + public SizeEstimate(RepairType repairType, + String keyspace, String table, Range tokenRange, + long partitions, long sizeInRange, long totalSize) + { + this.repairType = repairType; + this.keyspace = keyspace; + this.table = table; + this.tokenRange = tokenRange; + this.partitions = partitions; + this.sizeInRange = sizeInRange; + this.totalSize = totalSize; + + this.sizeForRepair = repairType == RepairType.INCREMENTAL ? totalSize : sizeInRange; + } + + @Override + public String toString() + { + return "SizeEstimate{" + + "repairType=" + repairType + + ", keyspace='" + keyspace + '\'' + + ", table='" + table + '\'' + + ", tokenRange=" + tokenRange + + ", partitions=" + partitions + + ", sizeInRange=" + sizeInRange + + ", totalSize=" + totalSize + + ", sizeForRepair=" + sizeForRepair + + '}'; + } + } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java index 3824c63b7e91..1b2c80b74271 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java @@ -32,7 +32,6 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.service.StorageService; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; @@ -96,13 +95,7 @@ private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(boolean primar String keyspaceName = repairPlan.getKeyspaceName(); List tableNames = repairPlan.getTableNames(); - Collection> tokens = StorageService.instance.getPrimaryRanges(keyspaceName); - if (!primaryRangeOnly) - { - // if we need to repair non-primary token ranges, then change the tokens accordingly - tokens = StorageService.instance.getLocalReplicas(keyspaceName).ranges(); - } - + Collection> tokens = AutoRepairUtils.getTokenRanges(primaryRangeOnly, keyspaceName); boolean byKeyspace = config.getRepairByKeyspace(repairType); // collect all token ranges. List> allRanges = new ArrayList<>(); @@ -115,10 +108,15 @@ private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(boolean primar if (byKeyspace) { + + // This calculation is the best effort for the FixedSplitTokenRangeSplitter. + // In practice, this metric may not give you an accurate view in case of uneven data distribution. + long totalBytes = repairPlan.getEstimatedBytes(); + long bytesPerRange = Math.max(1, totalBytes / splitsPerRange); for (Range splitRange : allRanges) { // add repair assignment for each range entire keyspace's tables - repairAssignments.add(new RepairAssignment(splitRange, keyspaceName, tableNames)); + repairAssignments.add(new RepairAssignment(splitRange, keyspaceName, tableNames, bytesPerRange)); } } else @@ -126,9 +124,11 @@ private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(boolean primar // add repair assignment per table for (String tableName : tableNames) { + long totalBytes = repairPlan.getTableEstimatedBytes(AutoRepairUtils.getKeyspaceTableName(keyspaceName, tableName)); + long bytesPerRange = Math.max(1, totalBytes / splitsPerRange); for (Range splitRange : allRanges) { - repairAssignments.add(new RepairAssignment(splitRange, keyspaceName, Collections.singletonList(tableName))); + repairAssignments.add(new RepairAssignment(splitRange, keyspaceName, Collections.singletonList(tableName), bytesPerRange)); } } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairPlan.java b/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairPlan.java index 3c13e3d80d08..750e56e9d2d3 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairPlan.java +++ b/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairPlan.java @@ -18,9 +18,16 @@ package org.apache.cassandra.repair.autorepair; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; + /** * Encapsulates an intent to repair the given keyspace's tables */ @@ -30,10 +37,14 @@ public class KeyspaceRepairPlan private final List tableNames; - public KeyspaceRepairPlan(String keyspaceName, List tableNames) + @VisibleForTesting + public Map, AutoRepairUtils.SizeEstimate>> ksTablesEstimatedBytes; + + public KeyspaceRepairPlan(String keyspaceName, List tableNames, Map, AutoRepairUtils.SizeEstimate>> ksTablesEstimatedBytes) { this.keyspaceName = keyspaceName; this.tableNames = tableNames; + this.ksTablesEstimatedBytes = ksTablesEstimatedBytes; } public String getKeyspaceName() @@ -46,18 +57,40 @@ public List getTableNames() return tableNames; } + public long getEstimatedBytes() + { + return ksTablesEstimatedBytes.values().stream() + .flatMap(tableMap -> tableMap.values().stream()) + .mapToLong(sizeEstimate -> sizeEstimate.sizeForRepair) + .sum(); + } + + public long getTableEstimatedBytes(String keyspaceTableName) + { + return ksTablesEstimatedBytes.getOrDefault(keyspaceTableName, + Collections.emptyMap()).values().stream().mapToLong(sizeEstimate -> sizeEstimate.sizeForRepair).sum(); + } + + public AutoRepairUtils.SizeEstimate getSizeEstimate(String keyspaceTableName, Range tokenRange) + { + return ksTablesEstimatedBytes == null ? null + : ksTablesEstimatedBytes.getOrDefault(keyspaceTableName, null) == null ? null + : ksTablesEstimatedBytes.get(keyspaceTableName).get(tokenRange); + } + @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; KeyspaceRepairPlan that = (KeyspaceRepairPlan) o; - return Objects.equals(keyspaceName, that.keyspaceName) && Objects.equals(tableNames, that.tableNames); + return Objects.equals(keyspaceName, that.keyspaceName) && Objects.equals(tableNames, that.tableNames) + && Objects.equals(ksTablesEstimatedBytes, that.ksTablesEstimatedBytes); } @Override public int hashCode() { - return Objects.hash(keyspaceName, tableNames); + return Objects.hash(keyspaceName, tableNames, ksTablesEstimatedBytes); } @Override @@ -66,6 +99,7 @@ public String toString() return "KeyspaceRepairPlan{" + "keyspaceName='" + keyspaceName + '\'' + ", tableNames=" + tableNames + + ", ksTablesEstimatedBytes=" + ksTablesEstimatedBytes + '}'; } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java index fbedb71b7c3c..4457ccdd348e 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java +++ b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java @@ -19,7 +19,6 @@ package org.apache.cassandra.repair.autorepair; import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -89,11 +88,11 @@ public String toString() * for their given priority. * * @param keyspacesToTableNames A mapping keyspace to table names - * @param repairType The repair type that is being executed - * @param orderFunc A function to order keyspace and tables in the returned plan. + * @param repairType The repair type that is being executed + * @param orderFunc A function to order keyspace and tables in the returned plan. * @return Ordered list of plan's by table priorities. */ - public static List build(Map> keyspacesToTableNames, AutoRepairConfig.RepairType repairType, Consumer> orderFunc) + public static List build(Map> keyspacesToTableNames, AutoRepairConfig.RepairType repairType, Consumer> orderFunc, boolean primaryRangeOnly) { // Build a map of priority -> (keyspace -> tables) Map>> plans = new HashMap<>(); @@ -124,31 +123,20 @@ public static List build(Map> keyspa List keyspaceNames = new ArrayList<>(keyspacesAndTables.keySet()); orderFunc.accept(keyspaceNames); - for(String keyspaceName : keyspaceNames) + for (String keyspaceName : keyspaceNames) { - List tableNames = keyspacesAndTables.get(keyspaceName); - orderFunc.accept(tableNames); - KeyspaceRepairPlan keyspaceRepairPlan = new KeyspaceRepairPlan(keyspaceName, new ArrayList<>(tableNames)); - keyspaceRepairPlans.add(keyspaceRepairPlan); + List tableNames = keyspacesAndTables.get(keyspaceName); + orderFunc.accept(tableNames); + KeyspaceRepairPlan keyspaceRepairPlan = + new KeyspaceRepairPlan(keyspaceName, new ArrayList<>(tableNames), + AutoRepairUtils.calcTotalBytesToBeRepaired(repairType, keyspaceName, tableNames, AutoRepairUtils.getTokenRanges(primaryRangeOnly, keyspaceName))); + keyspaceRepairPlans.add(keyspaceRepairPlan); } } return planList; } - /** - * Convenience method to build a repair plan for a single keyspace with tables. Primarily useful in testing. - * @param keyspaceName Keyspace to repair - * @param tableNames tables to repair for the given keyspace. - * @return Single repair plan. - */ - static List buildSingleKeyspacePlan(AutoRepairConfig.RepairType repairType, String keyspaceName, String ... tableNames) - { - Map> keyspaceMap = new HashMap<>(); - keyspaceMap.put(keyspaceName, Arrays.asList(tableNames)); - return build(keyspaceMap, repairType, (l) -> {}); - } - /** * @return The priority of the given table if defined, otherwise 0. */ diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java index 63f8fbed4426..6e07399aad7d 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java @@ -35,11 +35,14 @@ public class RepairAssignment final List tableNames; - public RepairAssignment(Range tokenRange, String keyspaceName, List tableNames) + protected final long estimatedBytes; + + public RepairAssignment(Range tokenRange, String keyspaceName, List tableNames, long estimatedBytes) { this.tokenRange = tokenRange; this.keyspaceName = keyspaceName; this.tableNames = tableNames; + this.estimatedBytes = estimatedBytes; } public Range getTokenRange() @@ -57,19 +60,25 @@ public List getTableNames() return tableNames; } + public long getEstimatedBytes() + { + return estimatedBytes; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RepairAssignment that = (RepairAssignment) o; - return Objects.equals(tokenRange, that.tokenRange) && Objects.equals(keyspaceName, that.keyspaceName) && Objects.equals(tableNames, that.tableNames); + return Objects.equals(tokenRange, that.tokenRange) && Objects.equals(keyspaceName, that.keyspaceName) + && Objects.equals(tableNames, that.tableNames) && Objects.equals(estimatedBytes, that.estimatedBytes); } @Override public int hashCode() { - return Objects.hash(tokenRange, keyspaceName, tableNames); + return Objects.hash(tokenRange, keyspaceName, tableNames, estimatedBytes); } @Override @@ -79,6 +88,7 @@ public String toString() "tokenRange=" + tokenRange + ", keyspaceName='" + keyspaceName + '\'' + ", tableNames=" + tableNames + + ", estimatedBytes=" + estimatedBytes + '}'; } } diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java index 2922f7f4044a..eea37b01ce61 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java +++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java @@ -18,7 +18,6 @@ package org.apache.cassandra.repair.autorepair; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -40,26 +39,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.clearspring.analytics.stream.cardinality.CardinalityMergeException; -import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; -import com.clearspring.analytics.stream.cardinality.ICardinality; import org.apache.cassandra.config.DataStorageSpec; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.lifecycle.SSTableIntervalTree; import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.db.lifecycle.View; -import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.big.BigTableScanner; -import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; -import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.service.AutoRepairService; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.concurrent.Refs; import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; @@ -226,7 +216,8 @@ public Iterator getRepairAssignments(boolean primaryR * A custom {@link RepairAssignmentIterator} that confines the number of repair assignments to * max_bytes_per_schedule. */ - private class BytesBasedRepairAssignmentIterator extends RepairAssignmentIterator { + private class BytesBasedRepairAssignmentIterator extends RepairAssignmentIterator + { private final boolean primaryRangeOnly; private long bytesSoFar = 0; @@ -248,14 +239,14 @@ protected KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repair return new KeyspaceRepairAssignments(priority, repairPlan.getKeyspaceName(), Collections.emptyList()); } - List> tokenRanges = getTokenRanges(primaryRangeOnly, repairPlan.getKeyspaceName()); + List> tokenRanges = AutoRepairUtils.getTokenRanges(primaryRangeOnly, repairPlan.getKeyspaceName()); // shuffle token ranges to unbias selection of ranges Collections.shuffle(tokenRanges); List repairAssignments = new ArrayList<>(); // Generate assignments for each range speparately for (Range tokenRange : tokenRanges) { - repairAssignments.addAll(getRepairAssignmentsForKeyspace(repairType, repairPlan.getKeyspaceName(), repairPlan.getTableNames(), tokenRange)); + repairAssignments.addAll(getRepairAssignmentsForKeyspace(repairType, repairPlan, tokenRange)); } FilteredRepairAssignments filteredRepairAssignments = filterRepairAssignments(priority, repairPlan.getKeyspaceName(), repairAssignments, bytesSoFar); @@ -265,7 +256,7 @@ protected KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repair } @VisibleForTesting - List getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType repairType, String keyspaceName, List tableNames, Range tokenRange) + List getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType repairType, KeyspaceRepairPlan repairPlan, Range tokenRange) { List repairAssignments = new ArrayList<>(); // this is used for batching minimal single assignment tables together @@ -275,12 +266,12 @@ List getRepairAssignmentsForKeyspace(AutoRepairConfig.Rep // If we can repair by keyspace, sort the tables by size so can batch the smallest ones together boolean repairByKeyspace = config.getRepairByKeyspace(repairType); - List tablesToProcess = tableNames; + List tablesToProcess = repairPlan.getTableNames(); if (repairByKeyspace) { - tablesToProcess = tableNames.stream().sorted((t1, t2) -> { - ColumnFamilyStore cfs1 = ColumnFamilyStore.getIfExists(keyspaceName, t1); - ColumnFamilyStore cfs2 = ColumnFamilyStore.getIfExists(keyspaceName, t2); + tablesToProcess = repairPlan.getTableNames().stream().sorted((t1, t2) -> { + ColumnFamilyStore cfs1 = ColumnFamilyStore.getIfExists(repairPlan.getKeyspaceName(), t1); + ColumnFamilyStore cfs2 = ColumnFamilyStore.getIfExists(repairPlan.getKeyspaceName(), t2); // If for whatever reason the CFS is not retrievable, we can assume it has been deleted, so give the // other cfs precedence. if (cfs1 == null) @@ -299,7 +290,7 @@ else if (cfs2 == null) for (String tableName : tablesToProcess) { - List tableAssignments = getRepairAssignmentsForTable(keyspaceName, tableName, tokenRange); + List tableAssignments = getRepairAssignmentsForTable(repairPlan, tableName, tokenRange); if (tableAssignments.isEmpty()) continue; @@ -345,8 +336,9 @@ else if (tableAssignments.size() == 1 && /** * Given a repair type and map of sized-based repair assignments, confine them by maxBytesPerSchedule. + * * @param repairAssignments the assignments to filter. - * @param bytesSoFar repair assignment bytes accumulated so far. + * @param bytesSoFar repair assignment bytes accumulated so far. * @return A list of repair assignments confined by maxBytesPerSchedule. */ @VisibleForTesting @@ -441,9 +433,9 @@ private String getBytesOfMaxBytesPerSchedule(long bytes) } /** + * @param repairAssignments The assignments to sum * @return The sum of {@link SizedRepairAssignment#getEstimatedBytes()} of all given * repairAssignments. - * @param repairAssignments The assignments to sum */ @VisibleForTesting protected static long getEstimatedBytes(List repairAssignments) @@ -481,10 +473,18 @@ static SizedRepairAssignment merge(List assignments) } @VisibleForTesting - protected List getRepairAssignmentsForTable(String keyspaceName, String tableName, Range tokenRange) + protected List getRepairAssignmentsForTable(KeyspaceRepairPlan repairPlan, String tableName, Range tokenRange) { - List sizeEstimates = getRangeSizeEstimate(keyspaceName, tableName, tokenRange); - return getRepairAssignments(sizeEstimates); + AutoRepairUtils.SizeEstimate sizeEstimate = repairPlan.getSizeEstimate(AutoRepairUtils.getKeyspaceTableName(repairPlan.getKeyspaceName(), tableName), tokenRange); + if (sizeEstimate == null) + { + // Ideally, it should have been cached already inside the KeyspaceRepairPlan, but incase it was not, + // then recalculating it. It is a bit expensive, but necessary for the repair + logger.warn("The size estimate for {}.{} range {} was not pre-calculated, calculating on-demand", + repairPlan.getKeyspaceName(), tableName, tokenRange); + sizeEstimate = AutoRepairUtils.getRangeSizeEstimate(repairType, repairPlan.getKeyspaceName(), tableName, tokenRange); + } + return getRepairAssignments(sizeEstimate); } private static void logSkippingTable(String keyspaceName, String tableName) @@ -493,82 +493,75 @@ private static void logSkippingTable(String keyspaceName, String tableName) } @VisibleForTesting - protected List getRepairAssignments(List sizeEstimates) + protected List getRepairAssignments(AutoRepairUtils.SizeEstimate estimate) { List repairAssignments = new ArrayList<>(); // since its possible for us to hit maxBytesPerSchedule before seeing all ranges, shuffle so there is chance // at least of hitting all the ranges _eventually_ for the worst case scenarios - Collections.shuffle(sizeEstimates); int totalExpectedSubRanges = 0; - for (SizeEstimate estimate : sizeEstimates) + if (estimate.sizeForRepair != 0) { - if (estimate.sizeForRepair != 0) + boolean needsSplitting = estimate.sizeForRepair > bytesPerAssignment.toBytes() || estimate.partitions > partitionsPerAssignment; + if (needsSplitting) { - boolean needsSplitting = estimate.sizeForRepair > bytesPerAssignment.toBytes() || estimate.partitions > partitionsPerAssignment; - if (needsSplitting) - { - totalExpectedSubRanges += calculateNumberOfSplits(estimate); - } + totalExpectedSubRanges += calculateNumberOfSplits(estimate); } } - for (SizeEstimate estimate : sizeEstimates) + if (estimate.sizeForRepair == 0) { - if (estimate.sizeForRepair == 0) - { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(estimate.keyspace, estimate.table); + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(estimate.keyspace, estimate.table); - if (cfs == null) - { - logSkippingTable(estimate.keyspace, estimate.table); - continue; - } + if (cfs == null) + { + logSkippingTable(estimate.keyspace, estimate.table); + return Collections.emptyList(); + } - long memtableSize = cfs.getTracker().getView().getCurrentMemtable().getLiveDataSize(); - if (memtableSize > 0L) - { - logger.debug("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", estimate.keyspace, estimate.table, estimate.tokenRange, memtableSize); - SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "full primary rangee for table with memtable only detected", memtableSize); - repairAssignments.add(assignment); - } - else - { - logger.debug("Included {}.{} range {}, has no SSTables or memtable data, but adding single repair assignment for entire range in case writes were missed", estimate.keyspace, estimate.table, estimate.tokenRange); - SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "full primary range for table with no data detected", 0L); - repairAssignments.add(assignment); - } + long memtableSize = cfs.getTracker().getView().getCurrentMemtable().getLiveDataSize(); + if (memtableSize > 0L) + { + logger.debug("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", estimate.keyspace, estimate.table, estimate.tokenRange, memtableSize); + SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "full primary rangee for table with memtable only detected", memtableSize); + repairAssignments.add(assignment); } else { - // Check if the estimate needs splitting based on the criteria - boolean needsSplitting = estimate.sizeForRepair > bytesPerAssignment.toBytes() || estimate.partitions > partitionsPerAssignment; - if (needsSplitting) - { - int numberOfSplits = calculateNumberOfSplits(estimate); - long approximateBytesPerSplit = estimate.sizeForRepair / numberOfSplits; - Collection> subranges = split(estimate.tokenRange, numberOfSplits); - for (Range subrange : subranges) - { - SizedRepairAssignment assignment = new SizedRepairAssignment(subrange, estimate.keyspace, Collections.singletonList(estimate.table), - String.format("subrange %d of %d", repairAssignments.size()+1, totalExpectedSubRanges), - approximateBytesPerSplit); - repairAssignments.add(assignment); - } - } - else + logger.debug("Included {}.{} range {}, has no SSTables or memtable data, but adding single repair assignment for entire range in case writes were missed", estimate.keyspace, estimate.table, estimate.tokenRange); + SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "full primary range for table with no data detected", 0L); + repairAssignments.add(assignment); + } + } + else + { + // Check if the estimate needs splitting based on the criteria + boolean needsSplitting = estimate.sizeForRepair > bytesPerAssignment.toBytes() || estimate.partitions > partitionsPerAssignment; + if (needsSplitting) + { + int numberOfSplits = calculateNumberOfSplits(estimate); + long approximateBytesPerSplit = estimate.sizeForRepair / numberOfSplits; + Collection> subranges = split(estimate.tokenRange, numberOfSplits); + for (Range subrange : subranges) { - // No splitting needed, repair the entire range as-is - SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, - Collections.singletonList(estimate.table), - "full primary range for table", estimate.sizeForRepair); + SizedRepairAssignment assignment = new SizedRepairAssignment(subrange, estimate.keyspace, Collections.singletonList(estimate.table), + String.format("subrange %d of %d", repairAssignments.size() + 1, totalExpectedSubRanges), + approximateBytesPerSplit); repairAssignments.add(assignment); } } + else + { + // No splitting needed, repair the entire range as-is + SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, + Collections.singletonList(estimate.table), + "full primary range for table", estimate.sizeForRepair); + repairAssignments.add(assignment); + } } return repairAssignments; } - private int calculateNumberOfSplits(SizeEstimate estimate) + private int calculateNumberOfSplits(AutoRepairUtils.SizeEstimate estimate) { // Calculate the number of splits needed for size and partitions int splitsForSize = (int) Math.ceil((double) estimate.sizeForRepair / bytesPerAssignment.toBytes()); @@ -594,98 +587,6 @@ private int calculateNumberOfSplits(SizeEstimate estimate) return splits; } - private List> getTokenRanges(boolean primaryRangeOnly, String keyspaceName) - { - // Collect all applicable token ranges - Collection> wrappedRanges; - if (primaryRangeOnly) - { - wrappedRanges = StorageService.instance.getPrimaryRanges(keyspaceName); - } - else - { - wrappedRanges = StorageService.instance.getLocalRanges(keyspaceName); - } - - // Unwrap each range as we need to account for ranges that overlap the ring - List> ranges = new ArrayList<>(); - for (Range wrappedRange : wrappedRanges) - { - ranges.addAll(wrappedRange.unwrap()); - } - return ranges; - } - - private List getRangeSizeEstimate(String keyspace, String table, Range tokenRange) - { - List sizeEstimates = new ArrayList<>(); - logger.debug("Calculating size estimate for {}.{} for range {}", keyspace, table, tokenRange); - try (Refs refs = getSSTableReaderRefs(repairType, keyspace, table, tokenRange)) - { - SizeEstimate estimate = getSizesForRangeOfSSTables(repairType, keyspace, table, tokenRange, refs); - logger.debug("Generated size estimate {}", estimate); - sizeEstimates.add(estimate); - } - return sizeEstimates; - } - - @VisibleForTesting - static SizeEstimate getSizesForRangeOfSSTables(AutoRepairConfig.RepairType repairType, String keyspace, String table, Range tokenRange, Refs refs) - { - ICardinality cardinality = new HyperLogLogPlus(13, 25); - long approxBytesInRange = 0L; - long totalBytes = 0L; - - for (SSTableReader reader : refs) - { - try - { - if (reader.openReason == SSTableReader.OpenReason.EARLY) - continue; - CompactionMetadata metadata = (CompactionMetadata) reader.descriptor.getMetadataSerializer().deserialize(reader.descriptor, MetadataType.COMPACTION); - if (metadata != null) - cardinality = cardinality.merge(metadata.cardinalityEstimator); - - long sstableSize = reader.bytesOnDisk(); - totalBytes += sstableSize; - // get the bounds of the sstable for this range using the index file but do not actually read it. - List> bounds = BigTableScanner.makeBounds(reader, Collections.singleton(tokenRange)); - - ISSTableScanner rangeScanner = BigTableScanner.getScanner(reader, Collections.singleton(tokenRange)); - // Type check scanner returned as it may be an EmptySSTableScanner if the range is not covered in the - // SSTable, in this case we will avoid incrementing approxBytesInRange. - if (rangeScanner instanceof BigTableScanner) - { - try (BigTableScanner scanner = (BigTableScanner) rangeScanner) - { - assert bounds.size() == 1; - - AbstractBounds bound = bounds.get(0); - long startPosition = scanner.getDataPosition(bound.left); - long endPosition = scanner.getDataPosition(bound.right); - // If end position is 0 we can assume the sstable ended before that token, bound at size of file - if (endPosition == 0) - { - endPosition = sstableSize; - } - - long approximateRangeBytesInSSTable = Math.max(0, endPosition - startPosition); - approxBytesInRange += Math.min(approximateRangeBytesInSSTable, sstableSize); - } - } - - } - catch (IOException | CardinalityMergeException e) - { - logger.error("Error calculating size estimate for {}.{} for range {} on {}", keyspace, table, tokenRange, reader, e); - } - } - double ratio = approxBytesInRange / (double) totalBytes; - // use the ratio from size to estimate the partitions in the range as well - long partitions = (long) Math.max(1, Math.ceil(cardinality.cardinality() * ratio)); - return new SizeEstimate(repairType, keyspace, table, tokenRange, partitions, approxBytesInRange, totalBytes); - } - @VisibleForTesting static Refs getSSTableReaderRefs(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName, Range tokenRange) { @@ -760,66 +661,15 @@ public Map getParameters() return Collections.unmodifiableMap(parameters); } - /** - * Represents a size estimate by both bytes and partition count for a given keyspace and table for a token range. - */ - @VisibleForTesting - protected static class SizeEstimate - { - public final AutoRepairConfig.RepairType repairType; - public final String keyspace; - public final String table; - public final Range tokenRange; - public final long partitions; - public final long sizeInRange; - public final long totalSize; - /** - * Size to consider in the repair. For incremental repair, we want to consider the total size - * of the estimate as we have to factor in anticompacting the entire SSTable. - * For full repair, just use the size containing the range. - */ - public final long sizeForRepair; - - public SizeEstimate(AutoRepairConfig.RepairType repairType, - String keyspace, String table, Range tokenRange, - long partitions, long sizeInRange, long totalSize) - { - this.repairType = repairType; - this.keyspace = keyspace; - this.table = table; - this.tokenRange = tokenRange; - this.partitions = partitions; - this.sizeInRange = sizeInRange; - this.totalSize = totalSize; - - this.sizeForRepair = repairType == AutoRepairConfig.RepairType.INCREMENTAL ? totalSize : sizeInRange; - } - - @Override - public String toString() - { - return "SizeEstimate{" + - "repairType=" + repairType + - ", keyspace='" + keyspace + '\'' + - ", table='" + table + '\'' + - ", tokenRange=" + tokenRange + - ", partitions=" + partitions + - ", sizeInRange=" + sizeInRange + - ", totalSize=" + totalSize + - ", sizeForRepair=" + sizeForRepair + - '}'; - } - } - /** * Implementation of RepairAssignment that also assigns an estimation of bytes involved * in the repair. */ @VisibleForTesting - protected static class SizedRepairAssignment extends RepairAssignment { + protected static class SizedRepairAssignment extends RepairAssignment + { final String description; - final long estimatedBytes; public SizedRepairAssignment(Range tokenRange, String keyspaceName, List tableNames) { @@ -830,9 +680,8 @@ public SizedRepairAssignment(Range tokenRange, String keyspaceName, List< String description, long estimatedBytes) { - super(tokenRange, keyspaceName, tableNames); + super(tokenRange, keyspaceName, tableNames, estimatedBytes); this.description = description; - this.estimatedBytes = estimatedBytes; } /** @@ -843,7 +692,8 @@ public String getDescription() { } /** - * Estimated bytes involved in the assignment. Typically Derived from {@link SizeEstimate#sizeForRepair}. + * Estimated bytes involved in the assignment. Typically Derived from {@link AutoRepairUtils.SizeEstimate#sizeForRepair}. + * * @return estimated bytes involved in the assignment. */ public long getEstimatedBytes() diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsHelper.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsHelper.java new file mode 100644 index 000000000000..adee92fe3c51 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsHelper.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.repair; + +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import com.google.common.collect.ImmutableMap; +import org.junit.Assert; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.TokenSupplier; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigTableScanner; +import org.apache.cassandra.metrics.AutoRepairMetrics; +import org.apache.cassandra.metrics.AutoRepairMetricsManager; +import org.apache.cassandra.repair.autorepair.AutoRepair; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils; +import org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitterTest; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SystemDistributedKeyspace; +import org.apache.cassandra.service.AutoRepairService; + +import static org.apache.cassandra.schema.SchemaConstants.DISTRIBUTED_KEYSPACE_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Helper class to test {@code totalBytesToRepair}, {@code bytesAlreadyRepaired}, {@code totalKeyspaceRepairPlansToRepair}, + * and {@code keyspaceRepairPlansAlreadyRepaired} + * for {@link org.apache.cassandra.repair.autorepair.AutoRepairState} scheduler + */ +public class AutoRepairSchedulerStatsHelper extends TestBaseImpl +{ + static final Logger logger = LoggerFactory.getLogger(AutoRepairSchedulerStatsHelper.class); + + private static Cluster cluster; + static SimpleDateFormat sdf; + private static final String KEYSPACE1 = "ks1"; + private static final String KEYSPACE2 = "ks2"; + private static final String TABLE1 = "tbl1"; + private static final String TABLE2 = "tbl2"; + + public static void init(int numTokens) throws IOException + { + // Define the expected date format pattern + String pattern = "EEE MMM dd HH:mm:ss z yyyy"; + // Create SimpleDateFormat object with the given pattern + sdf = new SimpleDateFormat(pattern); + sdf.setLenient(false); + CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); + cluster = Cluster.build(1) + .withTokenCount(numTokens) + .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(1, numTokens)) + .withConfig(config -> config + .set("num_tokens", numTokens) + .set("auto_repair", + ImmutableMap.of( + "repair_type_overrides", + ImmutableMap.of(AutoRepairConfig.RepairType.FULL.getConfigName(), + ImmutableMap.of( + "initial_scheduler_delay", "5s", + "enabled", "true", + "parallel_repair_count", "1", + // Allow parallel replica repair to allow replicas + // to execute full repair at same time. + "allow_parallel_replica_repair", "true", + // Set min_repair_interval to a higher number to + // run only one round of AutoRepair + "min_repair_interval", "48h")))) + .set("auto_repair.enabled", "true") + .set("auto_repair.global_settings.repair_retry_backoff", "5s") + .set("auto_repair.repair_task_min_duration", "0s") + .set("auto_repair.repair_check_interval", "5s")) + .start(); + + cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE1 + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};"); + cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE2 + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};"); + // disable the compression to calculate an accurate expected repair bytes because with compression enabled, + // we only get estimated bytes, which hinders the ability to do actual vs. expected checks in the test case + cluster.schemaChange(String.format("CREATE TABLE %s.%s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compression = { 'enabled' : false }", KEYSPACE1, TABLE1)); + cluster.schemaChange(String.format("CREATE TABLE %s.%s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compression = { 'enabled' : false }", KEYSPACE1, TABLE2)); + cluster.schemaChange(String.format("CREATE TABLE %s.%s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compression = { 'enabled' : false }", KEYSPACE2, TABLE1)); + cluster.schemaChange(String.format("CREATE TABLE %s.%s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compression = { 'enabled' : false }", KEYSPACE2, TABLE2)); + } + + public static void tearDown() + { + cluster.close(); + } + + public static void testSchedulerStats() throws ParseException + { + // ensure there was no history of previous repair runs through the scheduler + Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s", DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY), ConsistencyLevel.QUORUM); + assertEquals(0, rows.length); + + // disabling AutoRepair for system_distributed and system_auth tables to avoid + // interfering with the repaired bytes/plans calculation + disableAutoRepair(SystemDistributedKeyspace.NAME, new HashSet<>(RepairTokenRangeSplitterTest.SYSTEM_DISTRIBUTED_TABLE_NAMES)); + disableAutoRepair(SchemaConstants.AUTH_KEYSPACE_NAME, new HashSet<>(RepairTokenRangeSplitterTest.AUTH_TABLE_NAMES)); + + insertData(); + + cluster.get(1).runOnInstance(() -> { + try + { + AutoRepairService.setup(); + AutoRepair.instance.setup(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + }); + + cluster.forEach(i -> i.runOnInstance(() -> { + AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("2s"); + + AutoRepairMetrics fullMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.FULL); + // Since the AutoRepair sleeps up to SLEEP_IF_REPAIR_FINISHES_QUICKLY if the repair finishes quickly, + // so the "nodeRepairTimeInSec" metric should at least be greater than or equal to + // SLEEP_IF_REPAIR_FINISHES_QUICKLY + while (fullMetrics.nodeRepairTimeInSec.getValue().longValue() <= 1) + { + try + { + Thread.sleep(1000); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + logger.info("AutoRepair has completed one FULL repair cycle"); + + long expectedRepairBytes = calculateExpectedBytes(Arrays.asList(KEYSPACE1, KEYSPACE2)); + assertEquals(fullMetrics.totalKeyspaceRepairPlansToRepair.getValue(), fullMetrics.keyspaceRepairPlansAlreadyRepaired.getValue()); + // AutoRepair creates a repair plan per keyspace; + // Since there are two separate keyspaces, KEYSPACE1 and KEYSPACE2, the total expected plans should be "2" + assertEquals(2, fullMetrics.totalKeyspaceRepairPlansToRepair.getValue().intValue()); + assertEquals(fullMetrics.totalBytesToRepair.getValue().longValue(), fullMetrics.bytesAlreadyRepaired.getValue().longValue()); + assertEquals(expectedRepairBytes, fullMetrics.bytesAlreadyRepaired.getValue().longValue()); + })); + validate(AutoRepairConfig.RepairType.FULL.toString()); + } + + private static long calculateExpectedBytes(List keyspaces) + { + long totalBytes = 0; + for (String keyspace : keyspaces) + { + for (String table : Arrays.asList(TABLE1, TABLE2)) + { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, table); + assertNotNull(cfs); + Iterable sstables = cfs.getTracker().getView().select(SSTableSet.CANONICAL); + for (SSTableReader sstable : sstables) + { + long sstableSize = sstable.bytesOnDisk(); + List> tokenRanges = AutoRepairUtils.getTokenRanges(true, keyspace); + for (Range tokenRange : tokenRanges) + { + List> bounds = BigTableScanner.makeBounds(sstable, Collections.singleton(tokenRange)); + ISSTableScanner rangeScanner = BigTableScanner.getScanner(sstable, Collections.singleton(tokenRange)); + // Type check scanner returned as it may be an EmptySSTableScanner if the range is not covered in the + // SSTable, in this case we will avoid incrementing approxBytesInRange. + if (rangeScanner instanceof BigTableScanner) + { + try (BigTableScanner scanner = (BigTableScanner) rangeScanner) + { + AbstractBounds bound = bounds.get(0); + long startPosition = scanner.getDataPosition(bound.left); + long endPosition = scanner.getDataPosition(bound.right); + // If end position is 0 we can assume the sstable ended before that token, bound at size of file + if (endPosition == 0) + { + endPosition = sstableSize; + } + + long approximateRangeBytesInSSTable = Math.max(0, endPosition - startPosition); + totalBytes += Math.min(approximateRangeBytesInSSTable, sstableSize); + } + } + } + } + } + } + return totalBytes; + } + + private static void insertData() + { + for (int i = 0; i < 100; i++) + { + cluster.coordinator(1).execute(String.format("INSERT INTO %s.%s (pk, ck, v) VALUES (?,?,?)", KEYSPACE1, TABLE1), + ConsistencyLevel.ONE, i, i, i); + cluster.coordinator(1).execute(String.format("INSERT INTO %s.%s (pk, ck, v) VALUES (?,?,?)", KEYSPACE1, TABLE2), + ConsistencyLevel.ONE, i, i, i); + cluster.coordinator(1).execute(String.format("INSERT INTO %s.%s (pk, ck, v) VALUES (?,?,?)", KEYSPACE2, TABLE1), + ConsistencyLevel.ONE, i, i, i); + cluster.coordinator(1).execute(String.format("INSERT INTO %s.%s (pk, ck, v) VALUES (?,?,?)", KEYSPACE2, TABLE2), + ConsistencyLevel.ONE, i, i, i); + } + cluster.get(1).nodetool("flush", KEYSPACE1, TABLE1); + cluster.get(1).nodetool("flush", KEYSPACE1, TABLE2); + cluster.get(1).nodetool("flush", KEYSPACE2, TABLE1); + cluster.get(1).nodetool("flush", KEYSPACE2, TABLE2); + } + + private static void disableAutoRepair(String keyspaceName, Set distributedSystemTables) + { + for (String tableName : distributedSystemTables) + { + cluster.coordinator(1).execute(String.format("ALTER TABLE %s.%s WITH auto_repair = {'full_enabled': 'false'}", keyspaceName, tableName), + ConsistencyLevel.ONE); + } + } + + private static void validate(String repairType) throws ParseException + { + Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s where repair_type='%s'", DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType), ConsistencyLevel.QUORUM); + assertEquals(1, rows.length); + for (int node = 0; node < rows.length; node++) + { + Object[] row = rows[node]; + // repair_type + Assert.assertEquals(repairType, row[0].toString()); + // host_id + Assert.assertNotNull(UUID.fromString(row[1].toString())); + // ensure there is a legit repair_start_ts and repair_finish_ts + sdf.parse(row[2].toString()); + sdf.parse(row[3].toString()); + // the reason why the repair was scheduled + Assert.assertNotNull(row[4]); + Assert.assertEquals("MY_TURN", row[4].toString()); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsNoVNodesTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsNoVNodesTest.java new file mode 100644 index 000000000000..96376095b903 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsNoVNodesTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.repair; + +import java.io.IOException; +import java.text.ParseException; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.distributed.test.TestBaseImpl; + +/** + * Test {@code totalBytesToRepair}, {@code bytesAlreadyRepaired}, {@code totalKeyspaceRepairPlansToRepair}, + * and {@code keyspaceRepairPlansAlreadyRepaired} + * for {@link org.apache.cassandra.repair.autorepair.AutoRepairState} scheduler without v-nodes + */ +public class AutoRepairSchedulerStatsNoVNodesTest extends TestBaseImpl +{ + @BeforeClass + public static void init() throws IOException + { + AutoRepairSchedulerStatsHelper.init(1); + } + + @AfterClass + public static void tearDown() + { + AutoRepairSchedulerStatsHelper.tearDown(); + } + + @Test + public void testSchedulerStats() throws ParseException + { + AutoRepairSchedulerStatsHelper.testSchedulerStats(); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsVNodesTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsVNodesTest.java new file mode 100644 index 000000000000..8040a3032803 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsVNodesTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.repair; + +import java.io.IOException; +import java.text.ParseException; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.distributed.test.TestBaseImpl; + +/** + * Test {@code totalBytesToRepair}, {@code bytesAlreadyRepaired}, {@code totalKeyspaceRepairPlansToRepair}, + * and {@code keyspaceRepairPlansAlreadyRepaired} + * for {@link org.apache.cassandra.repair.autorepair.AutoRepairState} scheduler with v-nodes + */ +public class AutoRepairSchedulerStatsVNodesTest extends TestBaseImpl +{ + @BeforeClass + public static void init() throws IOException + { + AutoRepairSchedulerStatsHelper.init(16); + } + + @AfterClass + public static void tearDown() + { + AutoRepairSchedulerStatsHelper.tearDown(); + } + + @Test + public void testSchedulerStats() throws ParseException + { + AutoRepairSchedulerStatsHelper.testSchedulerStats(); + } +} diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java index 97e80364eed7..91c2c39ee4e7 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateFactoryTest.java @@ -34,15 +34,15 @@ public class AutoRepairStateFactoryTest @Test public void testGetRepairState() { - AutoRepairState state = RepairType.getAutoRepairState(RepairType.FULL); + AutoRepairState state = RepairType.getAutoRepairState(RepairType.FULL, new AutoRepairConfig()); assertTrue(state instanceof FullRepairState); - state = RepairType.getAutoRepairState(RepairType.INCREMENTAL); + state = RepairType.getAutoRepairState(RepairType.INCREMENTAL, new AutoRepairConfig()); assertTrue(state instanceof IncrementalRepairState); - state = RepairType.getAutoRepairState(RepairType.PREVIEW_REPAIRED); + state = RepairType.getAutoRepairState(RepairType.PREVIEW_REPAIRED, new AutoRepairConfig()); assertTrue(state instanceof PreviewRepairedState); } @@ -54,7 +54,7 @@ public void testGetRepairStateSupportsAllRepairTypes() { try { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); assertNotNull(state); } catch (IllegalArgumentException e) { diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java index 422ebdff5192..240a2db46fc9 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -34,8 +34,6 @@ import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; import org.apache.cassandra.repair.autorepair.AutoRepairUtils.AutoRepairHistory; import org.apache.cassandra.service.AutoRepairService; -import org.apache.cassandra.utils.progress.ProgressEvent; -import org.mockito.Mock; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -54,9 +52,6 @@ public class AutoRepairStateTest extends CQLTester @Parameterized.Parameter public RepairType repairType; - @Mock - ProgressEvent progressEvent; - @Parameterized.Parameters public static Collection repairTypes() { @@ -74,7 +69,7 @@ public void setUp() @Test public void testGetRepairRunnable() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); AutoRepairService.setup(); Runnable runnable = state.getRepairRunnable(KEYSPACE, ImmutableList.of(testTable), ImmutableSet.of(), false); @@ -85,7 +80,7 @@ public void testGetRepairRunnable() @Test public void testGetLastRepairTime() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.lastRepairTimeInMs = 1; assertEquals(1, state.getLastRepairTime()); @@ -94,7 +89,7 @@ public void testGetLastRepairTime() @Test public void testSetTotalTablesConsideredForRepair() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.setTotalTablesConsideredForRepair(1); @@ -104,7 +99,7 @@ public void testSetTotalTablesConsideredForRepair() @Test public void testGetTotalTablesConsideredForRepair() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.totalTablesConsideredForRepair = 1; assertEquals(1, state.getTotalTablesConsideredForRepair()); @@ -113,7 +108,7 @@ public void testGetTotalTablesConsideredForRepair() @Test public void testSetLastRepairTimeInMs() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.setLastRepairTime(1); @@ -123,7 +118,7 @@ public void testSetLastRepairTimeInMs() @Test public void testGetClusterRepairTimeInSec() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.clusterRepairTimeInSec = 1; assertEquals(1, state.getClusterRepairTimeInSec()); @@ -132,7 +127,7 @@ public void testGetClusterRepairTimeInSec() @Test public void testGetNodeRepairTimeInSec() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.nodeRepairTimeInSec = 1; assertEquals(1, state.getNodeRepairTimeInSec()); @@ -141,7 +136,7 @@ public void testGetNodeRepairTimeInSec() @Test public void testSetRepairInProgress() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.setRepairInProgress(true); @@ -151,7 +146,7 @@ public void testSetRepairInProgress() @Test public void testIsRepairInProgress() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.repairInProgress = true; assertTrue(state.isRepairInProgress()); @@ -160,7 +155,7 @@ public void testIsRepairInProgress() @Test public void testSetSkippedTokenRangesCount() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.setSkippedTokenRangesCount(1); @@ -170,7 +165,7 @@ public void testSetSkippedTokenRangesCount() @Test public void testGetSkippedTokenRangesCount() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.skippedTokenRangesCount = 1; assertEquals(1, state.getSkippedTokenRangesCount()); @@ -179,7 +174,7 @@ public void testGetSkippedTokenRangesCount() @Test public void testGetLongestUnrepairedSecNull() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.longestUnrepairedNode = null; try @@ -195,7 +190,7 @@ public void testGetLongestUnrepairedSecNull() @Test public void testGetLongestUnrepairedSec() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.longestUnrepairedNode = new AutoRepairHistory(UUID.randomUUID(), "", 0, 1000, null, 0, false); AutoRepairState.timeFunc = () -> 2000L; @@ -213,7 +208,7 @@ public void testGetLongestUnrepairedSec() @Test public void testSetTotalMVTablesConsideredForRepair() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.setTotalMVTablesConsideredForRepair(1); @@ -223,7 +218,7 @@ public void testSetTotalMVTablesConsideredForRepair() @Test public void testGetTotalMVTablesConsideredForRepair() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.totalMVTablesConsideredForRepair = 1; assertEquals(1, state.getTotalMVTablesConsideredForRepair()); @@ -232,7 +227,7 @@ public void testGetTotalMVTablesConsideredForRepair() @Test public void testSetNodeRepairTimeInSec() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.setNodeRepairTimeInSec(1); @@ -242,7 +237,7 @@ public void testSetNodeRepairTimeInSec() @Test public void testSetClusterRepairTimeInSec() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.setClusterRepairTimeInSec(1); @@ -252,7 +247,7 @@ public void testSetClusterRepairTimeInSec() @Test public void testSetRepairKeyspaceCount() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.setRepairKeyspaceCount(1); @@ -262,7 +257,7 @@ public void testSetRepairKeyspaceCount() @Test public void testGetRepairKeyspaceCount() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.repairKeyspaceCount = 1; assertEquals(1, state.getRepairKeyspaceCount()); @@ -271,7 +266,7 @@ public void testGetRepairKeyspaceCount() @Test public void testSetLongestUnrepairedNode() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); AutoRepairHistory history = new AutoRepairHistory(UUID.randomUUID(), "", 0, 0, null, 0, false); state.setLongestUnrepairedNode(history); @@ -282,7 +277,7 @@ public void testSetLongestUnrepairedNode() @Test public void testSetSucceededTokenRangesCount() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.setSucceededTokenRangesCount(1); @@ -292,7 +287,7 @@ public void testSetSucceededTokenRangesCount() @Test public void testGetSucceededTokenRangesCount() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.succeededTokenRangesCount = 1; assertEquals(1, state.getSucceededTokenRangesCount()); @@ -301,7 +296,7 @@ public void testGetSucceededTokenRangesCount() @Test public void testSetFailedTokenRangesCount() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.setFailedTokenRangesCount(1); @@ -311,7 +306,7 @@ public void testSetFailedTokenRangesCount() @Test public void testGetFailedTokenRangesCount() { - AutoRepairState state = RepairType.getAutoRepairState(repairType); + AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); state.failedTokenRangesCount = 1; assertEquals(1, state.getFailedTokenRangesCount()); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index 13c5fd5462b7..7f8bd524df21 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -27,7 +27,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.apache.cassandra.gms.*; +import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.gms.HeartBeatState; +import org.apache.cassandra.gms.VersionedValue; import org.junit.Before; import org.junit.BeforeClass; diff --git a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java index e0198c58bde0..68edb3aae3b0 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitterHelper.java @@ -22,8 +22,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -38,10 +40,13 @@ import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; import static org.apache.cassandra.Util.setAutoRepairEnabled; import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; import static org.apache.cassandra.repair.autorepair.FixedSplitTokenRangeSplitter.DEFAULT_NUMBER_OF_SUBRANGES; +import static org.apache.cassandra.repair.autorepair.FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -56,6 +61,8 @@ public class FixedSplitTokenRangeSplitterHelper extends CQLTester private static final String TABLE2 = "tbl2"; private static final String TABLE3 = "tbl3"; public static final String KEYSPACE = "ks"; + public static final List tables = Arrays.asList(TABLE1, TABLE2, TABLE3); + public static final Map, AutoRepairUtils.SizeEstimate>> ksTablesEstimatedBytes = new HashMap<>(); public static void setupClass(int numTokens) throws Exception { @@ -70,35 +77,59 @@ public static void setupClass(int numTokens) throws Exception TokenMetadata metadata = StorageService.instance.getTokenMetadata(); metadata.updateNormalTokens(BootStrapper.getRandomTokens(metadata, numTokens), FBUtilities.getBroadcastAddressAndPort()); + + Pair>, Integer> tokensAndWrappedAroundCount = getTokenRangesAndTotalWrapAroundCount(); + int totalToken = numTokens + tokensAndWrappedAroundCount.right(); + long perTokenSizeTable1 = 512L / totalToken; + long perTokenSizeTable2 = 1024L / totalToken; + long perTokenSizeTable3 = 2048L / totalToken; + for (Range tokenRange : tokensAndWrappedAroundCount.left) + { + ksTablesEstimatedBytes.put(AutoRepairUtils.getKeyspaceTableName(KEYSPACE, TABLE1), new HashMap, AutoRepairUtils.SizeEstimate>() + {{ + put(tokenRange, new AutoRepairUtils.SizeEstimate(AutoRepairConfig.RepairType.FULL, "", "", tokenRange, 0, perTokenSizeTable1, perTokenSizeTable1)); + }}); + ksTablesEstimatedBytes.put(AutoRepairUtils.getKeyspaceTableName(KEYSPACE, TABLE2), new HashMap, AutoRepairUtils.SizeEstimate>() + {{ + put(tokenRange, new AutoRepairUtils.SizeEstimate(AutoRepairConfig.RepairType.FULL, "", "", tokenRange, 0, perTokenSizeTable2, perTokenSizeTable2)); + }}); + ksTablesEstimatedBytes.put(AutoRepairUtils.getKeyspaceTableName(KEYSPACE, TABLE3), new HashMap, AutoRepairUtils.SizeEstimate>() + {{ + put(tokenRange, new AutoRepairUtils.SizeEstimate(AutoRepairConfig.RepairType.FULL, "", "", tokenRange, 0, perTokenSizeTable3, perTokenSizeTable3)); + }}); + } } public static void testTokenRangesSplitByTable(int numTokens, int numberOfSubRanges, AutoRepairConfig.RepairType repairType) { - int numberOfSplits = calcSplits(numTokens, numberOfSubRanges); - AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(repairType, false); - Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setRepairByKeyspace(repairType, false); + Pair>, Integer> tokensAndWrappedAroundCount = getTokenRangesAndTotalWrapAroundCount(); + Collection> tokens = tokensAndWrappedAroundCount.left(); + // For the test case, the tokens are allocated dynamically, so we do not know which token-ranges wrap around. + // As a result, we need to adjust the token count on a need basis. + numTokens += tokensAndWrappedAroundCount.right(); assertEquals(numTokens, tokens.size()); - List tables = Arrays.asList(TABLE1, TABLE2, TABLE3); List> expectedToken = new ArrayList<>(); - for (int i = 0; i range : tokens) + for (Range token : tokens) { - expectedToken.addAll(AutoRepairUtils.split(range, numberOfSplits)); + expectedToken.addAll(split(token, numberOfSplits)); } } - List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); - - Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRanges))) - .getRepairAssignments(true, plan); + Iterator keyspaceAssignments = + new FixedSplitTokenRangeSplitter(repairType, Collections.singletonMap(NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRanges))) + .getRepairAssignments(config.getRepairPrimaryTokenRangeOnly(repairType), getPlan(repairType)); // should be only 1 entry for the keyspace. assertTrue(keyspaceAssignments.hasNext()); - KeyspaceRepairAssignments keyspace = keyspaceAssignments.next(); + KeyspaceRepairAssignments keyspaceRepairAssignment = keyspaceAssignments.next(); assertFalse(keyspaceAssignments.hasNext()); - List assignments = keyspace.getRepairAssignments(); + List assignments = keyspaceRepairAssignment.getRepairAssignments(); assertEquals(numTokens * numberOfSplits * tables.size(), assignments.size()); assertEquals(expectedToken.size(), assignments.size()); @@ -109,9 +140,12 @@ public static void testTokenRangesSplitByTable(int numTokens, int numberOfSubRan List> expectedTokensForATable = new ArrayList<>(); for (int j = 0; j < assignmentsPerTable; j++) { - assertEquals(Collections.singletonList(tables.get(i)), assignments.get(i * assignmentsPerTable + j).getTableNames()); - assignmentForATable.add(assignments.get(i * assignmentsPerTable + j)); - expectedTokensForATable.add(expectedToken.get(i * assignmentsPerTable + j)); + long expectedBytes = ksTablesEstimatedBytes.get(AutoRepairUtils.getKeyspaceTableName(KEYSPACE, tables.get(i))).values().stream().mapToLong(sizeEstimate -> sizeEstimate.sizeForRepair).sum() / numberOfSplits; + int theTableAssignmentIdx = i * assignmentsPerTable + j; + assertEquals(expectedBytes, assignments.get(theTableAssignmentIdx).estimatedBytes); + assertEquals(Collections.singletonList(tables.get(i)), assignments.get(theTableAssignmentIdx).getTableNames()); + assignmentForATable.add(assignments.get(theTableAssignmentIdx)); + expectedTokensForATable.add(expectedToken.get(theTableAssignmentIdx)); } compare(numTokens, numberOfSplits, expectedTokensForATable, assignmentForATable); } @@ -119,20 +153,24 @@ public static void testTokenRangesSplitByTable(int numTokens, int numberOfSubRan public static void testTokenRangesSplitByKeyspace(int numTokens, int numberOfSubRanges, AutoRepairConfig.RepairType repairType) { - int numberOfSplits = calcSplits(numTokens, numberOfSubRanges); - AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(repairType, true); - Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + config.setRepairByKeyspace(repairType, true); + Pair>, Integer> tokensAndWrappedRanges = getTokenRangesAndTotalWrapAroundCount(); + Collection> tokens = tokensAndWrappedRanges.left(); + // For the test case, the tokens are allocated dynamically, so we do not know which token-ranges wrap around. + // As a result, we need to adjust the token count on a need basis. + numTokens += tokensAndWrappedRanges.right(); assertEquals(numTokens, tokens.size()); + int numberOfSplits = Math.max(1, numberOfSubRanges / tokens.size()); List> expectedToken = new ArrayList<>(); for (Range range : tokens) { expectedToken.addAll(AutoRepairUtils.split(range, numberOfSplits)); } - List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1, TABLE2, TABLE3); - - Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.singletonMap(FixedSplitTokenRangeSplitter.NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRanges))) - .getRepairAssignments(true, plan); + Iterator keyspaceAssignments = + new FixedSplitTokenRangeSplitter(repairType, Collections.singletonMap(NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubRanges))) + .getRepairAssignments(config.getRepairPrimaryTokenRangeOnly(repairType), getPlan(repairType)); // should be only 1 entry for the keyspace. assertTrue(keyspaceAssignments.hasNext()); @@ -146,35 +184,20 @@ public static void testTokenRangesSplitByKeyspace(int numTokens, int numberOfSub assertEquals(expectedToken.size(), assignments.size()); compare(numTokens, numberOfSplits, expectedToken, assignments); - } - public static void testTokenRangesWithDefaultSplit(int numTokens, AutoRepairConfig.RepairType repairType) - { - int numberOfSplits = calcSplits(numTokens, DEFAULT_NUMBER_OF_SUBRANGES); - Collection> tokens = StorageService.instance.getPrimaryRanges(KEYSPACE); - assertEquals(numTokens, tokens.size()); - List> expectedToken = new ArrayList<>(); - for (Range range : tokens) + for (int i = 0; i < assignments.size(); i++) { - expectedToken.addAll(AutoRepairUtils.split(range, numberOfSplits)); + assertEquals(assignments.get(i).estimatedBytes, + ksTablesEstimatedBytes.values().stream() + .flatMap(tableMap -> tableMap.values().stream()) + .mapToLong(sizeEstimate -> sizeEstimate.sizeForRepair) + .sum() / numberOfSplits); } + } - List plan = PrioritizedRepairPlan.buildSingleKeyspacePlan(repairType, KEYSPACE, TABLE1); - - Iterator keyspaceAssignments = new FixedSplitTokenRangeSplitter(repairType, Collections.emptyMap()).getRepairAssignments(true, plan); - - // should be only 1 entry for the keyspace. - assertTrue(keyspaceAssignments.hasNext()); - KeyspaceRepairAssignments keyspace = keyspaceAssignments.next(); - assertFalse(keyspaceAssignments.hasNext()); - - List assignments = keyspace.getRepairAssignments(); - assertNotNull(assignments); - - // should be 3 entries for the table which covers each token range. - assertEquals(numTokens * numberOfSplits, assignments.size()); - - compare(numTokens, numberOfSplits, expectedToken, assignments); + public static void testTokenRangesWithDefaultSplit(int numTokens, AutoRepairConfig.RepairType repairType) + { + testTokenRangesSplitByKeyspace(numTokens, DEFAULT_NUMBER_OF_SUBRANGES, repairType); } private static void compare(int numTokens, int numberOfSplits, List> expectedToken, List assignments) @@ -190,8 +213,34 @@ private static void compare(int numTokens, int numberOfSplits, List assertEquals(a, b); } - private static int calcSplits(int numTokens, int subRange) + private static Pair>, Integer> getTokenRangesAndTotalWrapAroundCount() + { + int wrappedRanges = 0; + Collection> ranges = StorageService.instance.getPrimaryRangesForEndpoint(KEYSPACE, FBUtilities.getBroadcastAddressAndPort()); + Collection> tokens = new ArrayList<>(); + for (Range wrappedRange : ranges) + { + if (wrappedRange.isWrapAround()) + { + wrappedRanges++; + } + tokens.addAll(wrappedRange.unwrap()); + } + return Pair.create(tokens, wrappedRanges); + } + + private static List getPlan(AutoRepairConfig.RepairType repairType) { - return Math.max(1, subRange / numTokens); + AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); + List plan = PrioritizedRepairPlan.build(new HashMap>() + {{ + put(KEYSPACE, tables); + }}, repairType, (l) -> { + }, + config.getRepairPrimaryTokenRangeOnly(repairType)); + assertEquals(1, plan.size()); + assertEquals(1, plan.get(0).getKeyspaceRepairPlans().size()); + plan.get(0).getKeyspaceRepairPlans().get(0).ksTablesEstimatedBytes = ksTablesEstimatedBytes; + return plan; } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java index 38f9c8538846..d8eb428c4bda 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlanTest.java @@ -18,14 +18,18 @@ package org.apache.cassandra.repair.autorepair; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import com.google.common.collect.Lists; +import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.service.StorageService; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -35,6 +39,13 @@ */ public class PrioritizedRepairPlanTest extends CQLTester { + + @BeforeClass + public static void setup() + { + StorageService.instance.doAutoRepairSetup(); + } + @Test public void testBuildWithDifferentPriorities() { @@ -43,7 +54,8 @@ public void testBuildWithDifferentPriorities() String table2 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '3'}"); String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '1'}"); - List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE, table1, table2, table3); + List prioritizedRepairPlans = PrioritizedRepairPlan.build(new HashMap>(){{put(KEYSPACE, Arrays.asList(table1, table2, table3));}}, AutoRepairConfig.RepairType.FULL, (l) -> {}, + AutoRepairService.instance.getAutoRepairConfig().getRepairPrimaryTokenRangeOnly(AutoRepairConfig.RepairType.FULL)); assertEquals(3, prioritizedRepairPlans.size()); // Verify the order is by descending priority and matches the expected tables @@ -66,7 +78,8 @@ public void testBuildWithSamePriority() String table3 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '2'}"); // Expect only 1 plan since all tables share the same priority - List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE, table1, table2, table3); + List prioritizedRepairPlans = PrioritizedRepairPlan.build(new HashMap>(){{put(KEYSPACE, Arrays.asList(table1, table2, table3));}}, AutoRepairConfig.RepairType.FULL, (l) -> {}, + AutoRepairService.instance.getAutoRepairConfig().getRepairPrimaryTokenRangeOnly(AutoRepairConfig.RepairType.FULL)); assertEquals(1, prioritizedRepairPlans.size()); // Verify all tables present in the plan @@ -101,7 +114,7 @@ public void testBuildWithMixedPriorities() keyspaceToTableMap.put(ks2, Lists.newArrayList(table6, table7)); // Expect 4 plans - List prioritizedRepairPlans = PrioritizedRepairPlan.build(keyspaceToTableMap, AutoRepairConfig.RepairType.FULL, java.util.Collections::sort); + List prioritizedRepairPlans = PrioritizedRepairPlan.build(keyspaceToTableMap, AutoRepairConfig.RepairType.FULL, java.util.Collections::sort, true); assertEquals(4, prioritizedRepairPlans.size()); // Verify the order is by descending priority and matches the expected tables @@ -143,7 +156,8 @@ public void testBuildWithMixedPriorities() public void testBuildWithEmptyTableList() { // Test with an empty table list (should remain empty) - List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE); + List prioritizedRepairPlans = PrioritizedRepairPlan.build(new HashMap>(){{put(KEYSPACE, Arrays.asList());}}, AutoRepairConfig.RepairType.FULL, (l) -> {}, + AutoRepairService.instance.getAutoRepairConfig().getRepairPrimaryTokenRangeOnly(AutoRepairConfig.RepairType.FULL)); assertTrue(prioritizedRepairPlans.isEmpty()); } @@ -154,7 +168,8 @@ public void testBuildWithOneTable() String table1 = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH auto_repair = {'full_enabled': 'true', 'priority': '5'}"); // Expect only 1 plans - List prioritizedRepairPlans = PrioritizedRepairPlan.buildSingleKeyspacePlan(AutoRepairConfig.RepairType.FULL, KEYSPACE, table1); + List prioritizedRepairPlans = PrioritizedRepairPlan.build(new HashMap>(){{put(KEYSPACE, Arrays.asList(table1));}}, AutoRepairConfig.RepairType.FULL, (l) -> {}, + AutoRepairService.instance.getAutoRepairConfig().getRepairPrimaryTokenRangeOnly(AutoRepairConfig.RepairType.FULL)); assertEquals(1, prioritizedRepairPlans.size()); // Verify the order is by descending priority and matches the expected tables diff --git a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java index 8bb8a6b81573..af09ff075856 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitterTest.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -36,16 +37,20 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; import org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.FilteredRepairAssignments; -import org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.SizeEstimate; +import org.apache.cassandra.repair.autorepair.AutoRepairUtils.SizeEstimate; import org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.SizedRepairAssignment; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.utils.concurrent.Refs; +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.getKeyspaceTableName; import static org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.MAX_BYTES_PER_SCHEDULE; import static org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.BYTES_PER_ASSIGNMENT; import static org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter.MAX_TABLES_PER_ASSIGNMENT; @@ -59,11 +64,13 @@ */ public class RepairTokenRangeSplitterTest extends CQLTester { + public static List SYSTEM_DISTRIBUTED_TABLE_NAMES = Arrays.asList(SystemDistributedKeyspace.REPAIR_HISTORY, SystemDistributedKeyspace.PARENT_REPAIR_HISTORY, SystemDistributedKeyspace.VIEW_BUILD_STATUS, SystemDistributedKeyspace.PARTITION_DENYLIST_TABLE, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY); + public static List AUTH_TABLE_NAMES = Arrays.asList(AuthKeyspace.ROLES, AuthKeyspace.ROLE_MEMBERS, AuthKeyspace.ROLE_PERMISSIONS, AuthKeyspace.RESOURCE_ROLE_INDEX, AuthKeyspace.NETWORK_PERMISSIONS); + private RepairTokenRangeSplitter repairRangeSplitter; private String tableName; private static Range FULL_RANGE; - private static List AUTH_TABLE_NAMES = Arrays.asList(AuthKeyspace.ROLES, AuthKeyspace.ROLE_MEMBERS, AuthKeyspace.ROLE_PERMISSIONS, AuthKeyspace.RESOURCE_ROLE_INDEX, AuthKeyspace.NETWORK_PERMISSIONS); @BeforeClass public static void setUpClass() @@ -89,7 +96,7 @@ public void testSizePartitionCount() throws Throwable try (Refs sstables = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE)) { assertEquals(10, sstables.iterator().next().getEstimatedPartitionSize().count()); - SizeEstimate sizes = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE, sstables); + SizeEstimate sizes = AutoRepairUtils.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, FULL_RANGE, sstables); assertEquals(10, sizes.partitions); } } @@ -110,8 +117,8 @@ public void testSizePartitionCountSplit() throws Throwable try (Refs sstables1 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange1); Refs sstables2 = RepairTokenRangeSplitter.getSSTableReaderRefs(RepairType.FULL, KEYSPACE, tableName, tokenRange2)) { - SizeEstimate sizes1 = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange1, sstables1); - SizeEstimate sizes2 = RepairTokenRangeSplitter.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange2, sstables2); + SizeEstimate sizes1 = AutoRepairUtils.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange1, sstables1); + SizeEstimate sizes2 = AutoRepairUtils.getSizesForRangeOfSSTables(RepairType.FULL, KEYSPACE, tableName, tokenRange2, sstables2); // +-5% because including entire compression blocks covering token range, HLL merge and the applying of range size approx ratio causes estimation errors long allowableDelta = (long) (partitionCount * .05); @@ -121,68 +128,150 @@ public void testSizePartitionCountSplit() throws Throwable } @Test - public void testGetRepairAssignmentsForTable_NoSSTables() + public void testGetRepairAssignmentsForTableNoSSTables() { // Should return 1 assignment if there are no SSTables - List assignments = repairRangeSplitter.getRepairAssignmentsForTable(CQLTester.KEYSPACE, tableName, FULL_RANGE); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(new KeyspaceRepairPlan(CQLTester.KEYSPACE, Collections.singletonList(tableName), AutoRepairUtils.calcTotalBytesToBeRepaired(RepairType.FULL, CQLTester.KEYSPACE, Collections.singletonList(tableName), Collections.singletonList(FULL_RANGE))), tableName, FULL_RANGE); assertEquals(1, assignments.size()); } @Test - public void testGetRepairAssignmentsForTable_Single() throws Throwable + public void testGetRepairAssignmentsForTableSingle() throws Throwable { insertAndFlushSingleTable(); - List assignments = repairRangeSplitter.getRepairAssignmentsForTable(CQLTester.KEYSPACE, tableName, FULL_RANGE); + List assignments = repairRangeSplitter.getRepairAssignmentsForTable(new KeyspaceRepairPlan(CQLTester.KEYSPACE, Collections.singletonList(tableName), AutoRepairUtils.calcTotalBytesToBeRepaired(RepairType.FULL, CQLTester.KEYSPACE, Collections.singletonList(tableName), Collections.singletonList(FULL_RANGE))), tableName, FULL_RANGE); assertEquals(1, assignments.size()); } @Test - public void testGetRepairAssignmentsForTable_BatchingTables() throws Throwable + public void testGetRepairAssignmentsForTableBatchingTablesCompressed() throws Throwable { repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "2")); - List tableNames = createAndInsertTables(3); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, FULL_RANGE); + List tableNames = createAndInsertTables(3, true); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, new KeyspaceRepairPlan(CQLTester.KEYSPACE, tableNames, AutoRepairUtils.calcTotalBytesToBeRepaired(RepairType.FULL, CQLTester.KEYSPACE, tableNames, Collections.singletonList(FULL_RANGE))), FULL_RANGE); // We expect two assignments, one with table1 and table2 batched, and one with table3 assertEquals(2, assignments.size()); assertEquals(2, assignments.get(0).getTableNames().size()); assertEquals(1, assignments.get(1).getTableNames().size()); + assertEquals(new HashSet<>(Arrays.asList(tableNames.get(0), tableNames.get(1))), new HashSet<>(assignments.get(0).getTableNames())); + assertEquals(tableNames.get(2), assignments.get(1).getTableNames().get(0)); } @Test - public void testGetRepairAssignmentsForTable_BatchSize() throws Throwable + public void testGetRepairAssignmentsForTableBatchingTablesUncompressed() throws Throwable { repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "2")); - List tableNames = createAndInsertTables(2); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, FULL_RANGE); + List tableNames = createAndInsertTables(3, false); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, new KeyspaceRepairPlan(CQLTester.KEYSPACE, tableNames, AutoRepairUtils.calcTotalBytesToBeRepaired(RepairType.FULL, CQLTester.KEYSPACE, tableNames, Collections.singletonList(FULL_RANGE))), FULL_RANGE); + + // We expect two assignments, one with table1 and table2 batched, and one with table3 + assertEquals(2, assignments.size()); + assertEquals(2, assignments.get(0).getTableNames().size()); + assertEquals(1, assignments.get(1).getTableNames().size()); + assertEquals(new HashSet<>(Arrays.asList(tableNames.get(0), tableNames.get(1))), new HashSet<>(assignments.get(0).getTableNames())); + assertEquals(tableNames.get(2), assignments.get(1).getTableNames().get(0)); + assertTrue(assignments.get(0).getEstimatedBytes() > 0); + assertTrue(assignments.get(1).getEstimatedBytes() > 0); + assertEquals(calculateSSTableSizeOnDisk(Arrays.asList(tableNames.get(0), tableNames.get(1))), assignments.get(0).getEstimatedBytes()); + assertEquals(calculateSSTableSizeOnDisk(Collections.singletonList(tableNames.get(2))), assignments.get(1).getEstimatedBytes()); + } + + @Test + public void testGetRepairAssignmentsForTableBatchSizeCompressed() throws Throwable + { + repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "2")); + + List tableNames = createAndInsertTables(2, true); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, new KeyspaceRepairPlan(CQLTester.KEYSPACE, tableNames, AutoRepairUtils.calcTotalBytesToBeRepaired(RepairType.FULL, CQLTester.KEYSPACE, tableNames, Collections.singletonList(FULL_RANGE))), FULL_RANGE); // We expect one assignment, with two tables batched assertEquals(1, assignments.size()); assertEquals(2, assignments.get(0).getTableNames().size()); + assertEquals(new HashSet<>(Arrays.asList(tableNames.get(0), tableNames.get(1))), new HashSet<>(assignments.get(0).getTableNames())); + } + + @Test + public void testGetRepairAssignmentsForTableBatchSizeUnCompressed() throws Throwable + { + repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "2")); + + List tableNames = createAndInsertTables(2, false); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, + new KeyspaceRepairPlan(CQLTester.KEYSPACE, tableNames, AutoRepairUtils.calcTotalBytesToBeRepaired(RepairType.FULL, CQLTester.KEYSPACE, tableNames, Collections.singletonList(FULL_RANGE))), FULL_RANGE); + + // We expect one assignment, with two tables batched + assertEquals(1, assignments.size()); + assertEquals(2, assignments.get(0).getTableNames().size()); + assertEquals(new HashSet<>(Arrays.asList(tableNames.get(0), tableNames.get(1))), new HashSet<>(assignments.get(0).getTableNames())); + assertTrue(assignments.get(0).getEstimatedBytes() > 0); + assertEquals(calculateSSTableSizeOnDisk(Arrays.asList(tableNames.get(0), tableNames.get(1))), assignments.get(0).getEstimatedBytes()); + } + + @Test + public void testGetRepairAssignmentsForTableNoBatchingCompressed() throws Throwable + { + repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "1")); + + List tableNames = createAndInsertTables(3, true); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, new KeyspaceRepairPlan(CQLTester.KEYSPACE, tableNames, AutoRepairUtils.calcTotalBytesToBeRepaired(RepairType.FULL, CQLTester.KEYSPACE, tableNames, Collections.singletonList(FULL_RANGE))), FULL_RANGE); + + assertEquals(3, assignments.size()); + assertEquals(Collections.singletonList(tableNames.get(0)), assignments.get(0).getTableNames()); + assertEquals(Collections.singletonList(tableNames.get(1)), assignments.get(1).getTableNames()); + assertEquals(Collections.singletonList(tableNames.get(2)), assignments.get(2).getTableNames()); } @Test - public void testGetRepairAssignmentsForTable_NoBatching() throws Throwable + public void testGetRepairAssignmentsForTableNoBatchingUncompressed() throws Throwable { repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "1")); - List tableNames = createAndInsertTables(3); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, FULL_RANGE); + List tableNames = createAndInsertTables(3, false); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, new KeyspaceRepairPlan(CQLTester.KEYSPACE, tableNames, AutoRepairUtils.calcTotalBytesToBeRepaired(RepairType.FULL, CQLTester.KEYSPACE, tableNames, Collections.singletonList(FULL_RANGE))), FULL_RANGE); assertEquals(3, assignments.size()); + assertEquals(Collections.singletonList(tableNames.get(0)), assignments.get(0).getTableNames()); + assertEquals(Collections.singletonList(tableNames.get(1)), assignments.get(1).getTableNames()); + assertEquals(Collections.singletonList(tableNames.get(2)), assignments.get(2).getTableNames()); + assertTrue(assignments.get(0).getEstimatedBytes() > 0); + assertTrue(assignments.get(1).getEstimatedBytes() > 0); + assertTrue(assignments.get(2).getEstimatedBytes() > 0); + assertEquals(calculateSSTableSizeOnDisk(Collections.singletonList(tableNames.get(0))), assignments.get(0).getEstimatedBytes()); + assertEquals(calculateSSTableSizeOnDisk(Collections.singletonList(tableNames.get(1))), assignments.get(1).getEstimatedBytes()); + assertEquals(calculateSSTableSizeOnDisk(Collections.singletonList(tableNames.get(2))), assignments.get(2).getEstimatedBytes()); + } + + @Test + public void testGetRepairAssignmentsForTableAllBatchedCompressed() throws Throwable + { + repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "100")); + + List tableNames = createAndInsertTables(5, true); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, new KeyspaceRepairPlan(CQLTester.KEYSPACE, tableNames, AutoRepairUtils.calcTotalBytesToBeRepaired(RepairType.FULL, CQLTester.KEYSPACE, tableNames, Collections.singletonList(FULL_RANGE))), FULL_RANGE); + + assertEquals(1, assignments.size()); + assertEquals(5, assignments.get(0).getTableNames().size()); + assertEquals(new HashSet<>(tableNames), + new HashSet<>(assignments.get(0).getTableNames())); } @Test - public void testGetRepairAssignmentsForTable_AllBatched() throws Throwable + public void testGetRepairAssignmentsForTableAllBatchedUncompressed() throws Throwable { repairRangeSplitter = new RepairTokenRangeSplitter(RepairType.FULL, Collections.singletonMap(MAX_TABLES_PER_ASSIGNMENT, "100")); - List tableNames = createAndInsertTables(5); - List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, KEYSPACE, tableNames, FULL_RANGE); + List tableNames = createAndInsertTables(5, false); + List assignments = repairRangeSplitter.getRepairAssignmentsForKeyspace(RepairType.FULL, new KeyspaceRepairPlan(CQLTester.KEYSPACE, tableNames, AutoRepairUtils.calcTotalBytesToBeRepaired(RepairType.FULL, CQLTester.KEYSPACE, tableNames, Collections.singletonList(FULL_RANGE))), FULL_RANGE); assertEquals(1, assignments.size()); + assertEquals(5, assignments.get(0).getTableNames().size()); + assertEquals(new HashSet<>(tableNames), + new HashSet<>(assignments.get(0).getTableNames())); + assertTrue(assignments.get(0).getEstimatedBytes() > 0); + assertEquals(calculateSSTableSizeOnDisk(tableNames), assignments.get(0).getEstimatedBytes()); } @Test(expected = IllegalStateException.class) @@ -290,7 +379,7 @@ public void testGetRepairAssignmentsSplitsBySubrangeSizeAndFilterLimitsByMaxByte // Given a size estimate of 1024GiB, we should expect 21 splits (50GiB*21 = 1050GiB < 1024GiB) SizeEstimate sizeEstimate = sizeEstimateByBytes(new LongMebibytesBound("1024GiB")); - List assignments = repairRangeSplitter.getRepairAssignments(Collections.singletonList(sizeEstimate)); + List assignments = repairRangeSplitter.getRepairAssignments(sizeEstimate); // Should be 21 assignments, each being ~48.76 GiB assertEquals(21, assignments.size()); @@ -313,7 +402,24 @@ public void testTokenRangesRepairByKeyspace() { AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(RepairType.FULL, true); - final KeyspaceRepairPlan repairPlan = new KeyspaceRepairPlan("system_auth", AUTH_TABLE_NAMES); + Map, AutoRepairUtils.SizeEstimate>> ksTablesEstimatedBytes = new HashMap<>(); + List> tokenRanges = AutoRepairUtils.getTokenRanges(true, SchemaConstants.AUTH_KEYSPACE_NAME); + long tableSizeInBytes = 100L; + long tableSizeInBytesPerTokenRange = tableSizeInBytes / tokenRanges.size(); + for (String tableName : AUTH_TABLE_NAMES) + { + String ksTableName = getKeyspaceTableName(SchemaConstants.AUTH_KEYSPACE_NAME, tableName); + ksTablesEstimatedBytes.putIfAbsent(ksTableName, new HashMap<>()); + Map, AutoRepairUtils.SizeEstimate> rangeSizeEstimateMap = ksTablesEstimatedBytes.get(ksTableName); + + for (Range tokenRange : tokenRanges) + { + rangeSizeEstimateMap.put(tokenRange, new AutoRepairUtils.SizeEstimate(AutoRepairConfig.RepairType.FULL, SchemaConstants.AUTH_KEYSPACE_NAME, tableName, tokenRange, 0, tableSizeInBytesPerTokenRange, tableSizeInBytesPerTokenRange)); + } + } + final KeyspaceRepairPlan repairPlan = new KeyspaceRepairPlan(SchemaConstants.AUTH_KEYSPACE_NAME, new ArrayList<>(AUTH_TABLE_NAMES), ksTablesEstimatedBytes); + assertEquals(tableSizeInBytes * AUTH_TABLE_NAMES.size(), repairPlan.getEstimatedBytes()); + final PrioritizedRepairPlan prioritizedRepairPlan = new PrioritizedRepairPlan(0, Arrays.asList(repairPlan)); Iterator keyspaceAssignments = repairRangeSplitter.getRepairAssignments(true, Arrays.asList(prioritizedRepairPlan)); @@ -341,7 +447,25 @@ public void testTokenRangesRepairByKeyspaceFalse() { AutoRepairService.instance.getAutoRepairConfig().setRepairByKeyspace(RepairType.FULL, false); - final KeyspaceRepairPlan repairPlan = new KeyspaceRepairPlan("system_auth", AUTH_TABLE_NAMES); + Map, AutoRepairUtils.SizeEstimate>> ksTablesEstimatedBytes = new HashMap<>(); + List> tokenRanges = AutoRepairUtils.getTokenRanges(true, SchemaConstants.AUTH_KEYSPACE_NAME); + long tableSizeInBytes = 100L; + long tableSizeInBytesPerTokenRange = tableSizeInBytes / tokenRanges.size(); + for (String tableName : AUTH_TABLE_NAMES) + { + String ksTableName = getKeyspaceTableName(SchemaConstants.AUTH_KEYSPACE_NAME, tableName); + ksTablesEstimatedBytes.putIfAbsent(ksTableName, new HashMap<>()); + Map, AutoRepairUtils.SizeEstimate> rangeSizeEstimateMap = ksTablesEstimatedBytes.get(ksTableName); + + for (Range tokenRange : tokenRanges) + { + rangeSizeEstimateMap.put(tokenRange, new AutoRepairUtils.SizeEstimate(AutoRepairConfig.RepairType.FULL, SchemaConstants.AUTH_KEYSPACE_NAME, tableName, tokenRange, 0, tableSizeInBytesPerTokenRange, tableSizeInBytesPerTokenRange)); + } + } + + final KeyspaceRepairPlan repairPlan = new KeyspaceRepairPlan(SchemaConstants.AUTH_KEYSPACE_NAME, new ArrayList<>(AUTH_TABLE_NAMES), ksTablesEstimatedBytes); + assertEquals(tableSizeInBytes * AUTH_TABLE_NAMES.size(), repairPlan.getEstimatedBytes()); + final PrioritizedRepairPlan prioritizedRepairPlan = new PrioritizedRepairPlan(0, Arrays.asList(repairPlan)); Iterator keyspaceAssignments = repairRangeSplitter.getRepairAssignments(true, Arrays.asList(prioritizedRepairPlan)); @@ -414,18 +538,43 @@ private void insertAndFlushSingleTable() throws Throwable flush(); } - private List createAndInsertTables(int count) throws Throwable + private List createAndInsertTables(int count, boolean enableCompression) throws Throwable { List tableNames = new ArrayList<>(); for (int i = 0; i < count; i++) { - String tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); + String tableName; + if (enableCompression) + { + tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT)"); + } + else + { + tableName = createTable("CREATE TABLE %s (k INT PRIMARY KEY, v INT) WITH compression = { 'enabled' : false }"); + } tableNames.add(tableName); insertAndFlushTable(tableName); } return tableNames; } + private long calculateSSTableSizeOnDisk(List tableNames) + { + long totalSSTableBytes = 0; + for (int i = 0; i < tableNames.size(); i++) + { + String tableName = tableNames.get(i); + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, tableName); + assertNotNull(cfs); + Iterable sstables = cfs.getTracker().getView().select(SSTableSet.CANONICAL); + for (SSTableReader sstable : sstables) + { + totalSSTableBytes += sstable.bytesOnDisk(); + } + } + return totalSSTableBytes; + } + private void insertAndFlushTable(String tableName) throws Throwable { insertAndFlushTable(tableName, 1); From f3448cea5c2bcf63b9e46c35d39b1fde51d9caee Mon Sep 17 00:00:00 2001 From: Paulo Motta Date: Tue, 13 Jan 2026 17:11:18 -0500 Subject: [PATCH 197/199] Fix initial auto-repairs skipped by too soon check patch by Paulo Motta; reviewed by Jaydeepkumar Chovatia for CASSANDRA-21115 (cherry picked from commit afa55123c87be7fd31a68abd87c3427141fe60c0) --- CHANGES.txt | 1 + .../repair/autorepair/AutoRepair.java | 30 +++++--- .../repair/autorepair/AutoRepairState.java | 22 ++++-- .../repair/autorepair/AutoRepairUtils.java | 31 +++++++- .../AutoRepairParameterizedTest.java | 36 ++++----- .../autorepair/AutoRepairStateTest.java | 8 +- .../repair/autorepair/AutoRepairTest.java | 76 +++++++++++++++++++ .../autorepair/AutoRepairUtilsTest.java | 24 +++++- 8 files changed, 187 insertions(+), 41 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 6e724426bbdb..ab50ea792ea5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1.6 + * Fix initial auto-repairs skipped by too soon check (CASSANDRA-21115) * Improved observability in AutoRepair to report both expected vs. actual repair bytes and expected vs. actual keyspaces (CASSANDRA-20581) * Stop repair scheduler if two major versions are detected (CASSANDRA-20048) * AutoRepair: Safeguard Full repair against disk protection(CASSANDRA-20045) diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java index c8ff80f2678a..5bedd9d30a1a 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java @@ -210,7 +210,8 @@ public void repair(AutoRepairConfig.RepairType repairType) long startTimeInMillis = timeFunc.get(); logger.info("My host id: {}, my turn to run repair...repair primary-ranges only? {}", myId, config.getRepairPrimaryTokenRangeOnly(repairType)); - AutoRepairUtils.updateStartAutoRepairHistory(repairType, myId, timeFunc.get(), turn); + AutoRepairUtils.updateStartAutoRepairHistory(repairType, myId, startTimeInMillis, turn); + repairState.setLastRepairStartTime(startTimeInMillis); repairState.setRepairKeyspaceCount(0); repairState.setRepairInProgress(true); @@ -403,20 +404,31 @@ else if (retryCount < config.getRepairMaxRetries(repairType)) } } - private boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoRepairState repairState, AutoRepairConfig config, UUID myId) + @VisibleForTesting + boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoRepairState repairState, AutoRepairConfig config, UUID myId) { - if (repairState.getLastRepairTime() == 0) + if (repairState.getLastRepairFinishTime() == 0) { - // the node has either just boooted or has not run repair before, + // the node has either just booted or has not run repair before, // we should check for the node's repair history in the DB - repairState.setLastRepairTime(AutoRepairUtils.getLastRepairTimeForNode(repairType, myId)); + repairState.setLastRepairFinishTime(AutoRepairUtils.getLastRepairFinishTimeForNode(repairType, myId)); + repairState.setLastRepairStartTime(AutoRepairUtils.getLastRepairStartTimeForNode(repairType, myId)); } + + // If repair has not completed (start >= finish), don't skip - allow it to continue/resume + if (repairState.getLastRepairStartTime() >= repairState.getLastRepairFinishTime()) + { + logger.info("Incomplete or unstarted repair detected (start_ts={} >= finish_ts={}), allowing resume", + repairState.getLastRepairStartTime(), repairState.getLastRepairFinishTime()); + return false; + } + /* * check if it is too soon to run repair. one of the reason we * should not run frequent repair is that repair triggers * memtable flush */ - long timeElapsedSinceLastRepair = TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - repairState.getLastRepairTime()); + long timeElapsedSinceLastRepair = TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - repairState.getLastRepairFinishTime()); if (timeElapsedSinceLastRepair < config.getRepairMinInterval(repairType).toSeconds()) { logger.info("Too soon to run repair, last repair was done {} seconds ago", @@ -498,14 +510,14 @@ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType "repairTokenRangesSkipCount {}, repairTablesSkipCount {}", repairType, timeInHours, repairState.getRepairKeyspaceCount(), repairState.getSucceededTokenRangesCount(), repairState.getFailedTokenRangesCount(), repairState.getSkippedTokenRangesCount(), repairState.getSkippedTablesCount()); - if (repairState.getLastRepairTime() != 0) + if (repairState.getLastRepairFinishTime() != 0) { repairState.setClusterRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - - repairState.getLastRepairTime())); + repairState.getLastRepairFinishTime())); logger.info("Cluster repair time for repair type {}: {} day(s)", repairType, TimeUnit.SECONDS.toDays(repairState.getClusterRepairTimeInSec())); } - repairState.setLastRepairTime(timeFunc.get()); + repairState.setLastRepairFinishTime(timeFunc.get()); repairState.setRepairInProgress(false); AutoRepairUtils.updateFinishAutoRepairHistory(repairType, myId, timeFunc.get()); diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java index f6e33dcae96f..c5154f516f59 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java @@ -64,7 +64,9 @@ public abstract class AutoRepairState @VisibleForTesting protected int totalTablesConsideredForRepair = 0; @VisibleForTesting - protected long lastRepairTimeInMs; + protected long lastRepairFinishTimeInMs; + @VisibleForTesting + protected long lastRepairStartTimeInMs; @VisibleForTesting protected int nodeRepairTimeInSec = 0; @VisibleForTesting @@ -120,9 +122,9 @@ public void updateRepairScheduleStatistics(List repairPla setTotalKeyspaceRepairPlansToRepair(repairPlans.stream().mapToInt(repairPlan -> repairPlan.getKeyspaceRepairPlans().size()).sum()); } - public long getLastRepairTime() + public long getLastRepairFinishTime() { - return lastRepairTimeInMs; + return lastRepairFinishTimeInMs; } public void setTotalTablesConsideredForRepair(int count) @@ -135,9 +137,19 @@ public int getTotalTablesConsideredForRepair() return totalTablesConsideredForRepair; } - public void setLastRepairTime(long lastRepairTime) + public void setLastRepairFinishTime(long lastRepairFinishTime) + { + lastRepairFinishTimeInMs = lastRepairFinishTime; + } + + public long getLastRepairStartTime() + { + return lastRepairStartTimeInMs; + } + + public void setLastRepairStartTime(long lastRepairStartTime) { - lastRepairTimeInMs = lastRepairTime; + lastRepairStartTimeInMs = lastRepairStartTime; } public int getClusterRepairTimeInSec() diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java index 8b9c93ab9c4d..313d2542feb5 100644 --- a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java +++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java @@ -177,11 +177,16 @@ public class AutoRepairUtils "SELECT %s FROM %s.%s WHERE %s = ? AND %s = ?", COL_REPAIR_FINISH_TS, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, COL_HOST_ID); + final static String SELECT_LAST_REPAIR_START_TIME_FOR_NODE = String.format( + "SELECT %s FROM %s.%s WHERE %s = ? AND %s = ?", COL_REPAIR_START_TS, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, + SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, COL_HOST_ID); + static ModificationStatement delStatementRepairHistory; static SelectStatement selectStatementRepairHistory; static ModificationStatement delStatementPriorityStatus; static SelectStatement selectStatementRepairPriority; static SelectStatement selectLastRepairTimeForNode; + static SelectStatement selectLastRepairStartTimeForNode; static ModificationStatement addPriorityHost; static ModificationStatement insertNewRepairHistoryStatement; static ModificationStatement recordStartRepairHistoryStatement; @@ -207,6 +212,8 @@ public static void setup() .forInternalCalls()); selectLastRepairTimeForNode = (SelectStatement) QueryProcessor.getStatement(SELECT_LAST_REPAIR_TIME_FOR_NODE, ClientState .forInternalCalls()); + selectLastRepairStartTimeForNode = (SelectStatement) QueryProcessor.getStatement(SELECT_LAST_REPAIR_START_TIME_FOR_NODE, ClientState + .forInternalCalls()); delStatementPriorityStatus = (ModificationStatement) QueryProcessor.getStatement(DEL_REPAIR_PRIORITY, ClientState .forInternalCalls()); addPriorityHost = (ModificationStatement) QueryProcessor.getStatement(ADD_PRIORITY_HOST, ClientState @@ -382,7 +389,8 @@ public static void setForceRepairNewNode(RepairType repairType) // this function will be called when a node bootstrap finished UUID hostId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); // insert the data first - insertNewRepairHistory(repairType, currentTimeMillis(), currentTimeMillis()); + long timestamp = currentTimeMillis(); + insertNewRepairHistory(repairType, timestamp, timestamp); setForceRepair(repairType, hostId); } @@ -407,7 +415,7 @@ public static void setForceRepair(RepairType repairType, UUID hostId) logger.info("Set force repair repair type: {}, node: {}", repairType, hostId); } - public static long getLastRepairTimeForNode(RepairType repairType, UUID hostId) + public static long getLastRepairFinishTimeForNode(RepairType repairType, UUID hostId) { ResultMessage.Rows rows = selectLastRepairTimeForNode.execute(QueryState.forInternalCalls(), QueryOptions.forInternalCalls(internalQueryCL, @@ -423,6 +431,22 @@ public static long getLastRepairTimeForNode(RepairType repairType, UUID hostId) return repairTime.one().getLong(COL_REPAIR_FINISH_TS); } + public static long getLastRepairStartTimeForNode(RepairType repairType, UUID hostId) + { + ResultMessage.Rows rows = selectLastRepairStartTimeForNode.execute(QueryState.forInternalCalls(), + QueryOptions.forInternalCalls(internalQueryCL, + Lists.newArrayList( + ByteBufferUtil.bytes(repairType.toString()), + ByteBufferUtil.bytes(hostId))), + Dispatcher.RequestTime.forImmediateExecution()); + UntypedResultSet repairTime = UntypedResultSet.create(rows.result); + if (repairTime.isEmpty()) + { + return 0; + } + return repairTime.one().getLong(COL_REPAIR_START_TS); + } + @VisibleForTesting public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType, List autoRepairHistories, UUID myId) { @@ -836,7 +860,8 @@ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId)) if (!autoRepairHistoryIds.contains(hostId)) { logger.info("{} for repair type {} doesn't exist in the auto repair history table, insert a new record.", repairType, hostId); - insertNewRepairHistory(repairType, hostId, currentTimeMillis(), currentTimeMillis()); + long timestamp = currentTimeMillis(); + insertNewRepairHistory(repairType, hostId, timestamp, timestamp); } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java index bd56b1e60ce8..c517923b6c8d 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java @@ -284,7 +284,7 @@ public void testRepair() AutoRepair.instance.repair(repairType); assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); - long lastRepairTime = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + long lastRepairTime = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime(); //if repair was done then lastRepairTime should be non-zero Assert.assertTrue(String.format("Expected lastRepairTime > 0, actual value lastRepairTime %d", lastRepairTime), lastRepairTime > 0); @@ -299,7 +299,7 @@ public void testTooFrequentRepairs() //in the first round let repair run config.setRepairMinInterval(repairType, "0s"); AutoRepair.instance.repair(repairType); - long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime(); int consideredTables = AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair(); Assert.assertNotEquals(String.format("Expected total repaired tables > 0, actual value %s ", consideredTables), 0, consideredTables); @@ -307,7 +307,7 @@ public void testTooFrequentRepairs() //if repair was done in last 24 hours then it should not trigger another repair config.setRepairMinInterval(repairType, "24h"); AutoRepair.instance.repair(repairType); - long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime(); Assert.assertEquals(String.format("Expected repair time to be same, actual value lastRepairTime1 %d, " + "lastRepairTime2 %d", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); @@ -322,14 +322,14 @@ public void testNonFrequentRepairs() long prevCount = state.getTotalMVTablesConsideredForRepair(); AutoRepairService.instance.getAutoRepairConfig().setRepairMinInterval(repairType, "0s"); AutoRepair.instance.repair(repairType); - long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime(); Assert.assertTrue(String.format("Expected lastRepairTime1 > 0, actual value lastRepairTime1 %d", lastRepairTime1), lastRepairTime1 > 0); UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort()); Assert.assertNotEquals("Expected my turn for the repair", NOT_MY_TURN, AutoRepairUtils.myTurnToRunRepair(repairType, myId)); AutoRepair.instance.repair(repairType); - long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime(); Assert.assertNotSame(String.format("Expected repair time to be same, actual value lastRepairTime1 %d, " + "lastRepairTime2 %d", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); assertEquals(prevCount, state.getTotalMVTablesConsideredForRepair()); @@ -365,9 +365,9 @@ public void testCheckAutoRepairStartStop() throws Throwable long prevCount = state.getTotalMVTablesConsideredForRepair(); config.setRepairMinInterval(repairType, "0s"); config.setAutoRepairEnabled(repairType, false); - long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime(); AutoRepair.instance.repair(repairType); - long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime(); //Since repair has not happened, both the last repair times should be same Assert.assertEquals(String.format("Expected lastRepairTime1 %d, and lastRepairTime2 %d to be same", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2); @@ -375,7 +375,7 @@ public void testCheckAutoRepairStartStop() throws Throwable config.setAutoRepairEnabled(repairType, true); AutoRepair.instance.repair(repairType); //since repair is done now, so lastRepairTime1/lastRepairTime2 and lastRepairTime3 should not be same - long lastRepairTime3 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime(); + long lastRepairTime3 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime(); Assert.assertNotSame(String.format("Expected lastRepairTime1 %d, and lastRepairTime3 %d to be not same", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime3); assertEquals(prevCount, state.getTotalMVTablesConsideredForRepair()); @@ -411,19 +411,19 @@ public void testMVRepair() AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); config.setMaterializedViewRepairEnabled(repairType, true); config.setRepairMinInterval(repairType, "0s"); - AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); + AutoRepair.instance.repairStates.get(repairType).setLastRepairFinishTime(System.currentTimeMillis()); AutoRepair.instance.repair(repairType); assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); config.setMaterializedViewRepairEnabled(repairType, false); - AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); + AutoRepair.instance.repairStates.get(repairType).setLastRepairFinishTime(System.currentTimeMillis()); AutoRepair.instance.repair(repairType); assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); config.setMaterializedViewRepairEnabled(repairType, true); - AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis()); + AutoRepair.instance.repairStates.get(repairType).setLastRepairFinishTime(System.currentTimeMillis()); AutoRepair.instance.repair(repairType); assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair()); assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); @@ -462,7 +462,7 @@ public void testSkipRepairSSTableCountHigherThreshold() config.setRepairSSTableCountHigherThreshold(repairType, 9); assertEquals(0, state.getSkippedTokenRangesCount()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue()); - state.setLastRepairTime(0); + state.setLastRepairFinishTime(0); AutoRepair.instance.repair(repairType); assertEquals(0, state.getTotalMVTablesConsideredForRepair()); assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue()); @@ -474,7 +474,7 @@ public void testSkipRepairSSTableCountHigherThreshold() // set it to higher value, and this time, the tables should not be skipped config.setRepairSSTableCountHigherThreshold(repairType, beforeCount); - state.setLastRepairTime(0); + state.setLastRepairFinishTime(0); state.setSkippedTablesCount(0); state.setTotalMVTablesConsideredForRepair(0); AutoRepair.instance.repair(repairType); @@ -509,7 +509,7 @@ public void testMetrics() timeFuncCalls++; return timeFuncCalls * 1000L; }; - AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(1000L); + AutoRepair.instance.repairStates.get(repairType).setLastRepairFinishTime(1000L); AutoRepair.instance.repair(repairType); @@ -544,7 +544,7 @@ public void testRepairWaitsForRepairToFinishBeforeSchedullingNewSession() { AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig(); AutoRepair.instance.repairStates.put(repairType, autoRepairState); - when(autoRepairState.getLastRepairTime()).thenReturn((long) 0); + when(autoRepairState.getLastRepairFinishTime()).thenReturn((long) 0); AtomicInteger getRepairRunnableCalls = new AtomicInteger(); AtomicReference prevListener = new AtomicReference<>(); doAnswer(invocation -> { @@ -643,7 +643,7 @@ public void testRepairTakesLastRepairTimeFromDB() config.setMaterializedViewRepairEnabled(repairType, true); long lastRepairTime = System.currentTimeMillis() - 1000; AutoRepairUtils.insertNewRepairHistory(repairType, 0, lastRepairTime); - AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(0); + AutoRepair.instance.repairStates.get(repairType).setLastRepairFinishTime(0); config.setRepairMinInterval(repairType, "1h"); AutoRepair.instance.repair(repairType); @@ -651,7 +651,7 @@ public void testRepairTakesLastRepairTimeFromDB() // repair scheduler should not attempt to run repair as last repair time in DB is current time - 1s assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair()); // repair scheduler should load the repair time from the DB - assertEquals(lastRepairTime, AutoRepair.instance.repairStates.get(repairType).getLastRepairTime()); + assertEquals(lastRepairTime, AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime()); } @Test @@ -879,7 +879,7 @@ public void testSchedulerIgnoresErrorsFromUnrelatedRepairRunables() } return runnable; }).when(spyState).getRepairRunnable(any(), any(), any(), anyBoolean()); - when(spyState.getLastRepairTime()).thenReturn((long) 0); + when(spyState.getLastRepairFinishTime()).thenReturn((long) 0); AutoRepairService.instance.getAutoRepairConfig().setRepairMaxRetries(repairType, 0); AutoRepair.instance.repairStates.put(repairType, spyState); diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java index 240a2db46fc9..0c61df9ba17c 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java @@ -81,9 +81,9 @@ public void testGetRepairRunnable() public void testGetLastRepairTime() { AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); - state.lastRepairTimeInMs = 1; + state.lastRepairFinishTimeInMs = 1; - assertEquals(1, state.getLastRepairTime()); + assertEquals(1, state.getLastRepairFinishTime()); } @Test @@ -110,9 +110,9 @@ public void testSetLastRepairTimeInMs() { AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig()); - state.setLastRepairTime(1); + state.setLastRepairFinishTime(1); - assertEquals(1, state.lastRepairTimeInMs); + assertEquals(1, state.lastRepairFinishTimeInMs); } @Test diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index 53eaca86dcbc..09baa182ea09 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -20,6 +20,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.UUID; import org.junit.Before; import org.junit.BeforeClass; @@ -29,16 +30,23 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType; +import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaTestUtil; +import org.apache.cassandra.schema.SystemDistributedKeyspace; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.Util.setAutoRepairEnabled; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -51,6 +59,8 @@ public static void setupClass() throws Exception { setAutoRepairEnabled(true); requireNetwork(); + AutoRepairUtils.setup(); + StorageService.instance.doAutoRepairSetup(); } @Before @@ -159,4 +169,70 @@ else if (ks.getName().equals(ksname2)) } } } + + @Test + public void testTooSoonToRunRepairAllowsResumeOfInProgressRepair() + { + RepairType repairType = RepairType.FULL; + UUID myId = StorageService.instance.getHostIdForEndpoint(FBUtilities.getBroadcastAddressAndPort()); + long now = System.currentTimeMillis(); + + // Truncate history table to start fresh + QueryProcessor.executeInternal(String.format( + "TRUNCATE %s.%s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY)); + + // Insert with start_ts > finish_ts to simulate in-progress repair (crashed before completion) + AutoRepairUtils.insertNewRepairHistory(repairType, myId, now, now - 1000); + + AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); + AutoRepairState repairState = RepairType.getAutoRepairState(repairType, config); + + // Even though finish_ts was very recent, should return false because repair is in progress + assertFalse(AutoRepair.instance.tooSoonToRunRepair(repairType, repairState, config, myId)); + } + + @Test + public void testTooSoonToRunRepairReturnsTrueWhenRepairCompletedRecently() + { + RepairType repairType = RepairType.FULL; + UUID myId = StorageService.instance.getHostIdForEndpoint(FBUtilities.getBroadcastAddressAndPort()); + long now = System.currentTimeMillis(); + + // Truncate history table to start fresh + QueryProcessor.executeInternal(String.format( + "TRUNCATE %s.%s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY)); + + // Insert with finish_ts > start_ts to simulate completed repair + AutoRepairUtils.insertNewRepairHistory(repairType, myId, now - 1000, now); + + AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); + AutoRepairState repairState = RepairType.getAutoRepairState(repairType, config); + + // Should return true because repair completed recently and min_repair_interval hasn't passed + assertTrue(AutoRepair.instance.tooSoonToRunRepair(repairType, repairState, config, myId)); + } + + @Test + public void testTooSoonToRunRepairAllowsResumeWhenStartEqualsFinish() + { + RepairType repairType = RepairType.FULL; + UUID myId = StorageService.instance.getHostIdForEndpoint(FBUtilities.getBroadcastAddressAndPort()); + long now = System.currentTimeMillis(); + + // Truncate history table to start fresh + QueryProcessor.executeInternal(String.format( + "TRUNCATE %s.%s", + SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY)); + + // Insert with start_ts == finish_ts (edge case: repair never started progressing) + AutoRepairUtils.insertNewRepairHistory(repairType, myId, now, now); + + AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig(); + AutoRepairState repairState = RepairType.getAutoRepairState(repairType, config); + + // Should return false because repair is incomplete (start >= finish edge case) + assertFalse(AutoRepair.instance.tooSoonToRunRepair(repairType, repairState, config, myId)); + } } diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java index 7f8bd524df21..93e2e16e5323 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java @@ -576,7 +576,7 @@ public void testGetLastRepairTimeForNode() AutoRepairUtils.insertNewRepairHistory(repairType, myID, currentMillis, currentMillis - 100); AutoRepairUtils.insertNewRepairHistory(repairType, otherID, currentMillis, currentMillis + 100); - assertEquals(currentMillis - 100, AutoRepairUtils.getLastRepairTimeForNode(repairType, myID)); + assertEquals(currentMillis - 100, AutoRepairUtils.getLastRepairFinishTimeForNode(repairType, myID)); } @Test @@ -584,7 +584,27 @@ public void testGetLastRepairTimeForNodeWhenHistoryIsEmpty() { UUID myID = UUID.randomUUID(); - assertEquals(0, AutoRepairUtils.getLastRepairTimeForNode(repairType, myID)); + assertEquals(0, AutoRepairUtils.getLastRepairFinishTimeForNode(repairType, myID)); + } + + @Test + public void testGetLastRepairStartTimeForNode() + { + UUID myID = UUID.randomUUID(); + UUID otherID = UUID.randomUUID(); + long currentMillis = System.currentTimeMillis(); + AutoRepairUtils.insertNewRepairHistory(repairType, myID, currentMillis, currentMillis - 100); + AutoRepairUtils.insertNewRepairHistory(repairType, otherID, currentMillis + 50, currentMillis + 100); + + assertEquals(currentMillis, AutoRepairUtils.getLastRepairStartTimeForNode(repairType, myID)); + } + + @Test + public void testGetLastRepairStartTimeForNodeWhenHistoryIsEmpty() + { + UUID myID = UUID.randomUUID(); + + assertEquals(0, AutoRepairUtils.getLastRepairStartTimeForNode(repairType, myID)); } @Test From 84ce9f4f82444fe8f372026f765f71e87c08cd5b Mon Sep 17 00:00:00 2001 From: sampreethb Date: Wed, 1 Jul 2026 11:41:11 -0700 Subject: [PATCH 198/199] Set SYSTEM_DISTRIBUTED_DEFAULT_RF=1 in AutoRepairTest to enable CAS on single-node test setup The new tests introduced by CASSANDRA-21115 call AutoRepairUtils.insertNewRepairHistory which uses a Paxos CAS (IF NOT EXISTS). system_distributed's default RF of 3 makes CAS fail in single-node unit-test setups with "Cannot achieve consistency level QUORUM". Trunk (5.1) does not hit this because its CMS/TCM-backed transactional metadata replaces the Paxos CAS path. On 4.1 we follow the existing convention used by AutoRepairUtilsTest, AutoRepairParameterizedTest, and AutoRepairServiceTest and set the RF to 1 in setupClass() before requireNetwork(). Verified locally: all 9 tests in AutoRepairTest now pass (0 failures). Co-authored-by: Cursor --- .../org/apache/cassandra/repair/autorepair/AutoRepairTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index 09baa182ea09..ccd2ec3ef47a 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -45,6 +45,7 @@ import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.Util.setAutoRepairEnabled; +import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -57,6 +58,7 @@ public class AutoRepairTest extends CQLTester @BeforeClass public static void setupClass() throws Exception { + SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1); setAutoRepairEnabled(true); requireNetwork(); AutoRepairUtils.setup(); From 16476c9a7af74bbd3b732201c477f2968e00858d Mon Sep 17 00:00:00 2001 From: sampreethb Date: Wed, 1 Jul 2026 22:06:50 -0700 Subject: [PATCH 199/199] Remove unused AutoRepairService import in AutoRepairTest Fixes checkstyle-test failure: [ERROR] test/unit/.../AutoRepairTest.java:43:8: Unused import - org.apache.cassandra.service.AutoRepairService. [UnusedImports] Import was inadvertently pulled in during the CASSANDRA-21115 cherry-pick conflict resolution (union of imports). It is used only on trunk (5.1), where the @Before setup() calls AutoRepairService.setup(); this branch does not have that call. Verified locally: `ant checkstyle checkstyle-test` clean; AutoRepairTest still 9/9 passing. Co-authored-by: Cursor --- .../org/apache/cassandra/repair/autorepair/AutoRepairTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java index ccd2ec3ef47a..6c49b7659bfd 100644 --- a/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java @@ -40,7 +40,6 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.schema.SystemDistributedKeyspace; -import org.apache.cassandra.service.AutoRepairService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities;