From fb7384b2427236c334e50414a0d5a0dc60865cc2 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Fri, 24 Jun 2022 20:57:09 +0800 Subject: [PATCH 01/35] Support repaired not adhering placement policy ledger. --- .../LocalBookieEnsemblePlacementPolicy.java | 9 +- .../bookkeeper/client/BookKeeperAdmin.java | 37 +++-- .../DefaultEnsemblePlacementPolicy.java | 9 +- .../client/EnsemblePlacementPolicy.java | 3 + .../bookkeeper/client/LedgerChecker.java | 37 ++++- .../bookkeeper/client/LedgerFragment.java | 20 ++- .../TopologyAwareEnsemblePlacementPolicy.java | 104 ++++++++++++- .../bookkeeper/conf/ServerConfiguration.java | 11 +- .../bookkeeper/net/NetworkTopology.java | 1 + .../bookkeeper/replication/Auditor.java | 17 ++- .../replication/ReplicationWorker.java | 74 ++++++++- .../AuditorPlacementPolicyCheckTest.java | 143 ++++++++++++++++++ 12 files changed, 435 insertions(+), 30 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java index 7c949cb3d4b..daa380847c1 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java @@ -125,7 +125,14 @@ public PlacementResult> newEnsemble(int ensembleSize, int writeQu public void updateBookieInfo(Map bookieToFreeSpaceMap) { return; } - + + @Override + public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, + int ackQuorumSize) { + //local bookie needn't support it. + return Collections.emptyMap(); + } + @Override public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List ensembleList, int writeQuorumSize, int ackQuorumSize) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java index 54cb566a306..a0ec1e87374 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java @@ -89,6 +89,7 @@ import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.util.AvailabilityOfEntriesOfLedger; import org.apache.bookkeeper.util.IOUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.zookeeper.AsyncCallback; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -1128,17 +1129,29 @@ private ArrayList replaceBookiesInEnsemble( * @param ledgerFragment * - LedgerFragment to replicate */ - public void replicateLedgerFragment(LedgerHandle lh, - final LedgerFragment ledgerFragment, - final BiConsumer onReadEntryFailureCallback) - throws InterruptedException, BKException { - Optional> excludedBookies = Optional.empty(); - Map targetBookieAddresses = - getReplacementBookiesByIndexes(lh, ledgerFragment.getEnsemble(), - ledgerFragment.getBookiesIndexes(), excludedBookies); + public void replicateLedgerFragment(LedgerHandle lh, final LedgerFragment ledgerFragment, + final BiConsumer onReadEntryFailureCallback) throws InterruptedException, BKException { + Map targetBookieAddresses = null; + if (LedgerFragment.ReplicateType.DATA_LOSS == ledgerFragment.getReplicateType()) { + Optional> excludedBookies = Optional.empty(); + targetBookieAddresses = getReplacementBookiesByIndexes(lh, ledgerFragment.getEnsemble(), + ledgerFragment.getBookiesIndexes(), excludedBookies); + + replicateLedgerFragment(lh, ledgerFragment, targetBookieAddresses, onReadEntryFailureCallback); + } else if (LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT == ledgerFragment.getReplicateType()) { + targetBookieAddresses = replaceNotAdheringPlacementPolicyBookie(ledgerFragment.getEnsemble(), + lh.getLedgerMetadata().getWriteQuorumSize(), lh.getLedgerMetadata().getAckQuorumSize()); + ledgerFragment.getBookiesIndexes().addAll(targetBookieAddresses.keySet()); + } + if (MapUtils.isEmpty(targetBookieAddresses)) { + LOG.warn("Could not replicate for {} ledger: {}, not find target bookie.", + ledgerFragment.getReplicateType(), ledgerFragment.getLedgerId()); + return; + } replicateLedgerFragment(lh, ledgerFragment, targetBookieAddresses, onReadEntryFailureCallback); + } - + private void replicateLedgerFragment(LedgerHandle lh, final LedgerFragment ledgerFragment, final Map targetBookieAddresses, @@ -1775,6 +1788,12 @@ public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List replaceNotAdheringPlacementPolicyBookie(List ensembleBookiesList, + int writeQuorumSize, int ackQuorumSize) { + return bkc.getPlacementPolicy() + .replaceNotAdheringPlacementPolicyBookie(ensembleBookiesList, writeQuorumSize, ackQuorumSize); + } /** * Makes async request for getting list of entries of ledger from a bookie diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java index 811766ba88d..79095be40b5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java @@ -214,7 +214,14 @@ public void updateBookieInfo(Map bookieInfoMap) { rwLock.writeLock().unlock(); } } - + + @Override + public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, + int ackQuorumSize) { + //Default ensemble placement policy always adhering. + return Collections.emptyMap(); + } + @Override public void uninitalize() { // do nothing diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index f2f5372eb6f..77d351c726c 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -363,6 +363,9 @@ DistributionSchedule.WriteSet reorderReadLACSequence( */ default void updateBookieInfo(Map bookieInfoMap) { } + + Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, + int ackQuorumSize); /** * Select one bookie to the "sticky" bookie where all reads for a particular diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java index f3a49d31a61..280b9526733 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java @@ -20,6 +20,8 @@ package org.apache.bookkeeper.client; import io.netty.buffer.ByteBuf; + +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -368,12 +370,12 @@ public void operationComplete(int rc, LedgerFragment result) { */ public void checkLedger(final LedgerHandle lh, final GenericCallback> cb) { - checkLedger(lh, cb, 0L); + checkLedger(lh, cb, 0L, Collections.emptySet()); } - + public void checkLedger(final LedgerHandle lh, final GenericCallback> cb, - long percentageOfLedgerFragmentToBeVerified) { + long percentageOfLedgerFragmentToBeVerified, Set ignoreFragments) { // build a set of all fragment replicas final Set fragments = new HashSet(); @@ -386,8 +388,9 @@ public void checkLedger(final LedgerHandle lh, for (int i = 0; i < curEnsemble.size(); i++) { bookieIndexes.add(i); } - fragments.add(new LedgerFragment(lh, curEntryId, - e.getKey() - 1, bookieIndexes)); + if (!isRepeatFragment(ignoreFragments, lh.getId(), curEntryId, e.getKey() - 1)) { + fragments.add(new LedgerFragment(lh, curEntryId, e.getKey() - 1, bookieIndexes)); + } } curEntryId = e.getKey(); curEnsemble = e.getValue(); @@ -430,7 +433,12 @@ public void checkLedger(final LedgerHandle lh, @Override public void operationComplete(int rc, Boolean result) { if (result) { - fragments.add(lastLedgerFragment); + if (!isRepeatFragment(ignoreFragments, + lastLedgerFragment.getLedgerId(), + lastLedgerFragment.getFirstEntryId(), + lastLedgerFragment.getLastKnownEntryId())) { + fragments.add(lastLedgerFragment); + } } checkFragments(fragments, cb, percentageOfLedgerFragmentToBeVerified); @@ -451,11 +459,26 @@ public void operationComplete(int rc, Boolean result) { writeSet.recycle(); return; } else { - fragments.add(lastLedgerFragment); + if (!isRepeatFragment(ignoreFragments, lastLedgerFragment.getLedgerId(), + lastLedgerFragment.getFirstEntryId(), lastLedgerFragment.getLastKnownEntryId())) { + fragments.add(lastLedgerFragment); + } } } checkFragments(fragments, cb, percentageOfLedgerFragmentToBeVerified); } + + private boolean isRepeatFragment(Set fragments, long ledgerId, long firstEntryId, long lastEntryId) { + boolean repeat = false; + for (LedgerFragment fragment : fragments) { + if (fragment.getLedgerId() == ledgerId && fragment.getFirstEntryId() == firstEntryId + && fragment.getLastKnownEntryId() == lastEntryId) { + repeat = true; + break; + } + } + return repeat; + } private void checkFragments(Set fragments, GenericCallback> cb, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java index 1fb1e50cb02..95309ff3d91 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java @@ -39,8 +39,9 @@ public class LedgerFragment { private final long ledgerId; private final DistributionSchedule schedule; private final boolean isLedgerClosed; + private ReplicateType replicateType = ReplicateType.DATA_LOSS; - LedgerFragment(LedgerHandle lh, + public LedgerFragment(LedgerHandle lh, long firstEntryId, long lastKnownEntryId, Set bookieIndexes) { @@ -56,7 +57,7 @@ public class LedgerFragment { || !ensemble.equals(ensembles.get(ensembles.lastKey())); } - LedgerFragment(LedgerFragment lf, Set subset) { + public LedgerFragment(LedgerFragment lf, Set subset) { this.ledgerId = lf.ledgerId; this.firstEntryId = lf.firstEntryId; this.lastKnownEntryId = lf.lastKnownEntryId; @@ -65,7 +66,7 @@ public class LedgerFragment { this.schedule = lf.schedule; this.isLedgerClosed = lf.isLedgerClosed; } - + /** * Return a ledger fragment contains subset of bookies. * @@ -216,6 +217,14 @@ public boolean isStoredEntryId(long entryId, int bookieIndex) { public List getEnsemble() { return this.ensemble; } + + public ReplicateType getReplicateType() { + return replicateType; + } + + public void setReplicateType(ReplicateType replicateType) { + this.replicateType = replicateType; + } @Override public String toString() { @@ -224,4 +233,9 @@ public String toString() { getFirstStoredEntryId(), lastKnownEntryId, getLastStoredEntryId(), getAddresses(), isLedgerClosed); } + + public enum ReplicateType { + DATA_LOSS, + DATA_NOT_ADHERING_PLACEMENT + } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index a1796efea16..82e7ae5d492 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -32,13 +32,18 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import com.google.common.graph.Network; import org.apache.bookkeeper.client.BookieInfoReader.BookieInfo; import org.apache.bookkeeper.client.WeightedRandomSelection.WeightedObject; import org.apache.bookkeeper.net.BookieId; @@ -54,6 +59,7 @@ import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.OpStatsLogger; import org.apache.bookkeeper.stats.annotations.StatsDoc; +import org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -789,7 +795,99 @@ public void updateBookieInfo(Map bookieInfoMap) { rwLock.writeLock().unlock(); } } - + + @Override + public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, + int ackQuorumSize) { + Map bookieIndex = new HashMap<>(); + for (int i = 0; i < ensemble.size(); i++) { + bookieIndex.put(ensemble.get(i), i); + } + + Map clone = new HashMap<>(knownBookies); + + Map> toPlaceGroup = new HashMap<>(); + for (BookieId bookieId : ensemble) { + //If the bookieId shutdown, put it to inactive. + BookieNode bookieNode = clone.get(bookieId); + if (bookieNode == null) { + List list = toPlaceGroup.computeIfAbsent(NetworkTopology.INACTIVE, k -> new ArrayList<>()); + list.add(new BookieNode(bookieId, NetworkTopology.INACTIVE)); + } else { + List list = toPlaceGroup.computeIfAbsent(bookieNode.getNetworkLocation(), + k -> new ArrayList<>()); + list.add(bookieNode); + } + } + + Map> knowsGroup = clone.values().stream() + .collect(Collectors.groupingBy(NodeBase::getNetworkLocation)); + + for (String key : toPlaceGroup.keySet()) { + knowsGroup.remove(key); + } + + if (knowsGroup.isEmpty()) { + return Collections.emptyMap(); + } + Map targetBookieAddresses = new HashMap<>(); + boolean placeSucceed = false; + while (knowsGroup.size() > 0) { + doReplaceNotAdhering(toPlaceGroup, knowsGroup, targetBookieAddresses, bookieIndex); + List ensembles = toPlaceGroup.values().stream().flatMap(Collection::stream).map( + BookieNode::getAddr).collect(Collectors.toList()); + PlacementPolicyAdherence ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(ensembles, + writeQuorumSize, ackQuorumSize); + if (PlacementPolicyAdherence.FAIL != ensembleAdheringToPlacementPolicy) { + placeSucceed = true; + break; + } + } + + if (!placeSucceed) { + targetBookieAddresses.clear(); + return Collections.emptyMap(); + } + return targetBookieAddresses; + } + + private void doReplaceNotAdhering(Map> toPlaceGroup, Map> knowsGroup, + Map targetBookieAddresses, Map bookieIndex) { + if (knowsGroup.isEmpty()) { + return; + } + List inactiveNodes = toPlaceGroup.get(NetworkTopology.INACTIVE); + if (!CollectionUtils.isEmpty(inactiveNodes)) { + BookieNode beReplaced = inactiveNodes.remove(inactiveNodes.size() - 1); + Integer index = bookieIndex.get(beReplaced.getAddr()); + Iterator>> iterator = knowsGroup.entrySet().iterator(); + if (iterator.hasNext()) { + Map.Entry> next = iterator.next(); + List list = toPlaceGroup.computeIfAbsent(next.getKey(), k -> new ArrayList<>()); + BookieNode toReplaced = new BookieNode(next.getValue().get(0).getAddr(), next.getValue().get(0).getNetworkLocation()); + list.add(toReplaced); + targetBookieAddresses.put(index, toReplaced.getAddr()); + iterator.remove(); + } + return; + } + Optional>> toPlaceEntry = toPlaceGroup.entrySet().stream() + .filter(ele -> ele.getValue().size() > 1).findAny(); + if (!toPlaceEntry.isPresent()) { + return; + } + + Map.Entry> entry = toPlaceEntry.get(); + entry.getValue().remove(entry.getValue().size() - 1); + Iterator>> iterator = knowsGroup.entrySet().iterator(); + if (iterator.hasNext()) { + Map.Entry> next = iterator.next(); + List list = toPlaceGroup.computeIfAbsent(next.getKey(), k -> new ArrayList<>()); + list.add(new BookieNode(next.getValue().get(0).getAddr(), next.getValue().get(0).getNetworkLocation())); + iterator.remove(); + } + } + protected BookieNode createBookieNode(BookieId addr) { return new BookieNode(addr, resolveNetworkLocation(addr)); } @@ -814,9 +912,9 @@ protected String resolveNetworkLocation(BookieId addr) { } } - protected Set convertBookiesToNodes(Collection excludeBookies) { + protected Set convertBookiesToNodes(Collection bookies) { Set nodes = new HashSet(); - for (BookieId addr : excludeBookies) { + for (BookieId addr : bookies) { BookieNode bn = knownBookies.get(addr); if (null == bn) { bn = createBookieNode(addr); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 3101dbbe37b..04c7414706f 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -202,6 +202,7 @@ public class ServerConfiguration extends AbstractConfiguration { + if (e != null) { + LOG.error("For ledger: {}, the placement policy not adhering bookie " + + "storage, mark it to under replication manager failed.", ledgerId, e); + return; + } + if (LOG.isDebugEnabled()) { + LOG.debug("For ledger: {}, the placement policy not adhering bookie" + + " storage, mark it to under replication manager", ledgerId); + } + }); + } } else if (foundSegmentSoftlyAdheringToPlacementPolicy) { numOfLedgersFoundSoftlyAdheringInPlacementPolicyCheck.incrementAndGet(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java index 721380a6561..2b62223577e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java @@ -34,7 +34,9 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.NavigableSet; @@ -42,6 +44,7 @@ import java.util.SortedMap; import java.util.Timer; import java.util.TimerTask; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -53,12 +56,15 @@ import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.EnsemblePlacementPolicy; import org.apache.bookkeeper.client.LedgerChecker; import org.apache.bookkeeper.client.LedgerFragment; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback; @@ -69,6 +75,7 @@ import org.apache.bookkeeper.stats.OpStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.stats.annotations.StatsDoc; +import org.apache.bookkeeper.versioning.Versioned; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,6 +108,7 @@ public class ReplicationWorker implements Runnable { private final long lockReleaseOfFailedLedgerGracePeriod; private final long baseBackoffForLockReleaseOfFailedLedger; private final BiConsumer onReadEntryFailureCallback; + private final LedgerManager ledgerManager; // Expose Stats private final StatsLogger statsLogger; @@ -176,6 +184,7 @@ public ReplicationWorker(final ServerConfiguration conf, this.ownBkc = ownBkc; this.underreplicationManager = bkc.getLedgerManagerFactory().newLedgerUnderreplicationManager(); + this.ledgerManager = bkc.getLedgerManagerFactory().newLedgerManager(); this.admin = new BookKeeperAdmin(bkc, statsLogger, new ClientConfiguration(conf)); this.ledgerChecker = new LedgerChecker(bkc); this.workerThread = new BookieThread(this, "ReplicationWorker"); @@ -356,6 +365,52 @@ private boolean tryReadingFaultyEntries(LedgerHandle lh, LedgerFragment ledgerFr } return (returnRCValue.get() == BKException.Code.OK); } + + private Set checkLedgerNeedRepairedPlacementNotAdheringBookie(long ledgerId, LedgerHandle lh) { + if (!conf.getRepairedPlacementPolicyNotAdheringBookieEnable()) { + return Collections.emptySet(); + } + Set placementNotAdheringFragments = new HashSet<>(); + CompletableFuture> future = ledgerManager.readLedgerMetadata( + ledgerId).whenComplete((metadataVer, exception) -> { + if (exception == null) { + LedgerMetadata metadata = metadataVer.getValue(); + int writeQuorumSize = metadata.getWriteQuorumSize(); + int ackQuorumSize = metadata.getAckQuorumSize(); + if (!metadata.isClosed()) { + return; + } + Long curEntryId = null; + List curEnsemble = null; + for (Map.Entry> entry : metadata.getAllEnsembles().entrySet()) { + if (curEntryId != null) { + EnsemblePlacementPolicy.PlacementPolicyAdherence segmentAdheringToPlacementPolicy + = admin.isEnsembleAdheringToPlacementPolicy(curEnsemble, writeQuorumSize, ackQuorumSize); + if (segmentAdheringToPlacementPolicy == EnsemblePlacementPolicy.PlacementPolicyAdherence.FAIL) { + LedgerFragment ledgerFragment = new LedgerFragment(lh, curEntryId, entry.getKey() - 1, + Collections.emptySet()); + ledgerFragment.setReplicateType(LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT); + placementNotAdheringFragments.add(ledgerFragment); + } + } + curEntryId = entry.getKey(); + curEnsemble = entry.getValue(); + } + } else if (BKException.getExceptionCode(exception) + == BKException.Code.NoSuchLedgerExistsOnMetadataServerException) { + LOG.debug("Ignoring replication of already deleted ledger {}", ledgerId); + } else { + LOG.warn("Unable to read the ledger: {} information", ledgerId); + } + }); + try { + FutureUtils.result(future); + } catch (Exception e) { + LOG.warn("Check ledger need repaired placement not adhering bookie failed", e); + return Collections.emptySet(); + } + return placementNotAdheringFragments; + } @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE") private boolean rereplicate(long ledgerIdToReplicate) throws InterruptedException, BKException, @@ -367,8 +422,12 @@ private boolean rereplicate(long ledgerIdToReplicate) throws InterruptedExceptio boolean deferLedgerLockRelease = false; try (LedgerHandle lh = admin.openLedgerNoRecovery(ledgerIdToReplicate)) { + + Set ledgerFragments = + checkLedgerNeedRepairedPlacementNotAdheringBookie(ledgerIdToReplicate, lh); + Set fragments = - getUnderreplicatedFragments(lh, conf.getAuditorLedgerVerificationPercentage()); + getUnderreplicatedFragments(lh, conf.getAuditorLedgerVerificationPercentage(), ledgerFragments); if (LOG.isDebugEnabled()) { LOG.debug("Founds fragments {} for replication from ledger: {}", fragments, ledgerIdToReplicate); @@ -498,10 +557,17 @@ private boolean isLastSegmentOpenAndMissingBookies(LedgerHandle lh) throws BKExc /** * Gets the under replicated fragments. */ - private Set getUnderreplicatedFragments(LedgerHandle lh, Long ledgerVerificationPercentage) - throws InterruptedException { + private Set getUnderreplicatedFragments(LedgerHandle lh, Long ledgerVerificationPercentage) throws InterruptedException { + return getUnderreplicatedFragments(lh, ledgerVerificationPercentage, Collections.emptySet()); + } + + /** + * Gets the under replicated fragments. + */ + private Set getUnderreplicatedFragments(LedgerHandle lh, Long ledgerVerificationPercentage, + Set ignoreFragments) throws InterruptedException { CheckerCallback checkerCb = new CheckerCallback(); - ledgerChecker.checkLedger(lh, checkerCb, ledgerVerificationPercentage); + ledgerChecker.checkLedger(lh, checkerCb, ledgerVerificationPercentage, ignoreFragments); Set fragments = checkerCb.waitAndGetResult(); return fragments; } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java index 7608e8e1cda..058c1d96847 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java @@ -292,6 +292,149 @@ public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicy() th } } } + + @Test + public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndNotMarkToUnderreplication() throws Exception { + int numOfBookies = 5; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList<>(); + try (RegistrationManager regManager = driver.createRegistrationManager()) { + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + } + + // only three racks + StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); + StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2"); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 3; + + /* + * this closed ledger doesn't adhere to placement policy because there are only + * 3 racks, and the ensembleSize is 5. + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + } + LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager(); + long unnderReplicateLedgerId = underreplicationManager.pollLedgerToRereplicate(); + assertEquals(unnderReplicateLedgerId, -1); + } + + @Test + public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndMarkToUnderreplication() throws Exception { + int numOfBookies = 5; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList<>(); + try (RegistrationManager regManager = driver.createRegistrationManager()) { + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + } + + // only three racks + StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); + StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2"); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 3; + + /* + * this closed ledger doesn't adhere to placement policy because there are only + * 3 racks, and the ensembleSize is 5. + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + servConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + } + LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager(); + long unnderReplicateLedgerId = underreplicationManager.pollLedgerToRereplicate(); + assertEquals(unnderReplicateLedgerId, 1L); + } @Test public void testPlacementPolicyCheckForURLedgersElapsedRecoveryGracePeriod() throws Exception { From 0108c9873ee8658f97165b03eddbc533c266b499 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Sat, 25 Jun 2022 12:12:24 +0800 Subject: [PATCH 02/35] add unit test. --- .../client/EnsemblePlacementPolicy.java | 8 + .../TopologyAwareEnsemblePlacementPolicy.java | 8 +- .../replication/ReplicationWorker.java | 39 ++--- .../TestRackawareEnsemblePlacementPolicy.java | 153 +++++++++++++++++- .../replication/TestReplicationWorker.java | 108 +++++++++++++ 5 files changed, 295 insertions(+), 21 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index 77d351c726c..1c18259bde2 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -364,6 +364,14 @@ DistributionSchedule.WriteSet reorderReadLACSequence( default void updateBookieInfo(Map bookieInfoMap) { } + /** + * Replace some bookie to adhering placement policy. + * + * @param ensemble + * @param writeQuorumSize + * @param ackQuorumSize + * @return Map: key means ensemble index, value means target replace bookieId. + */ Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, int ackQuorumSize); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index 82e7ae5d492..42228e0af9e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -878,12 +878,16 @@ private void doReplaceNotAdhering(Map> toPlaceGroup, Ma } Map.Entry> entry = toPlaceEntry.get(); - entry.getValue().remove(entry.getValue().size() - 1); + BookieNode beReplaced = entry.getValue().remove(entry.getValue().size() - 1); + Integer index = bookieIndex.get(beReplaced.getAddr()); + Iterator>> iterator = knowsGroup.entrySet().iterator(); if (iterator.hasNext()) { Map.Entry> next = iterator.next(); List list = toPlaceGroup.computeIfAbsent(next.getKey(), k -> new ArrayList<>()); - list.add(new BookieNode(next.getValue().get(0).getAddr(), next.getValue().get(0).getNetworkLocation())); + BookieNode toReplaced = new BookieNode(next.getValue().get(0).getAddr(), next.getValue().get(0).getNetworkLocation()); + list.add(toReplaced); + targetBookieAddresses.put(index, toReplaced.getAddr()); iterator.remove(); } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java index 2b62223577e..84b10390217 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java @@ -366,10 +366,11 @@ private boolean tryReadingFaultyEntries(LedgerHandle lh, LedgerFragment ledgerFr return (returnRCValue.get() == BKException.Code.OK); } - private Set checkLedgerNeedRepairedPlacementNotAdheringBookie(long ledgerId, LedgerHandle lh) { + private Set getNeedRepairedPlacementNotAdheringFragments(LedgerHandle lh) { if (!conf.getRepairedPlacementPolicyNotAdheringBookieEnable()) { return Collections.emptySet(); } + long ledgerId = lh.getId(); Set placementNotAdheringFragments = new HashSet<>(); CompletableFuture> future = ledgerManager.readLedgerMetadata( ledgerId).whenComplete((metadataVer, exception) -> { @@ -396,6 +397,18 @@ private Set checkLedgerNeedRepairedPlacementNotAdheringBookie(lo curEntryId = entry.getKey(); curEnsemble = entry.getValue(); } + if (curEntryId != null) { + long lastEntry = lh.getLastAddConfirmed(); + + if (!lh.isClosed() && lastEntry < curEntryId) { + lastEntry = curEntryId; + } + + LedgerFragment ledgerFragment = new LedgerFragment(lh, curEntryId, lastEntry, + Collections.emptySet()); + ledgerFragment.setReplicateType(LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT); + placementNotAdheringFragments.add(ledgerFragment); + } } else if (BKException.getExceptionCode(exception) == BKException.Code.NoSuchLedgerExistsOnMetadataServerException) { LOG.debug("Ignoring replication of already deleted ledger {}", ledgerId); @@ -411,7 +424,7 @@ private Set checkLedgerNeedRepairedPlacementNotAdheringBookie(lo } return placementNotAdheringFragments; } - + @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE") private boolean rereplicate(long ledgerIdToReplicate) throws InterruptedException, BKException, UnavailableException { @@ -422,12 +435,8 @@ private boolean rereplicate(long ledgerIdToReplicate) throws InterruptedExceptio boolean deferLedgerLockRelease = false; try (LedgerHandle lh = admin.openLedgerNoRecovery(ledgerIdToReplicate)) { - - Set ledgerFragments = - checkLedgerNeedRepairedPlacementNotAdheringBookie(ledgerIdToReplicate, lh); - - Set fragments = - getUnderreplicatedFragments(lh, conf.getAuditorLedgerVerificationPercentage(), ledgerFragments); + Set fragments = getUnderreplicatedFragments(lh, + conf.getAuditorLedgerVerificationPercentage()); if (LOG.isDebugEnabled()) { LOG.debug("Founds fragments {} for replication from ledger: {}", fragments, ledgerIdToReplicate); @@ -558,20 +567,14 @@ private boolean isLastSegmentOpenAndMissingBookies(LedgerHandle lh) throws BKExc * Gets the under replicated fragments. */ private Set getUnderreplicatedFragments(LedgerHandle lh, Long ledgerVerificationPercentage) throws InterruptedException { - return getUnderreplicatedFragments(lh, ledgerVerificationPercentage, Collections.emptySet()); - } - - /** - * Gets the under replicated fragments. - */ - private Set getUnderreplicatedFragments(LedgerHandle lh, Long ledgerVerificationPercentage, - Set ignoreFragments) throws InterruptedException { + Set notAdheringFragments = getNeedRepairedPlacementNotAdheringFragments(lh); CheckerCallback checkerCb = new CheckerCallback(); - ledgerChecker.checkLedger(lh, checkerCb, ledgerVerificationPercentage, ignoreFragments); + ledgerChecker.checkLedger(lh, checkerCb, ledgerVerificationPercentage, notAdheringFragments); Set fragments = checkerCb.waitAndGetResult(); + fragments.addAll(notAdheringFragments); return fragments; } - + void scheduleTaskWithDelay(TimerTask timerTask, long delayPeriod) { pendingReplicationTimer.schedule(timerTask, delayPeriod); } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index 568fdfc2f24..f17bf34e1ab 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -1416,7 +1416,158 @@ public boolean validate() { // this is expected } } - + + @Test + public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { + repp.uninitalize(); + + int minNumRacksPerWriteQuorum = 3; + ClientConfiguration clientConf = new ClientConfiguration(conf); + clientConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorum); + // set enforceMinNumRacksPerWriteQuorum + clientConf.setEnforceMinNumRacksPerWriteQuorum(true); + repp = new RackawareEnsemblePlacementPolicy(); + repp.initialize(clientConf, Optional. empty(), timer, DISABLE_ALL, + NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); + + int numOfRacks = 3; + int numOfBookiesPerRack = 3; + String[] rackLocationNames = new String[numOfRacks]; + List bookieSocketAddresses = new ArrayList(); + Map bookieRackMap = new HashMap(); + BookieId bookieAddress; + + for (int i = 0; i < numOfRacks; i++) { + rackLocationNames[i] = "/default-region/r" + i; + for (int j = 0; j < numOfBookiesPerRack; j++) { + int index = i * numOfBookiesPerRack + j; + bookieAddress = new BookieSocketAddress("128.0.0." + index, 3181).toBookieId(); + StaticDNSResolver.addNodeToRack("128.0.0." + index, rackLocationNames[i]); + bookieSocketAddresses.add(bookieAddress); + bookieRackMap.put(bookieAddress, rackLocationNames[i]); + } + } + + repp.onClusterChanged(new HashSet(bookieSocketAddresses), new HashSet()); + + int writeQuorum = 3; + int ackQuorum = 3; + + //test three knows bookie + List knowsEnsemble = new ArrayList<>(); + knowsEnsemble.add(BookieId.parse("128.0.0.0:3181")); + knowsEnsemble.add(BookieId.parse("128.0.0.1:3181")); + knowsEnsemble.add(BookieId.parse("128.0.0.2:3181")); + + PlacementPolicyAdherence placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( + knowsEnsemble, writeQuorum, ackQuorum); + assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); + + Map targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum); + //should replace two bookie + assertEquals(targetBookie.size(), 2); + + for (Map.Entry entry : targetBookie.entrySet()) { + knowsEnsemble.set(entry.getKey(), entry.getValue()); + } + + placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( + knowsEnsemble, writeQuorum, ackQuorum); + assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); + + //test three unknowns bookie + List unknownEnsembles = new ArrayList<>(); + unknownEnsembles.add(BookieId.parse("128.0.0.100:3181")); + unknownEnsembles.add(BookieId.parse("128.0.0.101:3181")); + unknownEnsembles.add(BookieId.parse("128.0.0.102:3181")); + + placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( + unknownEnsembles, writeQuorum, ackQuorum); + assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); + + //should replace three bookie + targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, ackQuorum, writeQuorum); + assertEquals(targetBookie.size(), 3); + + for (Map.Entry entry : targetBookie.entrySet()) { + unknownEnsembles.set(entry.getKey(), entry.getValue()); + } + + placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( + unknownEnsembles, writeQuorum, ackQuorum); + assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); + } + + @Test + public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() throws Exception { + repp.uninitalize(); + + int minNumRacksPerWriteQuorum = 3; + ClientConfiguration clientConf = new ClientConfiguration(conf); + clientConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorum); + // set enforceMinNumRacksPerWriteQuorum + clientConf.setEnforceMinNumRacksPerWriteQuorum(true); + repp = new RackawareEnsemblePlacementPolicy(); + repp.initialize(clientConf, Optional. empty(), timer, DISABLE_ALL, + NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); + + int numOfRacks = 2; + int numOfBookiesPerRack = 3; + String[] rackLocationNames = new String[numOfRacks]; + List bookieSocketAddresses = new ArrayList(); + Map bookieRackMap = new HashMap(); + BookieId bookieAddress; + + for (int i = 0; i < numOfRacks; i++) { + rackLocationNames[i] = "/default-region/r" + i; + for (int j = 0; j < numOfBookiesPerRack; j++) { + int index = i * numOfBookiesPerRack + j; + bookieAddress = new BookieSocketAddress("128.0.0." + index, 3181).toBookieId(); + StaticDNSResolver.addNodeToRack("128.0.0." + index, rackLocationNames[i]); + bookieSocketAddresses.add(bookieAddress); + bookieRackMap.put(bookieAddress, rackLocationNames[i]); + } + } + + repp.onClusterChanged(new HashSet(bookieSocketAddresses), + new HashSet()); + + + int writeQuorum = 3; + int ackQuorum = 3; + + //test three knows bookie + List knowsEnsemble = new ArrayList<>(); + knowsEnsemble.add(BookieId.parse("128.0.0.0:3181")); + knowsEnsemble.add(BookieId.parse("128.0.0.1:3181")); + knowsEnsemble.add(BookieId.parse("128.0.0.2:3181")); + + PlacementPolicyAdherence placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( + knowsEnsemble, writeQuorum, ackQuorum); + assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); + + Map targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum); + //should not replace + assertEquals(targetBookie.size(), 0); + + //test three unknowns bookie + List unknownEnsembles = new ArrayList<>(); + unknownEnsembles.add(BookieId.parse("128.0.0.100:3181")); + unknownEnsembles.add(BookieId.parse("128.0.0.101:3181")); + unknownEnsembles.add(BookieId.parse("128.0.0.102:3181")); + + placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( + unknownEnsembles, writeQuorum, ackQuorum); + assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); + + //should not replace + targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, ackQuorum, writeQuorum); + assertEquals(targetBookie.size(), 0); + } + + @Test public void testNewEnsembleWithMultipleRacks() throws Exception { BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java index bb302350a15..442ff700fbe 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -19,6 +19,7 @@ */ package org.apache.bookkeeper.replication; +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -28,23 +29,29 @@ import java.io.IOException; import java.net.URI; +import java.net.UnknownHostException; import java.util.Enumeration; import java.util.List; import java.util.Map.Entry; import java.util.Optional; import java.util.TimerTask; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import lombok.Cleanup; +import org.apache.bookkeeper.bookie.BookieImpl; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.ClientUtil; import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.meta.AbstractZkLedgerManager; import org.apache.bookkeeper.meta.LedgerManager; @@ -54,19 +61,23 @@ import org.apache.bookkeeper.meta.MetadataClientDriver; import org.apache.bookkeeper.meta.MetadataDrivers; import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.exceptions.MetadataException; import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException; import org.apache.bookkeeper.stats.Counter; +import org.apache.bookkeeper.stats.Gauge; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.test.TestStatsProvider; import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; import org.apache.bookkeeper.util.BookKeeperConstants; +import org.apache.bookkeeper.util.StaticDNSResolver; import org.apache.bookkeeper.zookeeper.BoundExponentialBackoffRetryPolicy; import org.apache.bookkeeper.zookeeper.ZooKeeperClient; import org.apache.bookkeeper.zookeeper.ZooKeeperWatcherBase; +import org.apache.commons.lang3.mutable.MutableObject; import org.apache.zookeeper.AsyncCallback.StatCallback; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; @@ -1077,4 +1088,101 @@ public void testRWShutDownInTheCaseOfZKOperationFailures() throws Exception { bkWithMockZK.close(); } } + + @Test + public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsWithNoMoreRackBookie() throws Exception { + for (int i = 0; i < 2; i++) { + startNewBookie(); + } + assertEquals(servers.size(), 5); + + for (int i = 0; i < servers.size(); i++) { + //0,1,2 at rack1 + if (i < 3) { + StaticDNSResolver.addNodeToRack(servers.get(i).getServer().getLocalAddress().getHostName(), "/rack1"); + } else { + StaticDNSResolver.addNodeToRack(servers.get(i).getServer().getLocalAddress().getHostName(), "/rack2"); + } + //3,4 at rack2 + } + //This ledger not adhering placement policy, the combine(0,1,2) rack is 1. + LedgerHandle lh = bkc.createLedger(5, 3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD); + + for (int i = 0; i < 100; i++) { + lh.addEntry(data); + } + lh.close(); + + int minNumRacksPerWriteQuorumConfValue = 2; + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + servConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName()); + servConf.setProperty(ClientConfiguration.ENSEMBLE_PLACEMENT_POLICY, RackawareEnsemblePlacementPolicy.class.getName()); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1000); + servConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); + + + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + 1, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + } + + baseConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + baseConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName()); + baseConf.setProperty(ClientConfiguration.ENSEMBLE_PLACEMENT_POLICY, RackawareEnsemblePlacementPolicy.class.getName()); + baseConf.setAuditorPeriodicPlacementPolicyCheckInterval(1000); + baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); + baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); + ReplicationWorker rw = new ReplicationWorker(baseConf); + rw.start(); + System.in.read(); + } + + private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfiguration servConf, + MutableObject auditorRef) throws MetadataException, CompatibilityException, KeeperException, + InterruptedException, ReplicationException.UnavailableException, UnknownHostException { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestStatsProvider.TestOpStatsLogger placementPolicyCheckStatsLogger = (TestStatsProvider.TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME); + + final AuditorPeriodicCheckTest.TestAuditor auditor = new AuditorPeriodicCheckTest.TestAuditor( + BookieImpl.getBookieId(servConf).toString(), servConf, + statsLogger); + auditorRef.setValue(auditor); + CountDownLatch latch = auditor.getLatch(); + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, placementPolicyCheckStatsLogger.getSuccessCount()); + urm.setPlacementPolicyCheckCTime(-1); + auditor.start(); + /* + * since placementPolicyCheckCTime is set to -1, placementPolicyCheck should be + * scheduled to run with no initialdelay + */ + assertTrue("placementPolicyCheck should have executed", latch.await(20, TimeUnit.SECONDS)); + for (int i = 0; i < 20; i++) { + Thread.sleep(100); + if (placementPolicyCheckStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1, placementPolicyCheckStatsLogger.getSuccessCount()); + return statsLogger; + } + } From f7a3b4931661ae432140ef496d24615e16012f4e Mon Sep 17 00:00:00 2001 From: horizonzy Date: Sat, 25 Jun 2022 12:16:44 +0800 Subject: [PATCH 03/35] code clean. --- .../org/apache/bookkeeper/replication/ReplicationWorker.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java index 84b10390217..1e25954fcb9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java @@ -399,11 +399,6 @@ private Set getNeedRepairedPlacementNotAdheringFragments(LedgerH } if (curEntryId != null) { long lastEntry = lh.getLastAddConfirmed(); - - if (!lh.isClosed() && lastEntry < curEntryId) { - lastEntry = curEntryId; - } - LedgerFragment ledgerFragment = new LedgerFragment(lh, curEntryId, lastEntry, Collections.emptySet()); ledgerFragment.setReplicateType(LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT); From dd8d7793d1913b6d489a556a61a219f4a9f8015d Mon Sep 17 00:00:00 2001 From: horizonzy Date: Sat, 25 Jun 2022 12:23:55 +0800 Subject: [PATCH 04/35] tuning doc. --- .../org/apache/bookkeeper/client/EnsemblePlacementPolicy.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index 1c18259bde2..b5f37cd01ce 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -365,7 +365,8 @@ default void updateBookieInfo(Map bookieInfoMap) { } /** - * Replace some bookie to adhering placement policy. + * Replace some bookie to adhering placement policy. If the all kinds of replacement + * didn't adhere placement policy, return empty map. * * @param ensemble * @param writeQuorumSize From 4a66b587963d7c1e5bfec56f901bd89e9b2edd7b Mon Sep 17 00:00:00 2001 From: horizonzy Date: Sun, 26 Jun 2022 14:35:47 +0800 Subject: [PATCH 05/35] code tuning --- .../bookkeeper/client/BookKeeperAdmin.java | 2 - .../bookkeeper/client/LedgerChecker.java | 32 ++--------- .../bookkeeper/client/LedgerFragment.java | 2 +- .../RackawareEnsemblePlacementPolicyImpl.java | 3 ++ .../TopologyAwareEnsemblePlacementPolicy.java | 13 +++-- .../meta/AbstractZkLedgerManager.java | 15 ++++++ .../bookkeeper/replication/Auditor.java | 2 +- .../replication/ReplicationWorker.java | 53 ++++++++++++++----- .../TestRackawareEnsemblePlacementPolicy.java | 15 ++++++ 9 files changed, 89 insertions(+), 48 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java index a0ec1e87374..f2e67bad5b4 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java @@ -1136,8 +1136,6 @@ public void replicateLedgerFragment(LedgerHandle lh, final LedgerFragment ledger Optional> excludedBookies = Optional.empty(); targetBookieAddresses = getReplacementBookiesByIndexes(lh, ledgerFragment.getEnsemble(), ledgerFragment.getBookiesIndexes(), excludedBookies); - - replicateLedgerFragment(lh, ledgerFragment, targetBookieAddresses, onReadEntryFailureCallback); } else if (LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT == ledgerFragment.getReplicateType()) { targetBookieAddresses = replaceNotAdheringPlacementPolicyBookie(ledgerFragment.getEnsemble(), lh.getLedgerMetadata().getWriteQuorumSize(), lh.getLedgerMetadata().getAckQuorumSize()); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java index 280b9526733..76e0c01aed5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java @@ -370,12 +370,12 @@ public void operationComplete(int rc, LedgerFragment result) { */ public void checkLedger(final LedgerHandle lh, final GenericCallback> cb) { - checkLedger(lh, cb, 0L, Collections.emptySet()); + checkLedger(lh, cb, 0L); } public void checkLedger(final LedgerHandle lh, final GenericCallback> cb, - long percentageOfLedgerFragmentToBeVerified, Set ignoreFragments) { + long percentageOfLedgerFragmentToBeVerified) { // build a set of all fragment replicas final Set fragments = new HashSet(); @@ -388,9 +388,7 @@ public void checkLedger(final LedgerHandle lh, for (int i = 0; i < curEnsemble.size(); i++) { bookieIndexes.add(i); } - if (!isRepeatFragment(ignoreFragments, lh.getId(), curEntryId, e.getKey() - 1)) { - fragments.add(new LedgerFragment(lh, curEntryId, e.getKey() - 1, bookieIndexes)); - } + fragments.add(new LedgerFragment(lh, curEntryId, e.getKey() - 1, bookieIndexes)); } curEntryId = e.getKey(); curEnsemble = e.getValue(); @@ -433,12 +431,7 @@ public void checkLedger(final LedgerHandle lh, @Override public void operationComplete(int rc, Boolean result) { if (result) { - if (!isRepeatFragment(ignoreFragments, - lastLedgerFragment.getLedgerId(), - lastLedgerFragment.getFirstEntryId(), - lastLedgerFragment.getLastKnownEntryId())) { - fragments.add(lastLedgerFragment); - } + fragments.add(lastLedgerFragment); } checkFragments(fragments, cb, percentageOfLedgerFragmentToBeVerified); @@ -459,27 +452,12 @@ public void operationComplete(int rc, Boolean result) { writeSet.recycle(); return; } else { - if (!isRepeatFragment(ignoreFragments, lastLedgerFragment.getLedgerId(), - lastLedgerFragment.getFirstEntryId(), lastLedgerFragment.getLastKnownEntryId())) { - fragments.add(lastLedgerFragment); - } + fragments.add(lastLedgerFragment); } } checkFragments(fragments, cb, percentageOfLedgerFragmentToBeVerified); } - private boolean isRepeatFragment(Set fragments, long ledgerId, long firstEntryId, long lastEntryId) { - boolean repeat = false; - for (LedgerFragment fragment : fragments) { - if (fragment.getLedgerId() == ledgerId && fragment.getFirstEntryId() == firstEntryId - && fragment.getLastKnownEntryId() == lastEntryId) { - repeat = true; - break; - } - } - return repeat; - } - private void checkFragments(Set fragments, GenericCallback> cb, long percentageOfLedgerFragmentToBeVerified) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java index 95309ff3d91..74568f4fbe2 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java @@ -92,7 +92,7 @@ public boolean isClosed() { return isLedgerClosed; } - long getLedgerId() { + public long getLedgerId() { return ledgerId; } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 46c5a10786b..c023b794035 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -1017,6 +1017,9 @@ DistributionSchedule.WriteSet reorderReadSequenceWithRegion( @Override public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List ensembleList, int writeQuorumSize, int ackQuorumSize) { + if (CollectionUtils.isEmpty(ensembleList)) { + return PlacementPolicyAdherence.FAIL; + } int ensembleSize = ensembleList.size(); int minNumRacksPerWriteQuorumForThisEnsemble = Math.min(writeQuorumSize, minNumRacksPerWriteQuorum); HashSet racksInQuorum = new HashSet(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index 42228e0af9e..892baf04ea0 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -795,10 +795,18 @@ public void updateBookieInfo(Map bookieInfoMap) { rwLock.writeLock().unlock(); } } - + @Override public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, int ackQuorumSize) { + if (CollectionUtils.isEmpty(ensemble)) { + return Collections.emptyMap(); + } + PlacementPolicyAdherence ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(ensemble, + writeQuorumSize, ackQuorumSize); + if (PlacementPolicyAdherence.FAIL != ensembleAdheringToPlacementPolicy) { + return Collections.emptyMap(); + } Map bookieIndex = new HashMap<>(); for (int i = 0; i < ensemble.size(); i++) { bookieIndex.put(ensemble.get(i), i); @@ -836,14 +844,13 @@ public Map replaceNotAdheringPlacementPolicyBookie(List ensembles = toPlaceGroup.values().stream().flatMap(Collection::stream).map( BookieNode::getAddr).collect(Collectors.toList()); - PlacementPolicyAdherence ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(ensembles, + ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(ensembles, writeQuorumSize, ackQuorumSize); if (PlacementPolicyAdherence.FAIL != ensembleAdheringToPlacementPolicy) { placeSucceed = true; break; } } - if (!placeSucceed) { targetBookieAddresses.clear(); return Collections.emptyMap(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java index cda93704e0a..82dfd3be4f8 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java @@ -385,6 +385,8 @@ public void processResult(int rc, String path, Object ctx) { return promise; } + //thread 2 registerLedgerMetadataListener (1, watcher2) + //thread 3 register (1, watcher3) @Override public void registerLedgerMetadataListener(long ledgerId, LedgerMetadataListener listener) { if (null != listener) { @@ -392,6 +394,7 @@ public void registerLedgerMetadataListener(long ledgerId, LedgerMetadataListener LOG.debug("Registered ledger metadata listener {} on ledger {}.", listener, ledgerId); } Set listenerSet = listeners.get(ledgerId); + //set(watcher1, watcher2) if (listenerSet == null) { Set newListenerSet = new HashSet(); Set oldListenerSet = listeners.putIfAbsent(ledgerId, newListenerSet); @@ -401,25 +404,37 @@ public void registerLedgerMetadataListener(long ledgerId, LedgerMetadataListener listenerSet = newListenerSet; } } + //set(watcher 2) synchronized (listenerSet) { + listenerSet = listeners.computeIfAbsent(ledgerId, k -> new HashSet<>()); listenerSet.add(listener); } new ReadLedgerMetadataTask(ledgerId).run(); } } + + //1 -> set(watcher 2) + //1 -> set + //thread 1 unregisterLedgerMetadataListener (1, watcher1) @Override public void unregisterLedgerMetadataListener(long ledgerId, LedgerMetadataListener listener) { Set listenerSet = listeners.get(ledgerId); + //set(watcher1) if (listenerSet != null) { + //set(watcher1, watcher2) + synchronized (listenerSet) { if (listenerSet.remove(listener)) { if (LOG.isDebugEnabled()) { LOG.debug("Unregistered ledger metadata listener {} on ledger {}.", listener, ledgerId); } } + //set() if (listenerSet.isEmpty()) { listeners.remove(ledgerId, listenerSet); + //1 -> set() + //listeners empty map } } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java index d0a5852983c..82ca81caec5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java @@ -1336,7 +1336,7 @@ void checkAllLedgers() throws BKException, IOException, InterruptedException, Ke checker.checkLedger(lh, // the ledger handle will be closed after checkLedger is done. new ProcessLostFragmentsCb(lh, callback), - conf.getAuditorLedgerVerificationPercentage(), Collections.emptySet()); + conf.getAuditorLedgerVerificationPercentage()); // we collect the following stats to get a measure of the // distribution of a single ledger within the bk cluster // the higher the number of fragments/bookies, the more distributed it is diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java index 1e25954fcb9..5fcd38b2567 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java @@ -382,27 +382,28 @@ private Set getNeedRepairedPlacementNotAdheringFragments(LedgerH return; } Long curEntryId = null; - List curEnsemble = null; + EnsemblePlacementPolicy.PlacementPolicyAdherence previousSegmentAdheringToPlacementPolicy = null; + for (Map.Entry> entry : metadata.getAllEnsembles().entrySet()) { if (curEntryId != null) { - EnsemblePlacementPolicy.PlacementPolicyAdherence segmentAdheringToPlacementPolicy - = admin.isEnsembleAdheringToPlacementPolicy(curEnsemble, writeQuorumSize, ackQuorumSize); - if (segmentAdheringToPlacementPolicy == EnsemblePlacementPolicy.PlacementPolicyAdherence.FAIL) { + if (previousSegmentAdheringToPlacementPolicy == EnsemblePlacementPolicy.PlacementPolicyAdherence.FAIL) { LedgerFragment ledgerFragment = new LedgerFragment(lh, curEntryId, entry.getKey() - 1, Collections.emptySet()); ledgerFragment.setReplicateType(LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT); placementNotAdheringFragments.add(ledgerFragment); } } + previousSegmentAdheringToPlacementPolicy = admin.isEnsembleAdheringToPlacementPolicy(entry.getValue(), writeQuorumSize, ackQuorumSize); curEntryId = entry.getKey(); - curEnsemble = entry.getValue(); } if (curEntryId != null) { - long lastEntry = lh.getLastAddConfirmed(); - LedgerFragment ledgerFragment = new LedgerFragment(lh, curEntryId, lastEntry, - Collections.emptySet()); - ledgerFragment.setReplicateType(LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT); - placementNotAdheringFragments.add(ledgerFragment); + if (previousSegmentAdheringToPlacementPolicy == EnsemblePlacementPolicy.PlacementPolicyAdherence.FAIL) { + long lastEntry = lh.getLedgerMetadata().getLastEntryId(); + LedgerFragment ledgerFragment = new LedgerFragment(lh, curEntryId, lastEntry, + Collections.emptySet()); + ledgerFragment.setReplicateType(LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT); + placementNotAdheringFragments.add(ledgerFragment); + } } } else if (BKException.getExceptionCode(exception) == BKException.Code.NoSuchLedgerExistsOnMetadataServerException) { @@ -562,12 +563,36 @@ private boolean isLastSegmentOpenAndMissingBookies(LedgerHandle lh) throws BKExc * Gets the under replicated fragments. */ private Set getUnderreplicatedFragments(LedgerHandle lh, Long ledgerVerificationPercentage) throws InterruptedException { + Set underreplicatedFragments = new HashSet<>(); + + Set dataLossFragments = getDataLossFragments(lh, ledgerVerificationPercentage); + underreplicatedFragments.addAll(dataLossFragments); + Set notAdheringFragments = getNeedRepairedPlacementNotAdheringFragments(lh); + + for (LedgerFragment notAdheringFragment : notAdheringFragments) { + if (!checkFragmentRepeat(underreplicatedFragments, notAdheringFragment)) { + underreplicatedFragments.add(notAdheringFragment); + } + } + return underreplicatedFragments; + } + + private Set getDataLossFragments(LedgerHandle lh, Long ledgerVerificationPercentage) throws InterruptedException { CheckerCallback checkerCb = new CheckerCallback(); - ledgerChecker.checkLedger(lh, checkerCb, ledgerVerificationPercentage, notAdheringFragments); - Set fragments = checkerCb.waitAndGetResult(); - fragments.addAll(notAdheringFragments); - return fragments; + ledgerChecker.checkLedger(lh, checkerCb, ledgerVerificationPercentage); + return checkerCb.waitAndGetResult(); + } + + private boolean checkFragmentRepeat(Set fragments, LedgerFragment needChecked) { + for (LedgerFragment fragment : fragments) { + if (fragment.getLedgerId() == needChecked.getLedgerId() + && fragment.getFirstEntryId() == needChecked.getFirstEntryId() + && fragment.getLastKnownEntryId() == needChecked.getLastKnownEntryId()) { + return true; + } + } + return false; } void scheduleTaskWithDelay(TimerTask timerTask, long delayPeriod) { diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index f17bf34e1ab..31cf8e1d84a 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -1497,6 +1497,21 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( unknownEnsembles, writeQuorum, ackQuorum); assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); + + + //test three knows bookie + List adheringEnsemble = new ArrayList<>(); + adheringEnsemble.add(BookieId.parse("128.0.0.0:3181")); + adheringEnsemble.add(BookieId.parse("128.0.0.3:3181")); + adheringEnsemble.add(BookieId.parse("128.0.0.6:3181")); + + placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( + adheringEnsemble, writeQuorum, ackQuorum); + assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); + + //should replace three bookie + targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(adheringEnsemble, ackQuorum, writeQuorum); + assertEquals(targetBookie.size(), 0); } @Test From 54338196c4bbd645d9d7c8f7d554d89215a0a0ff Mon Sep 17 00:00:00 2001 From: horizonzy Date: Mon, 27 Jun 2022 15:16:22 +0800 Subject: [PATCH 06/35] Complete the test case. --- .../apache/bookkeeper/client/BookKeeper.java | 2 +- .../TopologyAwareEnsemblePlacementPolicy.java | 23 ++-- .../replication/ReplicationWorker.java | 4 +- .../replication/TestReplicationWorker.java | 112 ++++++++++++++---- 4 files changed, 103 insertions(+), 38 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java index f08f47b98c6..4b6c075c7d5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java @@ -570,7 +570,7 @@ public BookKeeper(ClientConfiguration conf, ZooKeeper zk, EventLoopGroup eventLo bookieQuarantineRatio = 1.0; } - private EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, + protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, DNSToSwitchMapping dnsResolver, HashedWheelTimer timer, FeatureProvider featureProvider, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index 892baf04ea0..d24953a22f7 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -41,9 +41,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.Stream; -import com.google.common.graph.Network; import org.apache.bookkeeper.client.BookieInfoReader.BookieInfo; import org.apache.bookkeeper.client.WeightedRandomSelection.WeightedObject; import org.apache.bookkeeper.net.BookieId; @@ -866,26 +864,27 @@ private void doReplaceNotAdhering(Map> toPlaceGroup, Ma List inactiveNodes = toPlaceGroup.get(NetworkTopology.INACTIVE); if (!CollectionUtils.isEmpty(inactiveNodes)) { BookieNode beReplaced = inactiveNodes.remove(inactiveNodes.size() - 1); - Integer index = bookieIndex.get(beReplaced.getAddr()); - Iterator>> iterator = knowsGroup.entrySet().iterator(); - if (iterator.hasNext()) { - Map.Entry> next = iterator.next(); - List list = toPlaceGroup.computeIfAbsent(next.getKey(), k -> new ArrayList<>()); - BookieNode toReplaced = new BookieNode(next.getValue().get(0).getAddr(), next.getValue().get(0).getNetworkLocation()); - list.add(toReplaced); - targetBookieAddresses.put(index, toReplaced.getAddr()); - iterator.remove(); - } + doReplace(beReplaced, toPlaceGroup, knowsGroup, targetBookieAddresses, bookieIndex); return; } Optional>> toPlaceEntry = toPlaceGroup.entrySet().stream() .filter(ele -> ele.getValue().size() > 1).findAny(); if (!toPlaceEntry.isPresent()) { + Iterator>> iterator = toPlaceGroup.entrySet().stream().iterator(); + Map.Entry> entry = iterator.next(); + BookieNode beReplaced = entry.getValue().get(0); + iterator.remove(); + doReplace(beReplaced, toPlaceGroup, knowsGroup, targetBookieAddresses, bookieIndex); return; } Map.Entry> entry = toPlaceEntry.get(); BookieNode beReplaced = entry.getValue().remove(entry.getValue().size() - 1); + doReplace(beReplaced, toPlaceGroup, knowsGroup, targetBookieAddresses, bookieIndex); + } + + private void doReplace(BookieNode beReplaced, Map> toPlaceGroup, Map> knowsGroup, + Map targetBookieAddresses, Map bookieIndex) { Integer index = bookieIndex.get(beReplaced.getAddr()); Iterator>> iterator = knowsGroup.entrySet().iterator(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java index 5fcd38b2567..b6afe3680f1 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java @@ -388,7 +388,7 @@ private Set getNeedRepairedPlacementNotAdheringFragments(LedgerH if (curEntryId != null) { if (previousSegmentAdheringToPlacementPolicy == EnsemblePlacementPolicy.PlacementPolicyAdherence.FAIL) { LedgerFragment ledgerFragment = new LedgerFragment(lh, curEntryId, entry.getKey() - 1, - Collections.emptySet()); + new HashSet<>()); ledgerFragment.setReplicateType(LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT); placementNotAdheringFragments.add(ledgerFragment); } @@ -400,7 +400,7 @@ private Set getNeedRepairedPlacementNotAdheringFragments(LedgerH if (previousSegmentAdheringToPlacementPolicy == EnsemblePlacementPolicy.PlacementPolicyAdherence.FAIL) { long lastEntry = lh.getLedgerMetadata().getLastEntryId(); LedgerFragment ledgerFragment = new LedgerFragment(lh, curEntryId, lastEntry, - Collections.emptySet()); + new HashSet<>()); ledgerFragment.setReplicateType(LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT); placementNotAdheringFragments.add(ledgerFragment); } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java index 442ff700fbe..d5beb7362bf 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -33,26 +33,34 @@ import java.util.Enumeration; import java.util.List; import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; import java.util.TimerTask; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import io.netty.util.HashedWheelTimer; import lombok.Cleanup; import org.apache.bookkeeper.bookie.BookieImpl; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperTestClient; import org.apache.bookkeeper.client.ClientUtil; +import org.apache.bookkeeper.client.EnsemblePlacementPolicy; import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; +import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.feature.FeatureProvider; import org.apache.bookkeeper.meta.AbstractZkLedgerManager; import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.meta.LedgerManagerFactory; @@ -64,6 +72,8 @@ import org.apache.bookkeeper.meta.exceptions.MetadataException; import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.net.DNSToSwitchMapping; +import org.apache.bookkeeper.proto.BookieAddressResolver; import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException; import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.Gauge; @@ -1091,24 +1101,41 @@ public void testRWShutDownInTheCaseOfZKOperationFailures() throws Exception { @Test public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsWithNoMoreRackBookie() throws Exception { - for (int i = 0; i < 2; i++) { - startNewBookie(); - } - assertEquals(servers.size(), 5); + List rack1Bookie = servers.stream().map(ele -> { + try { + return ele.getServer().getBookieId(); + } catch (UnknownHostException e) { + return null; + } + }).filter(Objects::nonNull).collect(Collectors.toList()); - for (int i = 0; i < servers.size(); i++) { - //0,1,2 at rack1 - if (i < 3) { - StaticDNSResolver.addNodeToRack(servers.get(i).getServer().getLocalAddress().getHostName(), "/rack1"); - } else { - StaticDNSResolver.addNodeToRack(servers.get(i).getServer().getLocalAddress().getHostName(), "/rack2"); + bkc = new BookKeeperTestClient(baseClientConf) { + @Override + protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, + DNSToSwitchMapping dnsResolver, HashedWheelTimer timer, FeatureProvider featureProvider, + StatsLogger statsLogger, BookieAddressResolver bookieAddressResolver) throws IOException { + RackawareEnsemblePlacementPolicy rackawareEnsemblePlacementPolicy = new RackawareEnsemblePlacementPolicy() { + @Override + public String resolveNetworkLocation(BookieId addr) { + //The first five bookie is /rack1 + if (rack1Bookie.contains(addr)) { + return "/rack1"; + } + //The other bookie is /rack2 + return "/rack2"; + } + }; + rackawareEnsemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer, + featureProvider, statsLogger, bookieAddressResolver); + return rackawareEnsemblePlacementPolicy; } - //3,4 at rack2 - } + }; + //This ledger not adhering placement policy, the combine(0,1,2) rack is 1. - LedgerHandle lh = bkc.createLedger(5, 3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD); + LedgerHandle lh = bkc.createLedger(3, 3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD); - for (int i = 0; i < 100; i++) { + int entrySize = 10; + for (int i = 0; i < entrySize; i++) { lh.addEntry(data); } lh.close(); @@ -1141,15 +1168,55 @@ public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsWithNoMoreRackB } } - baseConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); - baseConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName()); - baseConf.setProperty(ClientConfiguration.ENSEMBLE_PLACEMENT_POLICY, RackawareEnsemblePlacementPolicy.class.getName()); - baseConf.setAuditorPeriodicPlacementPolicyCheckInterval(1000); - baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); - ReplicationWorker rw = new ReplicationWorker(baseConf); + BookKeeper bookKeeper = new BookKeeperTestClient(baseClientConf) { + @Override + protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, + DNSToSwitchMapping dnsResolver, HashedWheelTimer timer, FeatureProvider featureProvider, + StatsLogger statsLogger, BookieAddressResolver bookieAddressResolver) throws IOException { + RackawareEnsemblePlacementPolicy rackawareEnsemblePlacementPolicy = new RackawareEnsemblePlacementPolicy() { + @Override + public String resolveNetworkLocation(BookieId addr) { + //The first five bookie is /rack1 + if (rack1Bookie.contains(addr)) { + return "/rack1"; + } + //The other bookie is /rack2 + return "/rack2"; + } + }; + rackawareEnsemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer, + featureProvider, statsLogger, bookieAddressResolver); + return rackawareEnsemblePlacementPolicy; + } + }; + ReplicationWorker rw = new ReplicationWorker(baseConf, bookKeeper, false, NullStatsLogger.INSTANCE); rw.start(); - System.in.read(); + + //start new bookie, the rack is /rack2 + BookieId newBookieId = startNewBookieAndReturnBookieId(); + + CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> { + for (int i = 0; i < 200; i++) { + try { + LedgerMetadata metadata = bkc.getLedgerManager().readLedgerMetadata(lh.getId()).get().getValue(); + List newBookies = metadata.getAllEnsembles().get(0L); + if (newBookies.contains(newBookieId)) { + latch.countDown(); + break; + } + Thread.sleep(100); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + }).start(); + assertTrue("New bookie should take replace of old bookie.", latch.await(20, TimeUnit.SECONDS)); + + killAllBookies(lh, newBookieId); + + verifyRecoveredLedgers(lh, 0, entrySize - 1); } private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfiguration servConf, @@ -1163,8 +1230,7 @@ private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfigu .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME); final AuditorPeriodicCheckTest.TestAuditor auditor = new AuditorPeriodicCheckTest.TestAuditor( - BookieImpl.getBookieId(servConf).toString(), servConf, - statsLogger); + BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false, statsLogger); auditorRef.setValue(auditor); CountDownLatch latch = auditor.getLatch(); assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, placementPolicyCheckStatsLogger.getSuccessCount()); From 12582a79a80d17efb2a6b60f9eb2fb2f0d0ce45a Mon Sep 17 00:00:00 2001 From: horizonzy Date: Mon, 27 Jun 2022 15:43:02 +0800 Subject: [PATCH 07/35] fix checkstyle. --- .../LocalBookieEnsemblePlacementPolicy.java | 4 +- .../bookkeeper/client/BookKeeperAdmin.java | 5 +- .../DefaultEnsemblePlacementPolicy.java | 4 +- .../client/EnsemblePlacementPolicy.java | 2 +- .../bookkeeper/client/LedgerChecker.java | 5 +- .../bookkeeper/client/LedgerFragment.java | 11 ++- .../TopologyAwareEnsemblePlacementPolicy.java | 31 ++++---- .../bookkeeper/conf/ServerConfiguration.java | 9 ++- .../meta/AbstractZkLedgerManager.java | 14 ---- .../bookkeeper/replication/Auditor.java | 3 +- .../replication/ReplicationWorker.java | 38 ++++++---- .../TestRackawareEnsemblePlacementPolicy.java | 73 +++++++++---------- .../AuditorPlacementPolicyCheckTest.java | 26 ++++--- .../replication/TestReplicationWorker.java | 51 +++++++------ 14 files changed, 140 insertions(+), 136 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java index daa380847c1..9762c3cd785 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java @@ -125,14 +125,14 @@ public PlacementResult> newEnsemble(int ensembleSize, int writeQu public void updateBookieInfo(Map bookieToFreeSpaceMap) { return; } - + @Override public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, int ackQuorumSize) { //local bookie needn't support it. return Collections.emptyMap(); } - + @Override public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List ensembleList, int writeQuorumSize, int ackQuorumSize) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java index f2e67bad5b4..93c49aa9929 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java @@ -1147,9 +1147,8 @@ public void replicateLedgerFragment(LedgerHandle lh, final LedgerFragment ledger return; } replicateLedgerFragment(lh, ledgerFragment, targetBookieAddresses, onReadEntryFailureCallback); - } - + private void replicateLedgerFragment(LedgerHandle lh, final LedgerFragment ledgerFragment, final Map targetBookieAddresses, @@ -1786,7 +1785,7 @@ public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List replaceNotAdheringPlacementPolicyBookie(List ensembleBookiesList, int writeQuorumSize, int ackQuorumSize) { return bkc.getPlacementPolicy() diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java index 79095be40b5..88ca47e5af6 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java @@ -214,14 +214,14 @@ public void updateBookieInfo(Map bookieInfoMap) { rwLock.writeLock().unlock(); } } - + @Override public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, int ackQuorumSize) { //Default ensemble placement policy always adhering. return Collections.emptyMap(); } - + @Override public void uninitalize() { // do nothing diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index b5f37cd01ce..e80532458e7 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -363,7 +363,7 @@ DistributionSchedule.WriteSet reorderReadLACSequence( */ default void updateBookieInfo(Map bookieInfoMap) { } - + /** * Replace some bookie to adhering placement policy. If the all kinds of replacement * didn't adhere placement policy, return empty map. diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java index 76e0c01aed5..ad883fd1148 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java @@ -21,7 +21,6 @@ import io.netty.buffer.ByteBuf; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -372,7 +371,7 @@ public void checkLedger(final LedgerHandle lh, final GenericCallback> cb) { checkLedger(lh, cb, 0L); } - + public void checkLedger(final LedgerHandle lh, final GenericCallback> cb, long percentageOfLedgerFragmentToBeVerified) { @@ -457,7 +456,7 @@ public void operationComplete(int rc, Boolean result) { } checkFragments(fragments, cb, percentageOfLedgerFragmentToBeVerified); } - + private void checkFragments(Set fragments, GenericCallback> cb, long percentageOfLedgerFragmentToBeVerified) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java index 74568f4fbe2..2dffa10f836 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragment.java @@ -66,7 +66,7 @@ public LedgerFragment(LedgerFragment lf, Set subset) { this.schedule = lf.schedule; this.isLedgerClosed = lf.isLedgerClosed; } - + /** * Return a ledger fragment contains subset of bookies. * @@ -217,11 +217,11 @@ public boolean isStoredEntryId(long entryId, int bookieIndex) { public List getEnsemble() { return this.ensemble; } - + public ReplicateType getReplicateType() { return replicateType; } - + public void setReplicateType(ReplicateType replicateType) { this.replicateType = replicateType; } @@ -233,7 +233,10 @@ public String toString() { getFirstStoredEntryId(), lastKnownEntryId, getLastStoredEntryId(), getAddresses(), isLedgerClosed); } - + + /** + * ReplicateType. + */ public enum ReplicateType { DATA_LOSS, DATA_NOT_ADHERING_PLACEMENT diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index d24953a22f7..d08ce1a92f6 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -809,9 +809,9 @@ public Map replaceNotAdheringPlacementPolicyBookie(List clone = new HashMap<>(knownBookies); - + Map> toPlaceGroup = new HashMap<>(); for (BookieId bookieId : ensemble) { //If the bookieId shutdown, put it to inactive. @@ -825,14 +825,14 @@ public Map replaceNotAdheringPlacementPolicyBookie(List> knowsGroup = clone.values().stream() .collect(Collectors.groupingBy(NodeBase::getNetworkLocation)); - + for (String key : toPlaceGroup.keySet()) { knowsGroup.remove(key); } - + if (knowsGroup.isEmpty()) { return Collections.emptyMap(); } @@ -855,9 +855,10 @@ public Map replaceNotAdheringPlacementPolicyBookie(List> toPlaceGroup, Map> knowsGroup, - Map targetBookieAddresses, Map bookieIndex) { + + private void doReplaceNotAdhering(Map> toPlaceGroup, + Map> knowsGroup, Map targetBookieAddresses, + Map bookieIndex) { if (knowsGroup.isEmpty()) { return; } @@ -877,27 +878,29 @@ private void doReplaceNotAdhering(Map> toPlaceGroup, Ma doReplace(beReplaced, toPlaceGroup, knowsGroup, targetBookieAddresses, bookieIndex); return; } - + Map.Entry> entry = toPlaceEntry.get(); BookieNode beReplaced = entry.getValue().remove(entry.getValue().size() - 1); doReplace(beReplaced, toPlaceGroup, knowsGroup, targetBookieAddresses, bookieIndex); } - private void doReplace(BookieNode beReplaced, Map> toPlaceGroup, Map> knowsGroup, - Map targetBookieAddresses, Map bookieIndex) { + private void doReplace(BookieNode beReplaced, Map> toPlaceGroup, + Map> knowsGroup, Map targetBookieAddresses, + Map bookieIndex) { Integer index = bookieIndex.get(beReplaced.getAddr()); - + Iterator>> iterator = knowsGroup.entrySet().iterator(); if (iterator.hasNext()) { Map.Entry> next = iterator.next(); List list = toPlaceGroup.computeIfAbsent(next.getKey(), k -> new ArrayList<>()); - BookieNode toReplaced = new BookieNode(next.getValue().get(0).getAddr(), next.getValue().get(0).getNetworkLocation()); + BookieNode toReplaced = new BookieNode(next.getValue().get(0).getAddr(), + next.getValue().get(0).getNetworkLocation()); list.add(toReplaced); targetBookieAddresses.put(index, toReplaced.getAddr()); iterator.remove(); } } - + protected BookieNode createBookieNode(BookieId addr) { return new BookieNode(addr, resolveNetworkLocation(addr)); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 04c7414706f..6767f89439e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -202,7 +202,8 @@ public class ServerConfiguration extends AbstractConfiguration listenerSet = listeners.get(ledgerId); - //set(watcher1, watcher2) if (listenerSet == null) { Set newListenerSet = new HashSet(); Set oldListenerSet = listeners.putIfAbsent(ledgerId, newListenerSet); @@ -404,7 +401,6 @@ public void registerLedgerMetadataListener(long ledgerId, LedgerMetadataListener listenerSet = newListenerSet; } } - //set(watcher 2) synchronized (listenerSet) { listenerSet = listeners.computeIfAbsent(ledgerId, k -> new HashSet<>()); listenerSet.add(listener); @@ -412,29 +408,19 @@ public void registerLedgerMetadataListener(long ledgerId, LedgerMetadataListener new ReadLedgerMetadataTask(ledgerId).run(); } } - - //1 -> set(watcher 2) - //1 -> set - //thread 1 unregisterLedgerMetadataListener (1, watcher1) @Override public void unregisterLedgerMetadataListener(long ledgerId, LedgerMetadataListener listener) { Set listenerSet = listeners.get(ledgerId); - //set(watcher1) if (listenerSet != null) { - //set(watcher1, watcher2) - synchronized (listenerSet) { if (listenerSet.remove(listener)) { if (LOG.isDebugEnabled()) { LOG.debug("Unregistered ledger metadata listener {} on ledger {}.", listener, ledgerId); } } - //set() if (listenerSet.isEmpty()) { listeners.remove(ledgerId, listenerSet); - //1 -> set() - //listeners empty map } } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java index 82ca81caec5..b29e1a2a977 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java @@ -1451,7 +1451,8 @@ public void process(Long ledgerId, AsyncCallback.VoidCallback iterCallback) { Collections.emptyList()).whenComplete((res, e) -> { if (e != null) { LOG.error("For ledger: {}, the placement policy not adhering bookie " - + "storage, mark it to under replication manager failed.", ledgerId, e); + + "storage, mark it to under replication manager failed.", + ledgerId, e); return; } if (LOG.isDebugEnabled()) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java index b6afe3680f1..4e56dae9538 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java @@ -365,7 +365,7 @@ private boolean tryReadingFaultyEntries(LedgerHandle lh, LedgerFragment ledgerFr } return (returnRCValue.get() == BKException.Code.OK); } - + private Set getNeedRepairedPlacementNotAdheringFragments(LedgerHandle lh) { if (!conf.getRepairedPlacementPolicyNotAdheringBookieEnable()) { return Collections.emptySet(); @@ -383,21 +383,25 @@ private Set getNeedRepairedPlacementNotAdheringFragments(LedgerH } Long curEntryId = null; EnsemblePlacementPolicy.PlacementPolicyAdherence previousSegmentAdheringToPlacementPolicy = null; - + for (Map.Entry> entry : metadata.getAllEnsembles().entrySet()) { if (curEntryId != null) { - if (previousSegmentAdheringToPlacementPolicy == EnsemblePlacementPolicy.PlacementPolicyAdherence.FAIL) { - LedgerFragment ledgerFragment = new LedgerFragment(lh, curEntryId, entry.getKey() - 1, - new HashSet<>()); + if (EnsemblePlacementPolicy.PlacementPolicyAdherence.FAIL + == previousSegmentAdheringToPlacementPolicy) { + LedgerFragment ledgerFragment = new LedgerFragment(lh, curEntryId, + entry.getKey() - 1, new HashSet<>()); ledgerFragment.setReplicateType(LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT); placementNotAdheringFragments.add(ledgerFragment); } } - previousSegmentAdheringToPlacementPolicy = admin.isEnsembleAdheringToPlacementPolicy(entry.getValue(), writeQuorumSize, ackQuorumSize); + previousSegmentAdheringToPlacementPolicy = + admin.isEnsembleAdheringToPlacementPolicy(entry.getValue(), + writeQuorumSize, ackQuorumSize); curEntryId = entry.getKey(); } if (curEntryId != null) { - if (previousSegmentAdheringToPlacementPolicy == EnsemblePlacementPolicy.PlacementPolicyAdherence.FAIL) { + if (EnsemblePlacementPolicy.PlacementPolicyAdherence.FAIL + == previousSegmentAdheringToPlacementPolicy) { long lastEntry = lh.getLedgerMetadata().getLastEntryId(); LedgerFragment ledgerFragment = new LedgerFragment(lh, curEntryId, lastEntry, new HashSet<>()); @@ -420,7 +424,7 @@ private Set getNeedRepairedPlacementNotAdheringFragments(LedgerH } return placementNotAdheringFragments; } - + @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE") private boolean rereplicate(long ledgerIdToReplicate) throws InterruptedException, BKException, UnavailableException { @@ -562,14 +566,15 @@ private boolean isLastSegmentOpenAndMissingBookies(LedgerHandle lh) throws BKExc /** * Gets the under replicated fragments. */ - private Set getUnderreplicatedFragments(LedgerHandle lh, Long ledgerVerificationPercentage) throws InterruptedException { + private Set getUnderreplicatedFragments(LedgerHandle lh, Long ledgerVerificationPercentage) + throws InterruptedException { Set underreplicatedFragments = new HashSet<>(); - + Set dataLossFragments = getDataLossFragments(lh, ledgerVerificationPercentage); underreplicatedFragments.addAll(dataLossFragments); - + Set notAdheringFragments = getNeedRepairedPlacementNotAdheringFragments(lh); - + for (LedgerFragment notAdheringFragment : notAdheringFragments) { if (!checkFragmentRepeat(underreplicatedFragments, notAdheringFragment)) { underreplicatedFragments.add(notAdheringFragment); @@ -577,13 +582,14 @@ private Set getUnderreplicatedFragments(LedgerHandle lh, Long le } return underreplicatedFragments; } - - private Set getDataLossFragments(LedgerHandle lh, Long ledgerVerificationPercentage) throws InterruptedException { + + private Set getDataLossFragments(LedgerHandle lh, Long ledgerVerificationPercentage) + throws InterruptedException { CheckerCallback checkerCb = new CheckerCallback(); ledgerChecker.checkLedger(lh, checkerCb, ledgerVerificationPercentage); return checkerCb.waitAndGetResult(); } - + private boolean checkFragmentRepeat(Set fragments, LedgerFragment needChecked) { for (LedgerFragment fragment : fragments) { if (fragment.getLedgerId() == needChecked.getLedgerId() @@ -594,7 +600,7 @@ private boolean checkFragmentRepeat(Set fragments, LedgerFragmen } return false; } - + void scheduleTaskWithDelay(TimerTask timerTask, long delayPeriod) { pendingReplicationTimer.schedule(timerTask, delayPeriod); } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index 31cf8e1d84a..8e4055152ee 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -1416,11 +1416,11 @@ public boolean validate() { // this is expected } } - + @Test public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { repp.uninitalize(); - + int minNumRacksPerWriteQuorum = 3; ClientConfiguration clientConf = new ClientConfiguration(conf); clientConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorum); @@ -1430,14 +1430,14 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { repp.initialize(clientConf, Optional. empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); - + int numOfRacks = 3; int numOfBookiesPerRack = 3; String[] rackLocationNames = new String[numOfRacks]; List bookieSocketAddresses = new ArrayList(); Map bookieRackMap = new HashMap(); BookieId bookieAddress; - + for (int i = 0; i < numOfRacks; i++) { rackLocationNames[i] = "/default-region/r" + i; for (int j = 0; j < numOfBookiesPerRack; j++) { @@ -1448,76 +1448,76 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { bookieRackMap.put(bookieAddress, rackLocationNames[i]); } } - + repp.onClusterChanged(new HashSet(bookieSocketAddresses), new HashSet()); - + int writeQuorum = 3; int ackQuorum = 3; - + //test three knows bookie List knowsEnsemble = new ArrayList<>(); knowsEnsemble.add(BookieId.parse("128.0.0.0:3181")); knowsEnsemble.add(BookieId.parse("128.0.0.1:3181")); knowsEnsemble.add(BookieId.parse("128.0.0.2:3181")); - + PlacementPolicyAdherence placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( knowsEnsemble, writeQuorum, ackQuorum); assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); - - Map targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum); + + Map targetBookie = + repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum); //should replace two bookie assertEquals(targetBookie.size(), 2); - + for (Map.Entry entry : targetBookie.entrySet()) { knowsEnsemble.set(entry.getKey(), entry.getValue()); } - + placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( knowsEnsemble, writeQuorum, ackQuorum); assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); - + //test three unknowns bookie List unknownEnsembles = new ArrayList<>(); unknownEnsembles.add(BookieId.parse("128.0.0.100:3181")); unknownEnsembles.add(BookieId.parse("128.0.0.101:3181")); unknownEnsembles.add(BookieId.parse("128.0.0.102:3181")); - + placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( unknownEnsembles, writeQuorum, ackQuorum); assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); - + //should replace three bookie targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, ackQuorum, writeQuorum); assertEquals(targetBookie.size(), 3); - + for (Map.Entry entry : targetBookie.entrySet()) { unknownEnsembles.set(entry.getKey(), entry.getValue()); } - + placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( unknownEnsembles, writeQuorum, ackQuorum); assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); - - + //test three knows bookie List adheringEnsemble = new ArrayList<>(); adheringEnsemble.add(BookieId.parse("128.0.0.0:3181")); adheringEnsemble.add(BookieId.parse("128.0.0.3:3181")); adheringEnsemble.add(BookieId.parse("128.0.0.6:3181")); - + placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( adheringEnsemble, writeQuorum, ackQuorum); assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); - + //should replace three bookie targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(adheringEnsemble, ackQuorum, writeQuorum); assertEquals(targetBookie.size(), 0); } - + @Test public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() throws Exception { repp.uninitalize(); - + int minNumRacksPerWriteQuorum = 3; ClientConfiguration clientConf = new ClientConfiguration(conf); clientConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorum); @@ -1527,14 +1527,14 @@ public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() th repp.initialize(clientConf, Optional. empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); - + int numOfRacks = 2; int numOfBookiesPerRack = 3; String[] rackLocationNames = new String[numOfRacks]; List bookieSocketAddresses = new ArrayList(); Map bookieRackMap = new HashMap(); BookieId bookieAddress; - + for (int i = 0; i < numOfRacks; i++) { rackLocationNames[i] = "/default-region/r" + i; for (int j = 0; j < numOfBookiesPerRack; j++) { @@ -1545,44 +1545,43 @@ public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() th bookieRackMap.put(bookieAddress, rackLocationNames[i]); } } - + repp.onClusterChanged(new HashSet(bookieSocketAddresses), new HashSet()); - - + int writeQuorum = 3; int ackQuorum = 3; - + //test three knows bookie List knowsEnsemble = new ArrayList<>(); knowsEnsemble.add(BookieId.parse("128.0.0.0:3181")); knowsEnsemble.add(BookieId.parse("128.0.0.1:3181")); knowsEnsemble.add(BookieId.parse("128.0.0.2:3181")); - + PlacementPolicyAdherence placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( knowsEnsemble, writeQuorum, ackQuorum); assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); - - Map targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum); + + Map targetBookie = + repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum); //should not replace assertEquals(targetBookie.size(), 0); - + //test three unknowns bookie List unknownEnsembles = new ArrayList<>(); unknownEnsembles.add(BookieId.parse("128.0.0.100:3181")); unknownEnsembles.add(BookieId.parse("128.0.0.101:3181")); unknownEnsembles.add(BookieId.parse("128.0.0.102:3181")); - + placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( unknownEnsembles, writeQuorum, ackQuorum); assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); - + //should not replace targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, ackQuorum, writeQuorum); assertEquals(targetBookie.size(), 0); } - - + @Test public void testNewEnsembleWithMultipleRacks() throws Exception { BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java index 058c1d96847..0f4cd0e111a 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java @@ -292,9 +292,10 @@ public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicy() th } } } - + @Test - public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndNotMarkToUnderreplication() throws Exception { + public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndNotMarkToUnderreplication() + throws Exception { int numOfBookies = 5; int numOfLedgersNotAdheringToPlacementPolicy = 0; List bookieAddresses = new ArrayList<>(); @@ -305,21 +306,21 @@ public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndNo regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); } } - + // only three racks StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1"); StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2"); - + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); LedgerManager lm = mFactory.newLedgerManager(); int ensembleSize = 5; int writeQuorumSize = 3; int ackQuorumSize = 2; int minNumRacksPerWriteQuorumConfValue = 3; - + /* * this closed ledger doesn't adhere to placement policy because there are only * 3 racks, and the ensembleSize is 5. @@ -338,7 +339,7 @@ public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndNo .build(); lm.createLedgerMetadata(1L, initMeta).get(); numOfLedgersNotAdheringToPlacementPolicy++; - + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); setServerConfigPropertiesForRackPlacement(servConf); @@ -363,9 +364,10 @@ public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndNo long unnderReplicateLedgerId = underreplicationManager.pollLedgerToRereplicate(); assertEquals(unnderReplicateLedgerId, -1); } - + @Test - public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndMarkToUnderreplication() throws Exception { + public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndMarkToUnderreplication() + throws Exception { int numOfBookies = 5; int numOfLedgersNotAdheringToPlacementPolicy = 0; List bookieAddresses = new ArrayList<>(); @@ -376,21 +378,21 @@ public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndMa regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); } } - + // only three racks StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1"); StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2"); - + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); LedgerManager lm = mFactory.newLedgerManager(); int ensembleSize = 5; int writeQuorumSize = 3; int ackQuorumSize = 2; int minNumRacksPerWriteQuorumConfValue = 3; - + /* * this closed ledger doesn't adhere to placement policy because there are only * 3 racks, and the ensembleSize is 5. @@ -409,7 +411,7 @@ public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndMa .build(); lm.createLedgerMetadata(1L, initMeta).get(); numOfLedgersNotAdheringToPlacementPolicy++; - + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); servConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java index d5beb7362bf..7542ee3c91b 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -27,6 +27,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import io.netty.util.HashedWheelTimer; + import java.io.IOException; import java.net.URI; import java.net.UnknownHostException; @@ -44,7 +46,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import io.netty.util.HashedWheelTimer; import lombok.Cleanup; import org.apache.bookkeeper.bookie.BookieImpl; @@ -1098,7 +1099,7 @@ public void testRWShutDownInTheCaseOfZKOperationFailures() throws Exception { bkWithMockZK.close(); } } - + @Test public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsWithNoMoreRackBookie() throws Exception { List rack1Bookie = servers.stream().map(ele -> { @@ -1108,13 +1109,14 @@ public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsWithNoMoreRackB return null; } }).filter(Objects::nonNull).collect(Collectors.toList()); - + bkc = new BookKeeperTestClient(baseClientConf) { @Override protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, DNSToSwitchMapping dnsResolver, HashedWheelTimer timer, FeatureProvider featureProvider, StatsLogger statsLogger, BookieAddressResolver bookieAddressResolver) throws IOException { - RackawareEnsemblePlacementPolicy rackawareEnsemblePlacementPolicy = new RackawareEnsemblePlacementPolicy() { + RackawareEnsemblePlacementPolicy rackawareEnsemblePlacementPolicy = + new RackawareEnsemblePlacementPolicy() { @Override public String resolveNetworkLocation(BookieId addr) { //The first five bookie is /rack1 @@ -1130,26 +1132,26 @@ public String resolveNetworkLocation(BookieId addr) { return rackawareEnsemblePlacementPolicy; } }; - + //This ledger not adhering placement policy, the combine(0,1,2) rack is 1. LedgerHandle lh = bkc.createLedger(3, 3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD); - + int entrySize = 10; for (int i = 0; i < entrySize; i++) { lh.addEntry(data); } lh.close(); - + int minNumRacksPerWriteQuorumConfValue = 2; - + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); servConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName()); - servConf.setProperty(ClientConfiguration.ENSEMBLE_PLACEMENT_POLICY, RackawareEnsemblePlacementPolicy.class.getName()); + servConf.setProperty(ClientConfiguration.ENSEMBLE_PLACEMENT_POLICY, + RackawareEnsemblePlacementPolicy.class.getName()); servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1000); servConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); - - + MutableObject auditorRef = new MutableObject(); try { TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); @@ -1167,14 +1169,15 @@ public String resolveNetworkLocation(BookieId addr) { auditor.close(); } } - + baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); BookKeeper bookKeeper = new BookKeeperTestClient(baseClientConf) { @Override protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, DNSToSwitchMapping dnsResolver, HashedWheelTimer timer, FeatureProvider featureProvider, StatsLogger statsLogger, BookieAddressResolver bookieAddressResolver) throws IOException { - RackawareEnsemblePlacementPolicy rackawareEnsemblePlacementPolicy = new RackawareEnsemblePlacementPolicy() { + RackawareEnsemblePlacementPolicy rackawareEnsemblePlacementPolicy = + new RackawareEnsemblePlacementPolicy() { @Override public String resolveNetworkLocation(BookieId addr) { //The first five bookie is /rack1 @@ -1192,10 +1195,10 @@ public String resolveNetworkLocation(BookieId addr) { }; ReplicationWorker rw = new ReplicationWorker(baseConf, bookKeeper, false, NullStatsLogger.INSTANCE); rw.start(); - + //start new bookie, the rack is /rack2 BookieId newBookieId = startNewBookieAndReturnBookieId(); - + CountDownLatch latch = new CountDownLatch(1); new Thread(() -> { for (int i = 0; i < 200; i++) { @@ -1213,12 +1216,12 @@ public String resolveNetworkLocation(BookieId addr) { } }).start(); assertTrue("New bookie should take replace of old bookie.", latch.await(20, TimeUnit.SECONDS)); - + killAllBookies(lh, newBookieId); - + verifyRecoveredLedgers(lh, 0, entrySize - 1); } - + private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfiguration servConf, MutableObject auditorRef) throws MetadataException, CompatibilityException, KeeperException, InterruptedException, ReplicationException.UnavailableException, UnknownHostException { @@ -1226,14 +1229,16 @@ private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfigu LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); TestStatsProvider statsProvider = new TestStatsProvider(); TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); - TestStatsProvider.TestOpStatsLogger placementPolicyCheckStatsLogger = (TestStatsProvider.TestOpStatsLogger) statsLogger + TestStatsProvider.TestOpStatsLogger placementPolicyCheckStatsLogger = + (TestStatsProvider.TestOpStatsLogger) statsLogger .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME); - + final AuditorPeriodicCheckTest.TestAuditor auditor = new AuditorPeriodicCheckTest.TestAuditor( BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false, statsLogger); auditorRef.setValue(auditor); CountDownLatch latch = auditor.getLatch(); - assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, placementPolicyCheckStatsLogger.getSuccessCount()); + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, + placementPolicyCheckStatsLogger.getSuccessCount()); urm.setPlacementPolicyCheckCTime(-1); auditor.start(); /* @@ -1247,8 +1252,8 @@ private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfigu break; } } - assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1, placementPolicyCheckStatsLogger.getSuccessCount()); + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1, + placementPolicyCheckStatsLogger.getSuccessCount()); return statsLogger; } - } From 1e49b2214263956a37141b573bde09d3b0b610f7 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Wed, 6 Jul 2022 14:37:03 +0800 Subject: [PATCH 08/35] code clean. --- .../main/java/org/apache/bookkeeper/client/LedgerChecker.java | 4 ++-- .../client/RackawareEnsemblePlacementPolicyImpl.java | 3 --- .../org/apache/bookkeeper/meta/AbstractZkLedgerManager.java | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java index ad883fd1148..f3a49d31a61 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java @@ -20,7 +20,6 @@ package org.apache.bookkeeper.client; import io.netty.buffer.ByteBuf; - import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -387,7 +386,8 @@ public void checkLedger(final LedgerHandle lh, for (int i = 0; i < curEnsemble.size(); i++) { bookieIndexes.add(i); } - fragments.add(new LedgerFragment(lh, curEntryId, e.getKey() - 1, bookieIndexes)); + fragments.add(new LedgerFragment(lh, curEntryId, + e.getKey() - 1, bookieIndexes)); } curEntryId = e.getKey(); curEnsemble = e.getValue(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index c023b794035..46c5a10786b 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -1017,9 +1017,6 @@ DistributionSchedule.WriteSet reorderReadSequenceWithRegion( @Override public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List ensembleList, int writeQuorumSize, int ackQuorumSize) { - if (CollectionUtils.isEmpty(ensembleList)) { - return PlacementPolicyAdherence.FAIL; - } int ensembleSize = ensembleList.size(); int minNumRacksPerWriteQuorumForThisEnsemble = Math.min(writeQuorumSize, minNumRacksPerWriteQuorum); HashSet racksInQuorum = new HashSet(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java index a3a0da81d8c..cda93704e0a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java @@ -402,7 +402,6 @@ public void registerLedgerMetadataListener(long ledgerId, LedgerMetadataListener } } synchronized (listenerSet) { - listenerSet = listeners.computeIfAbsent(ledgerId, k -> new HashSet<>()); listenerSet.add(listener); } new ReadLedgerMetadataTask(ledgerId).run(); From d274c79efaecbd99d01ad72006a29026755da5a5 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Sun, 10 Jul 2022 21:52:58 +0800 Subject: [PATCH 09/35] do nothing when didn't find same rack bookies num > 1 --- .../client/TopologyAwareEnsemblePlacementPolicy.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index d08ce1a92f6..225d6c2f3b8 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -871,11 +871,7 @@ private void doReplaceNotAdhering(Map> toPlaceGroup, Optional>> toPlaceEntry = toPlaceGroup.entrySet().stream() .filter(ele -> ele.getValue().size() > 1).findAny(); if (!toPlaceEntry.isPresent()) { - Iterator>> iterator = toPlaceGroup.entrySet().stream().iterator(); - Map.Entry> entry = iterator.next(); - BookieNode beReplaced = entry.getValue().get(0); - iterator.remove(); - doReplace(beReplaced, toPlaceGroup, knowsGroup, targetBookieAddresses, bookieIndex); + toPlaceGroup.clear(); return; } From 552f2d1441b5daa95182f64cdc4783dd3ba0b4a4 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Mon, 11 Jul 2022 00:48:40 +0800 Subject: [PATCH 10/35] use replaceBookie. --- .../LocalBookieEnsemblePlacementPolicy.java | 2 +- .../bookkeeper/client/BookKeeperAdmin.java | 8 +- .../DefaultEnsemblePlacementPolicy.java | 2 +- .../client/EnsemblePlacementPolicy.java | 2 +- .../TopologyAwareEnsemblePlacementPolicy.java | 172 +++++++++--------- .../replication/ReplicationWorker.java | 5 +- .../TestRackawareEnsemblePlacementPolicy.java | 14 +- 7 files changed, 110 insertions(+), 95 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java index 9762c3cd785..cca9db73086 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java @@ -128,7 +128,7 @@ public void updateBookieInfo(Map bookieToFreeSpaceMap) { @Override public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, - int ackQuorumSize) { + int ackQuorumSize, Map customMetadata) { //local bookie needn't support it. return Collections.emptyMap(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java index 93c49aa9929..a89c2140569 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java @@ -1138,7 +1138,8 @@ public void replicateLedgerFragment(LedgerHandle lh, final LedgerFragment ledger ledgerFragment.getBookiesIndexes(), excludedBookies); } else if (LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT == ledgerFragment.getReplicateType()) { targetBookieAddresses = replaceNotAdheringPlacementPolicyBookie(ledgerFragment.getEnsemble(), - lh.getLedgerMetadata().getWriteQuorumSize(), lh.getLedgerMetadata().getAckQuorumSize()); + lh.getLedgerMetadata().getWriteQuorumSize(), lh.getLedgerMetadata().getAckQuorumSize(), + lh.getLedgerMetadata().getCustomMetadata()); ledgerFragment.getBookiesIndexes().addAll(targetBookieAddresses.keySet()); } if (MapUtils.isEmpty(targetBookieAddresses)) { @@ -1787,9 +1788,10 @@ public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List replaceNotAdheringPlacementPolicyBookie(List ensembleBookiesList, - int writeQuorumSize, int ackQuorumSize) { + int writeQuorumSize, int ackQuorumSize, Map customMetadata) { return bkc.getPlacementPolicy() - .replaceNotAdheringPlacementPolicyBookie(ensembleBookiesList, writeQuorumSize, ackQuorumSize); + .replaceNotAdheringPlacementPolicyBookie(ensembleBookiesList, writeQuorumSize, ackQuorumSize, + customMetadata); } /** diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java index 88ca47e5af6..2d73763e0c3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java @@ -217,7 +217,7 @@ public void updateBookieInfo(Map bookieInfoMap) { @Override public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, - int ackQuorumSize) { + int ackQuorumSize, Map customMetadata) { //Default ensemble placement policy always adhering. return Collections.emptyMap(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index e80532458e7..17ed9f9788e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -374,7 +374,7 @@ default void updateBookieInfo(Map bookieInfoMap) { * @return Map: key means ensemble index, value means target replace bookieId. */ Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, - int ackQuorumSize); + int ackQuorumSize, Map customMetadata); /** * Select one bookie to the "sticky" bookie where all reads for a particular diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index 225d6c2f3b8..5bed9b03054 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -796,105 +796,111 @@ public void updateBookieInfo(Map bookieInfoMap) { @Override public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, - int ackQuorumSize) { - if (CollectionUtils.isEmpty(ensemble)) { - return Collections.emptyMap(); - } - PlacementPolicyAdherence ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(ensemble, - writeQuorumSize, ackQuorumSize); - if (PlacementPolicyAdherence.FAIL != ensembleAdheringToPlacementPolicy) { - return Collections.emptyMap(); - } - Map bookieIndex = new HashMap<>(); - for (int i = 0; i < ensemble.size(); i++) { - bookieIndex.put(ensemble.get(i), i); - } - - Map clone = new HashMap<>(knownBookies); - - Map> toPlaceGroup = new HashMap<>(); - for (BookieId bookieId : ensemble) { - //If the bookieId shutdown, put it to inactive. - BookieNode bookieNode = clone.get(bookieId); - if (bookieNode == null) { - List list = toPlaceGroup.computeIfAbsent(NetworkTopology.INACTIVE, k -> new ArrayList<>()); - list.add(new BookieNode(bookieId, NetworkTopology.INACTIVE)); - } else { - List list = toPlaceGroup.computeIfAbsent(bookieNode.getNetworkLocation(), - k -> new ArrayList<>()); - list.add(bookieNode); + int ackQuorumSize, Map customMetadata) { + rwLock.readLock().lock(); + try { + if (CollectionUtils.isEmpty(ensemble)) { + return Collections.emptyMap(); } - } + PlacementPolicyAdherence ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(ensemble, + writeQuorumSize, ackQuorumSize); + if (PlacementPolicyAdherence.FAIL != ensembleAdheringToPlacementPolicy) { + return Collections.emptyMap(); + } + Map bookieIndex = new HashMap<>(); + for (int i = 0; i < ensemble.size(); i++) { + bookieIndex.put(ensemble.get(i), i); + } + + Map clone = new HashMap<>(knownBookies); + + Map> toPlaceGroup = new HashMap<>(); + for (BookieId bookieId : ensemble) { + //When ReplicationWorker.getUnderreplicatedFragments, the bookie is alive, so the fragment is not data_loss. + //When find other rack bookie to replace, the bookie maybe shutdown, so here we should pick the shutdown + // bookies. If the bookieId shutdown, put it to inactive. When do replace, we should replace inactive + // bookie firstly. + BookieNode bookieNode = clone.get(bookieId); + if (bookieNode == null) { + List list = toPlaceGroup.computeIfAbsent(NetworkTopology.INACTIVE, k -> new ArrayList<>()); + list.add(new BookieNode(bookieId, NetworkTopology.INACTIVE)); + } else { + List list = toPlaceGroup.computeIfAbsent(bookieNode.getNetworkLocation(), + k -> new ArrayList<>()); + list.add(bookieNode); + } + } + + Map> knownRackToBookies = clone.values().stream() + .collect(Collectors.groupingBy(NodeBase::getNetworkLocation)); + HashSet knownRacks = new HashSet<>(knownRackToBookies.keySet()); - Map> knowsGroup = clone.values().stream() - .collect(Collectors.groupingBy(NodeBase::getNetworkLocation)); + Set excludesBookies = new HashSet<>(); - for (String key : toPlaceGroup.keySet()) { - knowsGroup.remove(key); - } + for (String key : toPlaceGroup.keySet()) { + List sameRack = knownRackToBookies.get(key); + if (!CollectionUtils.isEmpty(sameRack)) { + excludesBookies.addAll(sameRack.stream().map(BookieNode::getAddr).collect(Collectors.toSet())); + } + } - if (knowsGroup.isEmpty()) { - return Collections.emptyMap(); - } - Map targetBookieAddresses = new HashMap<>(); - boolean placeSucceed = false; - while (knowsGroup.size() > 0) { - doReplaceNotAdhering(toPlaceGroup, knowsGroup, targetBookieAddresses, bookieIndex); - List ensembles = toPlaceGroup.values().stream().flatMap(Collection::stream).map( - BookieNode::getAddr).collect(Collectors.toList()); - ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(ensembles, - writeQuorumSize, ackQuorumSize); - if (PlacementPolicyAdherence.FAIL != ensembleAdheringToPlacementPolicy) { - placeSucceed = true; - break; + Map targetBookieAddresses = new HashMap<>(); + boolean placeSucceed = false; + while (knownRacks.size() > 0) { + BookieNode beReplaceNode = getBeReplaceNode(toPlaceGroup); + if (beReplaceNode == null) { + break; + } + Integer index = bookieIndex.get(beReplaceNode.getAddr()); + try { + PlacementResult placementResult = replaceBookie(ensemble.size(), writeQuorumSize, + ackQuorumSize, customMetadata, ensemble, beReplaceNode.getAddr(), excludesBookies); + BookieNode replaceNode = clone.get(placementResult.getResult()); + String replaceNodeNetwork = replaceNode.getNetworkLocation(); + knownRacks.remove(replaceNodeNetwork); + List nodes = toPlaceGroup.computeIfAbsent(replaceNodeNetwork, + k -> new ArrayList<>()); + nodes.add(replaceNode); + targetBookieAddresses.put(index, replaceNode.getAddr()); + List bookieNodes = knownRackToBookies.get(replaceNodeNetwork); + if (!CollectionUtils.isEmpty(bookieNodes)) { + excludesBookies.addAll( + bookieNodes.stream().map(BookieNode::getAddr).collect(Collectors.toSet())); + } + } catch (BKException.BKNotEnoughBookiesException e) { + return Collections.emptyMap(); + } + + List ensembles = toPlaceGroup.values().stream().flatMap(Collection::stream).map( + BookieNode::getAddr).collect(Collectors.toList()); + ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(ensembles, + writeQuorumSize, ackQuorumSize); + if (PlacementPolicyAdherence.FAIL != ensembleAdheringToPlacementPolicy) { + placeSucceed = true; + break; + } } + if (!placeSucceed) { + return Collections.emptyMap(); + } + return targetBookieAddresses; + } finally { + rwLock.readLock().unlock(); } - if (!placeSucceed) { - targetBookieAddresses.clear(); - return Collections.emptyMap(); - } - return targetBookieAddresses; } - private void doReplaceNotAdhering(Map> toPlaceGroup, - Map> knowsGroup, Map targetBookieAddresses, - Map bookieIndex) { - if (knowsGroup.isEmpty()) { - return; - } + private BookieNode getBeReplaceNode(Map> toPlaceGroup) { List inactiveNodes = toPlaceGroup.get(NetworkTopology.INACTIVE); if (!CollectionUtils.isEmpty(inactiveNodes)) { - BookieNode beReplaced = inactiveNodes.remove(inactiveNodes.size() - 1); - doReplace(beReplaced, toPlaceGroup, knowsGroup, targetBookieAddresses, bookieIndex); - return; + return inactiveNodes.remove(inactiveNodes.size() - 1); } Optional>> toPlaceEntry = toPlaceGroup.entrySet().stream() .filter(ele -> ele.getValue().size() > 1).findAny(); if (!toPlaceEntry.isPresent()) { - toPlaceGroup.clear(); - return; + return null; } - Map.Entry> entry = toPlaceEntry.get(); - BookieNode beReplaced = entry.getValue().remove(entry.getValue().size() - 1); - doReplace(beReplaced, toPlaceGroup, knowsGroup, targetBookieAddresses, bookieIndex); - } - - private void doReplace(BookieNode beReplaced, Map> toPlaceGroup, - Map> knowsGroup, Map targetBookieAddresses, - Map bookieIndex) { - Integer index = bookieIndex.get(beReplaced.getAddr()); - - Iterator>> iterator = knowsGroup.entrySet().iterator(); - if (iterator.hasNext()) { - Map.Entry> next = iterator.next(); - List list = toPlaceGroup.computeIfAbsent(next.getKey(), k -> new ArrayList<>()); - BookieNode toReplaced = new BookieNode(next.getValue().get(0).getAddr(), - next.getValue().get(0).getNetworkLocation()); - list.add(toReplaced); - targetBookieAddresses.put(index, toReplaced.getAddr()); - iterator.remove(); - } + return entry.getValue().remove(entry.getValue().size() - 1); } protected BookieNode createBookieNode(BookieId addr) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java index 4e56dae9538..3ffb19638f4 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java @@ -568,8 +568,11 @@ private boolean isLastSegmentOpenAndMissingBookies(LedgerHandle lh) throws BKExc */ private Set getUnderreplicatedFragments(LedgerHandle lh, Long ledgerVerificationPercentage) throws InterruptedException { + //The data loss fragments is first to repair. If a fragment is data_loss and not_adhering_placement + //at the same time, we only fix data_loss in this time. After fix data_loss, the fragment is still + //not_adhering_placement, Auditor will mark this ledger again. Set underreplicatedFragments = new HashSet<>(); - + Set dataLossFragments = getDataLossFragments(lh, ledgerVerificationPercentage); underreplicatedFragments.addAll(dataLossFragments); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index 8e4055152ee..d12ff40cbf5 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -1465,7 +1465,7 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); Map targetBookie = - repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum); + repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum, Collections.emptyMap()); //should replace two bookie assertEquals(targetBookie.size(), 2); @@ -1488,7 +1488,8 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); //should replace three bookie - targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, ackQuorum, writeQuorum); + targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, ackQuorum, writeQuorum, + Collections.emptyMap()); assertEquals(targetBookie.size(), 3); for (Map.Entry entry : targetBookie.entrySet()) { @@ -1510,7 +1511,8 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); //should replace three bookie - targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(adheringEnsemble, ackQuorum, writeQuorum); + targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(adheringEnsemble, ackQuorum, writeQuorum, + Collections.emptyMap()); assertEquals(targetBookie.size(), 0); } @@ -1563,7 +1565,8 @@ public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() th assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); Map targetBookie = - repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum); + repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum, + Collections.emptyMap()); //should not replace assertEquals(targetBookie.size(), 0); @@ -1578,7 +1581,8 @@ public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() th assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); //should not replace - targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, ackQuorum, writeQuorum); + targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, ackQuorum, writeQuorum, + Collections.emptyMap()); assertEquals(targetBookie.size(), 0); } From f6eeb35f578dc416495610323affaca32a4f3c6e Mon Sep 17 00:00:00 2001 From: horizonzy Date: Mon, 11 Jul 2022 00:57:03 +0800 Subject: [PATCH 11/35] fix check style. --- .../TopologyAwareEnsemblePlacementPolicy.java | 18 +++++++++--------- .../replication/ReplicationWorker.java | 2 +- .../TestRackawareEnsemblePlacementPolicy.java | 3 ++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index 5bed9b03054..d972438e002 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -32,7 +32,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -813,16 +812,17 @@ public Map replaceNotAdheringPlacementPolicyBookie(List clone = new HashMap<>(knownBookies); - + Map> toPlaceGroup = new HashMap<>(); for (BookieId bookieId : ensemble) { - //When ReplicationWorker.getUnderreplicatedFragments, the bookie is alive, so the fragment is not data_loss. - //When find other rack bookie to replace, the bookie maybe shutdown, so here we should pick the shutdown - // bookies. If the bookieId shutdown, put it to inactive. When do replace, we should replace inactive - // bookie firstly. + //When ReplicationWorker.getUnderreplicatedFragments, the bookie is alive, so the fragment is not + // data_loss. When find other rack bookie to replace, the bookie maybe shutdown, so here we should pick + // the shutdown bookies. If the bookieId shutdown, put it to inactive. When do replace, we should + // replace inactive bookie firstly. BookieNode bookieNode = clone.get(bookieId); if (bookieNode == null) { - List list = toPlaceGroup.computeIfAbsent(NetworkTopology.INACTIVE, k -> new ArrayList<>()); + List list = toPlaceGroup.computeIfAbsent(NetworkTopology.INACTIVE, + k -> new ArrayList<>()); list.add(new BookieNode(bookieId, NetworkTopology.INACTIVE)); } else { List list = toPlaceGroup.computeIfAbsent(bookieNode.getNetworkLocation(), @@ -830,7 +830,7 @@ public Map replaceNotAdheringPlacementPolicyBookie(List> knownRackToBookies = clone.values().stream() .collect(Collectors.groupingBy(NodeBase::getNetworkLocation)); HashSet knownRacks = new HashSet<>(knownRackToBookies.keySet()); @@ -870,7 +870,7 @@ public Map replaceNotAdheringPlacementPolicyBookie(List ensembles = toPlaceGroup.values().stream().flatMap(Collection::stream).map( BookieNode::getAddr).collect(Collectors.toList()); ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(ensembles, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java index 3ffb19638f4..e2b271a4e56 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java @@ -572,7 +572,7 @@ private Set getUnderreplicatedFragments(LedgerHandle lh, Long le //at the same time, we only fix data_loss in this time. After fix data_loss, the fragment is still //not_adhering_placement, Auditor will mark this ledger again. Set underreplicatedFragments = new HashSet<>(); - + Set dataLossFragments = getDataLossFragments(lh, ledgerVerificationPercentage); underreplicatedFragments.addAll(dataLossFragments); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index d12ff40cbf5..9cb58e4a8e8 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -1465,7 +1465,8 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); Map targetBookie = - repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum, Collections.emptyMap()); + repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum, + Collections.emptyMap()); //should replace two bookie assertEquals(targetBookie.size(), 2); From 828fcd0b7cbd5df44410c1395b1f2784b8240c71 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Mon, 11 Jul 2022 09:53:32 +0800 Subject: [PATCH 12/35] code clean. --- .../client/TopologyAwareEnsemblePlacementPolicy.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index d972438e002..324b26a61ce 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -810,7 +810,6 @@ public Map replaceNotAdheringPlacementPolicyBookie(List clone = new HashMap<>(knownBookies); Map> toPlaceGroup = new HashMap<>(); @@ -868,7 +867,8 @@ public Map replaceNotAdheringPlacementPolicyBookie(List ensembles = toPlaceGroup.values().stream().flatMap(Collection::stream).map( @@ -880,10 +880,7 @@ public Map replaceNotAdheringPlacementPolicyBookie(List Date: Mon, 11 Jul 2022 19:11:05 +0800 Subject: [PATCH 13/35] enhance test cases. --- bookkeeper-server/pom.xml | 5 + .../TopologyAwareEnsemblePlacementPolicy.java | 3 + .../bookie/datainteg/WriteSetsTest.java | 3 +- .../replication/TestReplicationWorker.java | 146 ++++++++++++------ pom.xml | 8 + 5 files changed, 114 insertions(+), 51 deletions(-) diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml index b4d1cfc474c..71782825edf 100644 --- a/bookkeeper-server/pom.xml +++ b/bookkeeper-server/pom.xml @@ -209,6 +209,11 @@ ${project.parent.version} test + + org.awaitility + awaitility + test + diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index 324b26a61ce..9cefd27d873 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -829,6 +829,9 @@ public Map replaceNotAdheringPlacementPolicyBookie(List bookieNodes : toPlaceGroup.values()) { + Collections.shuffle(bookieNodes); + } Map> knownRackToBookies = clone.values().stream() .collect(Collectors.groupingBy(NodeBase::getNetworkLocation)); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/WriteSetsTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/WriteSetsTest.java index 1a82b0bde03..b9c94283182 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/WriteSetsTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/WriteSetsTest.java @@ -34,6 +34,7 @@ import org.apache.bookkeeper.client.DistributionSchedule; import org.apache.bookkeeper.client.RoundRobinDistributionSchedule; +import org.hamcrest.collection.IsIn; import org.junit.Test; import org.slf4j.Logger; @@ -168,7 +169,7 @@ private static void assertContentsMatch(ImmutableList writeSet, } for (int i = 0; i < distWriteSet.size(); i++) { - assertThat(distWriteSet.get(i), isIn(writeSet)); + assertThat(distWriteSet.get(i), new IsIn<>(writeSet)); } } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java index 7542ee3c91b..7defc0ba581 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -24,6 +24,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -40,7 +42,6 @@ import java.util.TimerTask; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -57,6 +58,8 @@ import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; +import org.apache.bookkeeper.client.RegionAwareEnsemblePlacementPolicy; +import org.apache.bookkeeper.client.ZoneawareEnsemblePlacementPolicy; import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.conf.ClientConfiguration; @@ -97,6 +100,8 @@ import org.apache.zookeeper.Watcher.Event.KeeperState; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.ZooKeeper.States; +import org.apache.zookeeper.data.Stat; +import org.awaitility.Awaitility; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1101,8 +1106,17 @@ public void testRWShutDownInTheCaseOfZKOperationFailures() throws Exception { } @Test - public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsWithNoMoreRackBookie() throws Exception { - List rack1Bookie = servers.stream().map(ele -> { + public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsOnZone() throws Exception { + testRepairedNotAdheringPlacementPolicyLedgerFragments(ZoneawareEnsemblePlacementPolicy.class); + } + + @Test + public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsOnRack() throws Exception { + testRepairedNotAdheringPlacementPolicyLedgerFragments(RackawareEnsemblePlacementPolicy.class); + } + + private void testRepairedNotAdheringPlacementPolicyLedgerFragments(Class placementPolicyClass) throws Exception { + List firstThreeBookies = servers.stream().map(ele -> { try { return ele.getServer().getBookieId(); } catch (UnknownHostException e) { @@ -1110,26 +1124,22 @@ public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsWithNoMoreRackB } }).filter(Objects::nonNull).collect(Collectors.toList()); + baseClientConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName()); + baseClientConf.setProperty("enforceStrictZoneawarePlacement", false); bkc = new BookKeeperTestClient(baseClientConf) { @Override protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, DNSToSwitchMapping dnsResolver, HashedWheelTimer timer, FeatureProvider featureProvider, StatsLogger statsLogger, BookieAddressResolver bookieAddressResolver) throws IOException { - RackawareEnsemblePlacementPolicy rackawareEnsemblePlacementPolicy = - new RackawareEnsemblePlacementPolicy() { - @Override - public String resolveNetworkLocation(BookieId addr) { - //The first five bookie is /rack1 - if (rack1Bookie.contains(addr)) { - return "/rack1"; - } - //The other bookie is /rack2 - return "/rack2"; - } - }; - rackawareEnsemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer, + EnsemblePlacementPolicy ensemblePlacementPolicy = null; + if (ZoneawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildZoneAwareEnsemblePlacementPolicy(firstThreeBookies); + } else if (RackawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildRackAwareEnsemblePlacementPolicy(firstThreeBookies); + } + ensemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer, featureProvider, statsLogger, bookieAddressResolver); - return rackawareEnsemblePlacementPolicy; + return ensemblePlacementPolicy; } }; @@ -1147,8 +1157,6 @@ public String resolveNetworkLocation(BookieId addr) { ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); servConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName()); - servConf.setProperty(ClientConfiguration.ENSEMBLE_PLACEMENT_POLICY, - RackawareEnsemblePlacementPolicy.class.getName()); servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1000); servConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); @@ -1170,27 +1178,27 @@ public String resolveNetworkLocation(BookieId addr) { } } + Stat stat = bkc.getZkHandle() + .exists("/ledgers/underreplication/ledgers/0000/0000/0000/0000/urL0000000000", false); + assertNotNull(stat); + baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); BookKeeper bookKeeper = new BookKeeperTestClient(baseClientConf) { @Override protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, DNSToSwitchMapping dnsResolver, HashedWheelTimer timer, FeatureProvider featureProvider, StatsLogger statsLogger, BookieAddressResolver bookieAddressResolver) throws IOException { - RackawareEnsemblePlacementPolicy rackawareEnsemblePlacementPolicy = - new RackawareEnsemblePlacementPolicy() { - @Override - public String resolveNetworkLocation(BookieId addr) { - //The first five bookie is /rack1 - if (rack1Bookie.contains(addr)) { - return "/rack1"; - } - //The other bookie is /rack2 - return "/rack2"; - } - }; - rackawareEnsemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer, + EnsemblePlacementPolicy ensemblePlacementPolicy = null; + if (ZoneawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildZoneAwareEnsemblePlacementPolicy(firstThreeBookies); + } else if (RegionAwareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildRegionAwareEnsemblePlacementPolicy(firstThreeBookies); + } else if (RackawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildRackAwareEnsemblePlacementPolicy(firstThreeBookies); + } + ensemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer, featureProvider, statsLogger, bookieAddressResolver); - return rackawareEnsemblePlacementPolicy; + return ensemblePlacementPolicy; } }; ReplicationWorker rw = new ReplicationWorker(baseConf, bookKeeper, false, NullStatsLogger.INSTANCE); @@ -1199,27 +1207,65 @@ public String resolveNetworkLocation(BookieId addr) { //start new bookie, the rack is /rack2 BookieId newBookieId = startNewBookieAndReturnBookieId(); - CountDownLatch latch = new CountDownLatch(1); - new Thread(() -> { - for (int i = 0; i < 200; i++) { - try { - LedgerMetadata metadata = bkc.getLedgerManager().readLedgerMetadata(lh.getId()).get().getValue(); - List newBookies = metadata.getAllEnsembles().get(0L); - if (newBookies.contains(newBookieId)) { - latch.countDown(); - break; - } - Thread.sleep(100); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); + Awaitility.await().untilAsserted(() -> { + LedgerMetadata metadata = bkc.getLedgerManager().readLedgerMetadata(lh.getId()).get().getValue(); + List newBookies = metadata.getAllEnsembles().get(0L); + assertTrue(newBookies.contains(newBookieId)); + }); + + stat = bkc.getZkHandle() + .exists("/ledgers/underreplication/ledgers/0000/0000/0000/0000/urL0000000000", false); + assertNull(stat); + + for (BookieId rack1Book : firstThreeBookies) { + killBookie(rack1Book); + } + + verifyRecoveredLedgers(lh, 0, entrySize - 1); + } + + private EnsemblePlacementPolicy buildRackAwareEnsemblePlacementPolicy(List bookieIds) { + return new RackawareEnsemblePlacementPolicy() { + @Override + public String resolveNetworkLocation(BookieId addr) { + if (bookieIds.contains(addr)) { + return "/rack1"; } + //The other bookie is /rack2 + return "/rack2"; } - }).start(); - assertTrue("New bookie should take replace of old bookie.", latch.await(20, TimeUnit.SECONDS)); + }; + } - killAllBookies(lh, newBookieId); + private EnsemblePlacementPolicy buildRegionAwareEnsemblePlacementPolicy(List firstThreeBookies) { + return new RegionAwareEnsemblePlacementPolicy() { + @Override + protected String resolveNetworkLocation(BookieId addr) { + //The first three bookie is /rack1 + if (firstThreeBookies.contains(addr)) { + return "/rack1"; + } + //The other bookie is /rack2 + return "/rack2"; + } + }; + } - verifyRecoveredLedgers(lh, 0, entrySize - 1); + private EnsemblePlacementPolicy buildZoneAwareEnsemblePlacementPolicy(List firstThreeBookies) { + return new ZoneawareEnsemblePlacementPolicy() { + @Override + protected String resolveNetworkLocation(BookieId addr) { + //The first three bookie 1 is /zone1/ud1 + //The first three bookie 2,3 is /zone1/ud2 + if (firstThreeBookies.get(0).equals(addr)) { + return "/zone1/ud1"; + } else if (firstThreeBookies.contains(addr)) { + return "/zone1/ud2"; + } + //The other bookie is /zone2/ud1 + return "/zone2/ud1"; + } + }; } private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfiguration servConf, diff --git a/pom.xml b/pom.xml index 7f7390275bf..5010fac2034 100644 --- a/pom.xml +++ b/pom.xml @@ -210,6 +210,7 @@ 4.0.0 3.0.1 3.0.0-M6 + 4.2.0 @@ -779,6 +780,13 @@ rxjava ${rxjava.version} + + + org.awaitility + awaitility + ${awaitility.version} + test + From 88808efc8654313625a0a3b3f6d00e7623f26ade Mon Sep 17 00:00:00 2001 From: horizonzy Date: Mon, 11 Jul 2022 19:17:29 +0800 Subject: [PATCH 14/35] enhance unit tests. --- .../replication/TestReplicationWorker.java | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java index 7defc0ba581..1a3efcd3f9f 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -1191,8 +1191,6 @@ protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfig EnsemblePlacementPolicy ensemblePlacementPolicy = null; if (ZoneawareEnsemblePlacementPolicy.class == placementPolicyClass) { ensemblePlacementPolicy = buildZoneAwareEnsemblePlacementPolicy(firstThreeBookies); - } else if (RegionAwareEnsemblePlacementPolicy.class == placementPolicyClass) { - ensemblePlacementPolicy = buildRegionAwareEnsemblePlacementPolicy(firstThreeBookies); } else if (RackawareEnsemblePlacementPolicy.class == placementPolicyClass) { ensemblePlacementPolicy = buildRackAwareEnsemblePlacementPolicy(firstThreeBookies); } @@ -1213,9 +1211,11 @@ protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfig assertTrue(newBookies.contains(newBookieId)); }); - stat = bkc.getZkHandle() - .exists("/ledgers/underreplication/ledgers/0000/0000/0000/0000/urL0000000000", false); - assertNull(stat); + Awaitility.await().untilAsserted(() -> { + Stat stat1 = bkc.getZkHandle() + .exists("/ledgers/underreplication/ledgers/0000/0000/0000/0000/urL0000000000", false); + assertNull(stat1); + }); for (BookieId rack1Book : firstThreeBookies) { killBookie(rack1Book); @@ -1237,20 +1237,6 @@ public String resolveNetworkLocation(BookieId addr) { }; } - private EnsemblePlacementPolicy buildRegionAwareEnsemblePlacementPolicy(List firstThreeBookies) { - return new RegionAwareEnsemblePlacementPolicy() { - @Override - protected String resolveNetworkLocation(BookieId addr) { - //The first three bookie is /rack1 - if (firstThreeBookies.contains(addr)) { - return "/rack1"; - } - //The other bookie is /rack2 - return "/rack2"; - } - }; - } - private EnsemblePlacementPolicy buildZoneAwareEnsemblePlacementPolicy(List firstThreeBookies) { return new ZoneawareEnsemblePlacementPolicy() { @Override From 44dcf84c716e020b41397e7997f5d4675bffc142 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Tue, 12 Jul 2022 15:21:37 +0800 Subject: [PATCH 15/35] enhance unit test. --- .../TestRackawareEnsemblePlacementPolicy.java | 2 +- .../TestZoneawareEnsemblePlacementPolicy.java | 195 ++++++++++++++++++ 2 files changed, 196 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index 9cb58e4a8e8..0b80f7af0a3 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -1511,7 +1511,7 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { adheringEnsemble, writeQuorum, ackQuorum); assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); - //should replace three bookie + //shouldn't replace already meets_strict ensemble targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(adheringEnsemble, ackQuorum, writeQuorum, Collections.emptyMap()); assertEquals(targetBookie.size(), 0); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java index 12c06742495..8cf43ab0caf 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java @@ -1364,4 +1364,199 @@ public void testIsEnsembleAdheringToPlacementPolicy() throws Exception { assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_SOFT, zepp.isEnsembleAdheringToPlacementPolicy(ensemble, 3, 2)); } + + @Test + public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { + zepp.uninitalize(); + updateMyUpgradeDomain(NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); + + // Update cluster + BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.6", 3181); + BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.7", 3181); + BookieSocketAddress addr7 = new BookieSocketAddress("127.0.0.8", 3181); + BookieSocketAddress addr8 = new BookieSocketAddress("127.0.0.9", 3181); + BookieSocketAddress addr9 = new BookieSocketAddress("127.0.0.10", 3181); + BookieSocketAddress addr10 = new BookieSocketAddress("127.0.0.11", 3181); + + // update dns mapping + StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/zone1/ud1"); + StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/zone1/ud2"); + StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/zone1/ud2"); + StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/zone2/ud1"); + StaticDNSResolver.addNodeToRack(addr5.getHostName(), "/zone2/ud2"); + StaticDNSResolver.addNodeToRack(addr6.getHostName(), "/zone2/ud2"); + StaticDNSResolver.addNodeToRack(addr7.getHostName(), "/zone3/ud1"); + StaticDNSResolver.addNodeToRack(addr8.getHostName(), "/zone3/ud2"); + StaticDNSResolver.addNodeToRack(addr9.getHostName(), "/zone3/ud2"); + StaticDNSResolver.addNodeToRack(addr10.getHostName(), NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); + + zepp = new ZoneawareEnsemblePlacementPolicy(); + ClientConfiguration confLocal = new ClientConfiguration(); + confLocal.addConfiguration(conf); + confLocal.setEnforceStrictZoneawarePlacement(true); + confLocal.setMinNumZonesPerWriteQuorum(2); + confLocal.setDesiredNumZonesPerWriteQuorum(3); + zepp.initialize(confLocal, Optional. empty(), timer, DISABLE_ALL, + NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + zepp.withDefaultFaultDomain(NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); + + Set writableBookies = new HashSet<>(); + writableBookies.add(addr1.toBookieId()); + writableBookies.add(addr2.toBookieId()); + writableBookies.add(addr3.toBookieId()); + writableBookies.add(addr4.toBookieId()); + writableBookies.add(addr5.toBookieId()); + writableBookies.add(addr6.toBookieId()); + writableBookies.add(addr7.toBookieId()); + writableBookies.add(addr8.toBookieId()); + writableBookies.add(addr9.toBookieId()); + + zepp.onClusterChanged(new HashSet(writableBookies), new HashSet()); + + //all bookie in /zone1 + List knowsEnsemble = new ArrayList(); + + knowsEnsemble.add(BookieId.parse("127.0.0.2:3181")); // /zone1/ud1 + knowsEnsemble.add(BookieId.parse("127.0.0.3:3181")); // /zone1/ud2 + knowsEnsemble.add(BookieId.parse("127.0.0.4:3181")); // /zone1/ud2 + + assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, + zepp.isEnsembleAdheringToPlacementPolicy(knowsEnsemble, 3, 2)); + + Map targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, 3, 2, + Collections.emptyMap()); + + assertEquals(1, targetBookie.size()); + + for (Map.Entry entry : targetBookie.entrySet()) { + knowsEnsemble.set(entry.getKey(), entry.getValue()); + } + + assertNotEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, + zepp.isEnsembleAdheringToPlacementPolicy(knowsEnsemble, 3, 2)); + + List unknownEnsembles = new ArrayList(); + unknownEnsembles.add(BookieId.parse("127.0.0.100:3181")); // /inactive + unknownEnsembles.add(BookieId.parse("127.0.0.101:3181")); // /inactive + unknownEnsembles.add(BookieId.parse("127.0.0.102:3181")); // /inactive + + assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, + zepp.isEnsembleAdheringToPlacementPolicy(unknownEnsembles, 3, 2)); + + targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, 3, 2, + Collections.emptyMap()); + + assertEquals(3, targetBookie.size()); + + for (Map.Entry entry : targetBookie.entrySet()) { + unknownEnsembles.set(entry.getKey(), entry.getValue()); + } + + assertNotEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, + zepp.isEnsembleAdheringToPlacementPolicy(unknownEnsembles, 3, 2)); + + //test three knows bookie + List adheringStrictEnsemble = new ArrayList<>(); + adheringStrictEnsemble.add(BookieId.parse("127.0.0.2:3181")); // /zone1/ud1 + adheringStrictEnsemble.add(BookieId.parse("127.0.0.6:3181")); // /zone2/ud2 + adheringStrictEnsemble.add(BookieId.parse("127.0.0.10:3181"));// /zone3/ud2 + + assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_STRICT, + zepp.isEnsembleAdheringToPlacementPolicy(adheringStrictEnsemble, 3, 2)); + + //shouldn't replace already meets_strict ensemble + targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(adheringStrictEnsemble, 3, 2, + Collections.emptyMap()); + assertEquals(targetBookie.size(), 0); + + //test three knows bookie + List adheringSoftEnsemble = new ArrayList<>(); + adheringSoftEnsemble.add(BookieId.parse("127.0.0.2:3181")); // /zone1/ud1 + adheringSoftEnsemble.add(BookieId.parse("127.0.0.8:3181")); // /zone3/ud1 + adheringSoftEnsemble.add(BookieId.parse("127.0.0.9:3181")); // /zone3/ud2 + + assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_SOFT, + zepp.isEnsembleAdheringToPlacementPolicy(adheringSoftEnsemble, 3, 2)); + + //shouldn't replace already meets_soft ensemble + targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(adheringSoftEnsemble, 3, 2, + Collections.emptyMap()); + assertEquals(targetBookie.size(), 0); + } + + @Test + public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() throws Exception { + zepp.uninitalize(); + updateMyUpgradeDomain(NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); + + // Update cluster + BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.6", 3181); + BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.7", 3181); + BookieSocketAddress addr7 = new BookieSocketAddress("127.0.0.8", 3181); + BookieSocketAddress addr8 = new BookieSocketAddress("127.0.0.9", 3181); + BookieSocketAddress addr9 = new BookieSocketAddress("127.0.0.10", 3181); + BookieSocketAddress addr10 = new BookieSocketAddress("127.0.0.11", 3181); + + // update dns mapping + StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/zone1/ud1"); + StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/zone1/ud2"); + StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/zone1/ud3"); + StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/zone1/ud4"); + StaticDNSResolver.addNodeToRack(addr5.getHostName(), "/zone1/ud5"); + StaticDNSResolver.addNodeToRack(addr6.getHostName(), "/zone1/ud6"); + StaticDNSResolver.addNodeToRack(addr7.getHostName(), "/zone1/ud7"); + StaticDNSResolver.addNodeToRack(addr8.getHostName(), "/zone1/ud8"); + StaticDNSResolver.addNodeToRack(addr9.getHostName(), "/zone1/ud9"); + StaticDNSResolver.addNodeToRack(addr10.getHostName(), NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); + + zepp = new ZoneawareEnsemblePlacementPolicy(); + ClientConfiguration confLocal = new ClientConfiguration(); + confLocal.addConfiguration(conf); + confLocal.setEnforceStrictZoneawarePlacement(true); + confLocal.setMinNumZonesPerWriteQuorum(2); + confLocal.setDesiredNumZonesPerWriteQuorum(3); + zepp.initialize(confLocal, Optional. empty(), timer, DISABLE_ALL, + NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + zepp.withDefaultFaultDomain(NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); + + Set writableBookies = new HashSet<>(); + writableBookies.add(addr1.toBookieId()); + writableBookies.add(addr2.toBookieId()); + writableBookies.add(addr3.toBookieId()); + writableBookies.add(addr4.toBookieId()); + writableBookies.add(addr5.toBookieId()); + writableBookies.add(addr6.toBookieId()); + writableBookies.add(addr7.toBookieId()); + writableBookies.add(addr8.toBookieId()); + writableBookies.add(addr9.toBookieId()); + + zepp.onClusterChanged(new HashSet(writableBookies), new HashSet()); + + //all bookie in /zone1 + List knowsEnsemble = new ArrayList(); + knowsEnsemble.add(BookieId.parse("127.0.0.2:3181")); // /zone1/ud1 + knowsEnsemble.add(BookieId.parse("127.0.0.3:3181")); // /zone1/ud2 + knowsEnsemble.add(BookieId.parse("127.0.0.4:3181")); // /zone1/ud3 + + assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, + zepp.isEnsembleAdheringToPlacementPolicy(knowsEnsemble, 3, 2)); + + Map targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, 3, 2, + Collections.emptyMap()); + + assertEquals(0, targetBookie.size()); + + List unknownEnsembles = new ArrayList(); + unknownEnsembles.add(BookieId.parse("127.0.0.100:3181")); // /inactive + unknownEnsembles.add(BookieId.parse("127.0.0.101:3181")); // /inactive + unknownEnsembles.add(BookieId.parse("127.0.0.102:3181")); // /inactive + + assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, + zepp.isEnsembleAdheringToPlacementPolicy(unknownEnsembles, 3, 2)); + //no more bookie can be replaced + targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, 3, 2, + Collections.emptyMap()); + + assertEquals(0, targetBookie.size()); + } } From a2f33d07f407554d3304235838dd76828ddd711a Mon Sep 17 00:00:00 2001 From: horizonzy Date: Tue, 12 Jul 2022 15:30:23 +0800 Subject: [PATCH 16/35] code clean. --- .../bookkeeper/bookie/datainteg/WriteSetsTest.java | 1 - .../client/TestZoneawareEnsemblePlacementPolicy.java | 9 +++++---- .../bookkeeper/replication/TestReplicationWorker.java | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/WriteSetsTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/WriteSetsTest.java index b9c94283182..0a6f384babd 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/WriteSetsTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/datainteg/WriteSetsTest.java @@ -23,7 +23,6 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.isIn; import com.google.common.collect.ImmutableList; diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java index 8cf43ab0caf..e51b3c8f044 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java @@ -1457,9 +1457,9 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { //test three knows bookie List adheringStrictEnsemble = new ArrayList<>(); - adheringStrictEnsemble.add(BookieId.parse("127.0.0.2:3181")); // /zone1/ud1 - adheringStrictEnsemble.add(BookieId.parse("127.0.0.6:3181")); // /zone2/ud2 - adheringStrictEnsemble.add(BookieId.parse("127.0.0.10:3181"));// /zone3/ud2 + adheringStrictEnsemble.add(BookieId.parse("127.0.0.2:3181")); // /zone1/ud1 + adheringStrictEnsemble.add(BookieId.parse("127.0.0.6:3181")); // /zone2/ud2 + adheringStrictEnsemble.add(BookieId.parse("127.0.0.10:3181")); // /zone3/ud2 assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_STRICT, zepp.isEnsembleAdheringToPlacementPolicy(adheringStrictEnsemble, 3, 2)); @@ -1540,10 +1540,11 @@ public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() th assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, zepp.isEnsembleAdheringToPlacementPolicy(knowsEnsemble, 3, 2)); - + Map targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, 3, 2, Collections.emptyMap()); + assertEquals(0, targetBookie.size()); List unknownEnsembles = new ArrayList(); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java index 1a3efcd3f9f..e3cb603d86f 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -58,7 +58,6 @@ import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; -import org.apache.bookkeeper.client.RegionAwareEnsemblePlacementPolicy; import org.apache.bookkeeper.client.ZoneawareEnsemblePlacementPolicy; import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.common.util.OrderedScheduler; @@ -1115,7 +1114,8 @@ public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsOnRack() throws testRepairedNotAdheringPlacementPolicyLedgerFragments(RackawareEnsemblePlacementPolicy.class); } - private void testRepairedNotAdheringPlacementPolicyLedgerFragments(Class placementPolicyClass) throws Exception { + private void testRepairedNotAdheringPlacementPolicyLedgerFragments( + Class placementPolicyClass) throws Exception { List firstThreeBookies = servers.stream().map(ele -> { try { return ele.getServer().getBookieId(); From 0ff5d96f344b05cd14d43fe72f214588f0c013b5 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Tue, 12 Jul 2022 15:48:06 +0800 Subject: [PATCH 17/35] code clean. --- .../bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java | 1 - 1 file changed, 1 deletion(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java index e51b3c8f044..415a15e0e8d 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java @@ -1544,7 +1544,6 @@ public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() th Map targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, 3, 2, Collections.emptyMap()); - assertEquals(0, targetBookie.size()); List unknownEnsembles = new ArrayList(); From 3f03848c162366f7fc86b33628cf63c483eb0286 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Fri, 29 Jul 2022 22:03:43 +0800 Subject: [PATCH 18/35] make replaceNotAdheringPlacementPolicyBookie default implements. --- .../bookie/LocalBookieEnsemblePlacementPolicy.java | 7 ------- .../bookkeeper/client/DefaultEnsemblePlacementPolicy.java | 7 ------- .../apache/bookkeeper/client/EnsemblePlacementPolicy.java | 8 ++++++-- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java index cca9db73086..7c949cb3d4b 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LocalBookieEnsemblePlacementPolicy.java @@ -126,13 +126,6 @@ public void updateBookieInfo(Map bookieToFreeSpaceMap) { return; } - @Override - public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, - int ackQuorumSize, Map customMetadata) { - //local bookie needn't support it. - return Collections.emptyMap(); - } - @Override public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List ensembleList, int writeQuorumSize, int ackQuorumSize) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java index 93df1ec878b..94cd30344f1 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DefaultEnsemblePlacementPolicy.java @@ -213,13 +213,6 @@ public void updateBookieInfo(Map bookieInfoMap) { } } - @Override - public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, - int ackQuorumSize, Map customMetadata) { - //Default ensemble placement policy always adhering. - return Collections.emptyMap(); - } - @Override public void uninitalize() { // do nothing diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index dfe2121855c..bb99e104f85 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -18,6 +18,8 @@ package org.apache.bookkeeper.client; import io.netty.util.HashedWheelTimer; + +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -372,8 +374,10 @@ default void updateBookieInfo(Map bookieInfoMap) { * @param ackQuorumSize * @return Map: key means ensemble index, value means target replace bookieId. */ - Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, - int ackQuorumSize, Map customMetadata); + default Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, + int ackQuorumSize, Map customMetadata) { + return Collections.emptyMap(); + } /** * Select one bookie to the "sticky" bookie where all reads for a particular From c9b3b5a0263f18aaa6051f8bb23998c6fbe9980a Mon Sep 17 00:00:00 2001 From: horizonzy Date: Sun, 31 Jul 2022 12:11:21 +0800 Subject: [PATCH 19/35] fix checkstyle. --- .../org/apache/bookkeeper/client/EnsemblePlacementPolicy.java | 1 - .../client/TopologyAwareEnsemblePlacementPolicy.java | 1 - .../apache/bookkeeper/replication/TestReplicationWorker.java | 3 --- 3 files changed, 5 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index bb99e104f85..0b2ec6d7e44 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -18,7 +18,6 @@ package org.apache.bookkeeper.client; import io.netty.util.HashedWheelTimer; - import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index c66c44faa22..1b0b2c4de93 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -39,7 +39,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; import java.util.stream.Collectors; - import org.apache.bookkeeper.client.BookieInfoReader.BookieInfo; import org.apache.bookkeeper.client.WeightedRandomSelection.WeightedObject; import org.apache.bookkeeper.net.BookieId; diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java index 442e082b2a5..70042191a67 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -30,7 +30,6 @@ import static org.junit.Assert.fail; import io.netty.util.HashedWheelTimer; - import java.io.IOException; import java.net.URI; import java.net.UnknownHostException; @@ -46,9 +45,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; - import lombok.Cleanup; - import org.apache.bookkeeper.bookie.BookieImpl; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; From af1d76305a4c8c6bf683aa598a05ea44242240de Mon Sep 17 00:00:00 2001 From: horizonzy Date: Sun, 31 Jul 2022 17:47:56 +0800 Subject: [PATCH 20/35] remove duplicated dependency. --- pom.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pom.xml b/pom.xml index 51c12126385..7adeae84666 100644 --- a/pom.xml +++ b/pom.xml @@ -786,13 +786,6 @@ rxjava ${rxjava.version} - - - org.awaitility - awaitility - ${awaitility.version} - test - From e4dc79abb7c5f329aaa1a0c089e48e2cd48f255b Mon Sep 17 00:00:00 2001 From: horizonzy Date: Mon, 1 Aug 2022 21:24:47 +0800 Subject: [PATCH 21/35] add config in read me file. --- .../apache/bookkeeper/client/EnsemblePlacementPolicy.java | 4 ++-- site3/website/docs/reference/config.md | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index 0b2ec6d7e44..ce0f775d5e2 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -365,10 +365,10 @@ default void updateBookieInfo(Map bookieInfoMap) { } /** - * Replace some bookie to adhering placement policy. If the all kinds of replacement + * Replace some bookie to adhere placement policy. If the all kinds of replacement * didn't adhere placement policy, return empty map. * - * @param ensemble + * @param ensemble which would be take place. * @param writeQuorumSize * @param ackQuorumSize * @return Map: key means ensemble index, value means target replace bookieId. diff --git a/site3/website/docs/reference/config.md b/site3/website/docs/reference/config.md index 57d9abd89bc..b0737daf05b 100644 --- a/site3/website/docs/reference/config.md +++ b/site3/website/docs/reference/config.md @@ -326,10 +326,11 @@ The table below lists parameters that you can set to configure bookies. All conf ## AutoRecovery auditor settings | Parameter | Description | Default -| --------- | ----------- | ------- | +| --------- | ----------- |--| | auditorPeriodicBookieCheckInterval | The time interval between auditor bookie checks, in seconds. The auditor bookie check checks ledger metadata to see which bookies should contain entries for each ledger. If a bookie that should contain entries is unavailable, then the ledger containing that entry is marked for recovery. Setting this to 0 disables the periodic check. Bookie checks will still run when a bookie fails. The default is once per day. | 86400 | | auditorPeriodicCheckInterval | The time interval, in seconds, at which the auditor will check all ledgers in the cluster. By default this runs once a week.

Set this to 0 to disable the periodic check completely. Note that periodic checking will put extra load on the cluster, so it should not be run more frequently than once a day.
| 604800 | -| auditorPeriodicPlacementPolicyCheckInterval | The time interval between auditor placement policy checks, in seconds. The auditor placement policy check validates if the ensemble of segments of all the closed ledgers is adhering to the placement policy. It is just monitoring scrutiny but doesn't take any corrective measure other than logging error and reporting metrics. By default it is disabled. | | +| auditorPeriodicPlacementPolicyCheckInterval | The time interval between auditor placement policy checks, in seconds. The auditor placement policy check validates if the ensemble of segments of all the closed ledgers is adhering to the placement policy. It is just monitoring scrutiny but doesn't take any corrective measure other than logging error and reporting metrics. By default it is disabled. | | +| repairedPlacementPolicyNotAdheringBookieEnabled | In Auditor, it combines with auditorPeriodicPlacementPolicyCheckInterval, to control is marked ledger id to under replication managed when found a ledger ensemble not adhere to placement policy. In ReplicationWorker, to control is to repair the ledger which the ensemble does not adhere to the placement policy. By default it is disabled | | | auditorLedgerVerificationPercentage | The percentage of a ledger (fragment)'s entries will be verified before claiming a fragment as missing. If it is 0, it only verifies the first and last entries of a given fragment.
| | | lostBookieRecoveryDelay | How long to wait, in seconds, before starting autorecovery of a lost bookie. | | | storeSystemTimeAsLedgerUnderreplicatedMarkTime | Enable the Auditor to use system time as underreplicated ledger mark time. If this is enabled, Auditor will write a ctime field into the underreplicated ledger znode. | true | From 15ea48649798524c8a6690f2a05678c4787ba83c Mon Sep 17 00:00:00 2001 From: horizonzy Date: Thu, 4 Aug 2022 11:41:43 +0800 Subject: [PATCH 22/35] tuning readme file. --- .../bookkeeper/conf/ServerConfiguration.java | 13 ++++++++++- site3/website/docs/reference/config.md | 22 +++++++++---------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index c9882c8841a..a4a212df3c2 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -2611,7 +2611,7 @@ public void setAuditorPeriodicPlacementPolicyCheckInterval(long interval) { * Get the regularity at which the auditor does placement policy check of * all ledgers, which are closed. * - * @return The interval in seconds. By default it is disabled. + * @return The interval in seconds. By default, it is disabled. */ public long getAuditorPeriodicPlacementPolicyCheckInterval() { return getLong(AUDITOR_PERIODIC_PLACEMENT_POLICY_CHECK_INTERVAL, 0); @@ -2621,6 +2621,17 @@ public void setRepairedPlacementPolicyNotAdheringBookieEnable(boolean enabled) { setProperty(REPAIRED_PLACEMENT_POLICY_NOT_ADHERING_BOOKIE_ENABLED, enabled); } + /** + * In Auditor, it combines with {@link #getAuditorPeriodicPlacementPolicyCheckInterval}, to control is marked + * ledger id to under replication managed when found a ledger ensemble not adhere to placement policy. + * In ReplicationWorker, to control is to repair the ledger which the ensemble does not adhere to the placement + * policy. By default, it is disabled. + * + * If you want to enable this feature, there maybe lots of ledger will be mark underreplicated. + * The replicationWorker will replicate lots of ledger, it will increase read request and write request in bookie + * server. You should set a suitable rereplicationEntryBatchSize to avoid bookie server pressure. + * + */ public boolean getRepairedPlacementPolicyNotAdheringBookieEnable() { return getBoolean(REPAIRED_PLACEMENT_POLICY_NOT_ADHERING_BOOKIE_ENABLED, false); } diff --git a/site3/website/docs/reference/config.md b/site3/website/docs/reference/config.md index b0737daf05b..63bd279a4e5 100644 --- a/site3/website/docs/reference/config.md +++ b/site3/website/docs/reference/config.md @@ -325,17 +325,17 @@ The table below lists parameters that you can set to configure bookies. All conf ## AutoRecovery auditor settings -| Parameter | Description | Default -| --------- | ----------- |--| -| auditorPeriodicBookieCheckInterval | The time interval between auditor bookie checks, in seconds. The auditor bookie check checks ledger metadata to see which bookies should contain entries for each ledger. If a bookie that should contain entries is unavailable, then the ledger containing that entry is marked for recovery. Setting this to 0 disables the periodic check. Bookie checks will still run when a bookie fails. The default is once per day. | 86400 | -| auditorPeriodicCheckInterval | The time interval, in seconds, at which the auditor will check all ledgers in the cluster. By default this runs once a week.

Set this to 0 to disable the periodic check completely. Note that periodic checking will put extra load on the cluster, so it should not be run more frequently than once a day.
| 604800 | -| auditorPeriodicPlacementPolicyCheckInterval | The time interval between auditor placement policy checks, in seconds. The auditor placement policy check validates if the ensemble of segments of all the closed ledgers is adhering to the placement policy. It is just monitoring scrutiny but doesn't take any corrective measure other than logging error and reporting metrics. By default it is disabled. | | -| repairedPlacementPolicyNotAdheringBookieEnabled | In Auditor, it combines with auditorPeriodicPlacementPolicyCheckInterval, to control is marked ledger id to under replication managed when found a ledger ensemble not adhere to placement policy. In ReplicationWorker, to control is to repair the ledger which the ensemble does not adhere to the placement policy. By default it is disabled | | -| auditorLedgerVerificationPercentage | The percentage of a ledger (fragment)'s entries will be verified before claiming a fragment as missing. If it is 0, it only verifies the first and last entries of a given fragment.
| | -| lostBookieRecoveryDelay | How long to wait, in seconds, before starting autorecovery of a lost bookie. | | -| storeSystemTimeAsLedgerUnderreplicatedMarkTime | Enable the Auditor to use system time as underreplicated ledger mark time. If this is enabled, Auditor will write a ctime field into the underreplicated ledger znode. | true | -| underreplicatedLedgerRecoveryGracePeriod | The grace period (in seconds) for underreplicated ledgers recovery. If ledger is marked underreplicated for more than this period then it will be reported by placementPolicyCheck in Auditor. Setting this to 0 will disable this check. | | -| auditorReplicasCheckInterval | Sets the regularity/interval at which the auditor will run a replicas check of all ledgers, which are closed. This should not be run very often since it validates availability of replicas of all ledgers by querying bookies. Setting this to 0 will completely disable the periodic replicas check. By default it is disabled. | | +| Parameter | Description | Default +| --------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| auditorPeriodicBookieCheckInterval | The time interval between auditor bookie checks, in seconds. The auditor bookie check checks ledger metadata to see which bookies should contain entries for each ledger. If a bookie that should contain entries is unavailable, then the ledger containing that entry is marked for recovery. Setting this to 0 disables the periodic check. Bookie checks will still run when a bookie fails. The default is once per day. | 86400 | +| auditorPeriodicCheckInterval | The time interval, in seconds, at which the auditor will check all ledgers in the cluster. By default this runs once a week.

Set this to 0 to disable the periodic check completely. Note that periodic checking will put extra load on the cluster, so it should not be run more frequently than once a day.
| 604800 | +| auditorPeriodicPlacementPolicyCheckInterval | The time interval between auditor placement policy checks, in seconds. The auditor placement policy check validates if the ensemble of segments of all the closed ledgers is adhering to the placement policy. It is just monitoring scrutiny but doesn't take any corrective measure other than logging error and reporting metrics. By default, it is disabled. | 0 | +| repairedPlacementPolicyNotAdheringBookieEnabled | In Auditor, it combines with auditorPeriodicPlacementPolicyCheckInterval, to control is marked ledger id to under replication managed when found a ledger ensemble not adhere to placement policy. In ReplicationWorker, to control is to repair the ledger which the ensemble does not adhere to the placement policy. By default, it is disabled. If you want to enable this feature, there maybe lots of ledger will be mark underreplicated. The replicationWorker will replicate lots of ledger, it will increase read request and write request in bookie server. You should set a suitable rereplicationEntryBatchSize to avoid bookie server pressure. | false | +| auditorLedgerVerificationPercentage | The percentage of a ledger (fragment)'s entries will be verified before claiming a fragment as missing. If it is 0, it only verifies the first and last entries of a given fragment.
| | +| lostBookieRecoveryDelay | How long to wait, in seconds, before starting autorecovery of a lost bookie. | | +| storeSystemTimeAsLedgerUnderreplicatedMarkTime | Enable the Auditor to use system time as underreplicated ledger mark time. If this is enabled, Auditor will write a ctime field into the underreplicated ledger znode. | true | +| underreplicatedLedgerRecoveryGracePeriod | The grace period (in seconds) for underreplicated ledgers recovery. If ledger is marked underreplicated for more than this period then it will be reported by placementPolicyCheck in Auditor. Setting this to 0 will disable this check. | | +| auditorReplicasCheckInterval | Sets the regularity/interval at which the auditor will run a replicas check of all ledgers, which are closed. This should not be run very often since it validates availability of replicas of all ledgers by querying bookies. Setting this to 0 will completely disable the periodic replicas check. By default it is disabled. | | ## AutoRecovery replication worker settings From 7669c7ad15952f5ea0c448d4003e5aabcebebd94 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Thu, 4 Aug 2022 17:30:18 +0800 Subject: [PATCH 23/35] address comment. --- .../TopologyAwareEnsemblePlacementPolicy.java | 10 +++--- .../TestRackawareEnsemblePlacementPolicy.java | 34 ++++++++----------- .../TestZoneawareEnsemblePlacementPolicy.java | 6 ++-- 3 files changed, 21 insertions(+), 29 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index 1b0b2c4de93..c79cbda1399 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -858,9 +858,6 @@ public Map replaceNotAdheringPlacementPolicyBookie(List nodes = toPlaceGroup.computeIfAbsent(replaceNodeNetwork, - k -> new ArrayList<>()); - nodes.add(replaceNode); targetBookieAddresses.put(index, replaceNode.getAddr()); List bookieNodes = knownRackToBookies.get(replaceNodeNetwork); if (!CollectionUtils.isEmpty(bookieNodes)) { @@ -872,9 +869,10 @@ public Map replaceNotAdheringPlacementPolicyBookie(List ensembles = toPlaceGroup.values().stream().flatMap(Collection::stream).map( - BookieNode::getAddr).collect(Collectors.toList()); - ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(ensembles, + List toCheckEnsemble = new ArrayList<>(ensemble); + targetBookieAddresses.forEach(toCheckEnsemble::set); + + ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(toCheckEnsemble, writeQuorumSize, ackQuorumSize); if (PlacementPolicyAdherence.FAIL != ensembleAdheringToPlacementPolicy) { placeSucceed = true; diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index e2864843622..ccfccd2a223 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -1472,8 +1472,6 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { int minNumRacksPerWriteQuorum = 3; ClientConfiguration clientConf = new ClientConfiguration(conf); clientConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorum); - // set enforceMinNumRacksPerWriteQuorum - clientConf.setEnforceMinNumRacksPerWriteQuorum(true); repp = new RackawareEnsemblePlacementPolicy(); repp.initialize(clientConf, Optional. empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); @@ -1513,10 +1511,10 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); Map targetBookie = - repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum, + repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, writeQuorum, ackQuorum, Collections.emptyMap()); //should replace two bookie - assertEquals(targetBookie.size(), 2); + assertEquals(2, targetBookie.size()); for (Map.Entry entry : targetBookie.entrySet()) { knowsEnsemble.set(entry.getKey(), entry.getValue()); @@ -1524,7 +1522,7 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( knowsEnsemble, writeQuorum, ackQuorum); - assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, placementPolicyAdherence); //test three unknowns bookie List unknownEnsembles = new ArrayList<>(); @@ -1534,10 +1532,10 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( unknownEnsembles, writeQuorum, ackQuorum); - assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); + assertEquals(PlacementPolicyAdherence.FAIL, placementPolicyAdherence); //should replace three bookie - targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, ackQuorum, writeQuorum, + targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, writeQuorum, ackQuorum, Collections.emptyMap()); assertEquals(targetBookie.size(), 3); @@ -1547,7 +1545,7 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( unknownEnsembles, writeQuorum, ackQuorum); - assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, placementPolicyAdherence); //test three knows bookie List adheringEnsemble = new ArrayList<>(); @@ -1557,12 +1555,12 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( adheringEnsemble, writeQuorum, ackQuorum); - assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.MEETS_STRICT); + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, placementPolicyAdherence); //shouldn't replace already meets_strict ensemble - targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(adheringEnsemble, ackQuorum, writeQuorum, + targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(adheringEnsemble, writeQuorum, ackQuorum, Collections.emptyMap()); - assertEquals(targetBookie.size(), 0); + assertEquals(0, targetBookie.size()); } @Test @@ -1572,8 +1570,6 @@ public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() th int minNumRacksPerWriteQuorum = 3; ClientConfiguration clientConf = new ClientConfiguration(conf); clientConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorum); - // set enforceMinNumRacksPerWriteQuorum - clientConf.setEnforceMinNumRacksPerWriteQuorum(true); repp = new RackawareEnsemblePlacementPolicy(); repp.initialize(clientConf, Optional. empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); @@ -1611,13 +1607,13 @@ public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() th PlacementPolicyAdherence placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( knowsEnsemble, writeQuorum, ackQuorum); - assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); + assertEquals(PlacementPolicyAdherence.FAIL, placementPolicyAdherence); Map targetBookie = - repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, ackQuorum, writeQuorum, + repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, writeQuorum, ackQuorum, Collections.emptyMap()); //should not replace - assertEquals(targetBookie.size(), 0); + assertEquals(0, targetBookie.size()); //test three unknowns bookie List unknownEnsembles = new ArrayList<>(); @@ -1627,12 +1623,12 @@ public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() th placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( unknownEnsembles, writeQuorum, ackQuorum); - assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); + assertEquals(PlacementPolicyAdherence.FAIL, placementPolicyAdherence); //should not replace - targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, ackQuorum, writeQuorum, + targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, writeQuorum, ackQuorum, Collections.emptyMap()); - assertEquals(targetBookie.size(), 0); + assertEquals(0, targetBookie.size()); } @Test diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java index 371d8711517..be66c78ba08 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java @@ -1394,7 +1394,6 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { zepp = new ZoneawareEnsemblePlacementPolicy(); ClientConfiguration confLocal = new ClientConfiguration(); confLocal.addConfiguration(conf); - confLocal.setEnforceStrictZoneawarePlacement(true); confLocal.setMinNumZonesPerWriteQuorum(2); confLocal.setDesiredNumZonesPerWriteQuorum(3); zepp.initialize(confLocal, Optional. empty(), timer, DISABLE_ALL, @@ -1468,7 +1467,7 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { //shouldn't replace already meets_strict ensemble targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(adheringStrictEnsemble, 3, 2, Collections.emptyMap()); - assertEquals(targetBookie.size(), 0); + assertEquals(0, targetBookie.size()); //test three knows bookie List adheringSoftEnsemble = new ArrayList<>(); @@ -1482,7 +1481,7 @@ public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { //shouldn't replace already meets_soft ensemble targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(adheringSoftEnsemble, 3, 2, Collections.emptyMap()); - assertEquals(targetBookie.size(), 0); + assertEquals(0, targetBookie.size()); } @Test @@ -1513,7 +1512,6 @@ public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() th zepp = new ZoneawareEnsemblePlacementPolicy(); ClientConfiguration confLocal = new ClientConfiguration(); confLocal.addConfiguration(conf); - confLocal.setEnforceStrictZoneawarePlacement(true); confLocal.setMinNumZonesPerWriteQuorum(2); confLocal.setDesiredNumZonesPerWriteQuorum(3); zepp.initialize(confLocal, Optional. empty(), timer, DISABLE_ALL, From 5028740a682280dd2dcdbd17eadaf46f16e8d919 Mon Sep 17 00:00:00 2001 From: Yuri Mizushima Date: Mon, 15 Nov 2021 15:57:41 +0800 Subject: [PATCH 24/35] feat: add relocate placement command # Conflicts: # bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java --- .../client/EnsemblePlacementPolicy.java | 25 +++ .../RackawareEnsemblePlacementPolicy.java | 22 +++ .../RackawareEnsemblePlacementPolicyImpl.java | 174 ++++++++++++++++++ .../TopologyAwareEnsemblePlacementPolicy.java | 14 +- 4 files changed, 230 insertions(+), 5 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index ce0f775d5e2..140f9be173e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -455,6 +455,31 @@ default boolean areAckedBookiesAdheringToPlacementPolicy(Set ackedBook return true; } + /** + * Returns placement result. If the currentEnsemble is not adhering placement policy, returns new ensemble that + * adheres placement policy. It should be implemented so as to minify the number of bookies replaced. + * + * @param ensembleSize + * ensemble size + * @param writeQuorumSize + * writeQuorumSize of the ensemble + * @param ackQuorumSize + * ackQuorumSize of the ensemble + * @param excludeBookies + * bookies that should not be considered as targets + * @param currentEnsemble + * current ensemble + * @return a placement result + */ + default PlacementResult> replaceToAdherePlacementPolicy( + int ensembleSize, + int writeQuorumSize, + int ackQuorumSize, + Set excludeBookies, + List currentEnsemble) { + throw new UnsupportedOperationException(); + } + /** * enum for PlacementPolicyAdherence. Currently we are supporting tri-value * enum for PlacementPolicyAdherence. If placement policy is met strictly diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicy.java index 653b1a5e5db..a841fd966bd 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicy.java @@ -235,6 +235,28 @@ public BookieNode selectFromNetworkLocation( } } + @Override + public PlacementResult> replaceToAdherePlacementPolicy( + int ensembleSize, + int writeQuorumSize, + int ackQuorumSize, + Set excludeBookies, + List currentEnsemble) { + final PlacementResult> placementResult = + super.replaceToAdherePlacementPolicy(ensembleSize, writeQuorumSize, ackQuorumSize, + excludeBookies, currentEnsemble); + if (placementResult.isAdheringToPolicy() != PlacementPolicyAdherence.FAIL) { + return placementResult; + } else { + if (slave == null) { + return placementResult; + } else { + return slave.replaceToAdherePlacementPolicy(ensembleSize, writeQuorumSize, ackQuorumSize, + excludeBookies, currentEnsemble); + } + } + } + @Override public void handleBookiesThatLeft(Set leftBookies) { super.handleBookiesThatLeft(leftBookies); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index c6625b55e80..34bbab44e38 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -43,6 +43,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; import org.apache.bookkeeper.client.BookieInfoReader.BookieInfo; import org.apache.bookkeeper.client.WeightedRandomSelection.WeightedObject; @@ -66,6 +68,7 @@ import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.stats.annotations.StatsDoc; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1079,4 +1082,175 @@ public boolean areAckedBookiesAdheringToPlacementPolicy(Set ackedBooki } return rackCounter.size() >= minWriteQuorumNumRacksPerWriteQuorum; } + + @Override + public PlacementResult> replaceToAdherePlacementPolicy( + int ensembleSize, + int writeQuorumSize, + int ackQuorumSize, + Set excludeBookies, + List currentEnsemble) { + rwLock.readLock().lock(); + try { + final List provisionalEnsembleNodes = currentEnsemble.stream() + .map(this::convertBookieToNode).collect(Collectors.toList()); + final Set excludeNodes = convertBookiesToNodes( + addDefaultRackBookiesIfMinNumRacksIsEnforced(excludeBookies)); + int minNumRacksPerWriteQuorumForThisEnsemble = Math.min(writeQuorumSize, minNumRacksPerWriteQuorum); + final RRTopologyAwareCoverageEnsemble ensemble = + new RRTopologyAwareCoverageEnsemble( + ensembleSize, + writeQuorumSize, + ackQuorumSize, + RACKNAME_DISTANCE_FROM_LEAVES, + null, + null, + minNumRacksPerWriteQuorumForThisEnsemble); + + int numRacks = topology.getNumOfRacks(); + // only one rack or less than minNumRacksPerWriteQuorumForThisEnsemble, stop calculation to skip relocation + if (numRacks < 2 || numRacks < minNumRacksPerWriteQuorumForThisEnsemble) { + LOG.warn("Skip ensemble relocation because the cluster has only {} rack.", numRacks); + return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); + } + + BookieNode prevNode = null; + final BookieNode firstNode = provisionalEnsembleNodes.get(0); + // use same bookie at first to reduce ledger replication + if (!excludeNodes.contains(firstNode) && ensemble.apply(firstNode, ensemble) + && ensemble.addNode(firstNode)) { + excludeNodes.add(firstNode); + prevNode = firstNode; + } + + for (int i = prevNode == null ? 0 : 1; i < ensembleSize; i++) { + final String curRack; + if (null == prevNode) { + if ((null == localNode) || defaultRack.equals(localNode.getNetworkLocation())) { + curRack = NodeBase.ROOT; + } else { + curRack = localNode.getNetworkLocation(); + } + } else { + curRack = "~" + prevNode.getNetworkLocation(); + } + + try { + prevNode = replaceToAdherePlacementPolicyInternal( + curRack, excludeNodes, ensemble, ensemble, + provisionalEnsembleNodes, i, ensembleSize, minNumRacksPerWriteQuorumForThisEnsemble); + // replace to newer node + provisionalEnsembleNodes.set(i, prevNode); + } catch (BKNotEnoughBookiesException e) { + LOG.warn("Skip ensemble relocation because the cluster has not enough bookies."); + return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); + } + } + List bookieList = ensemble.toList(); + if (ensembleSize != bookieList.size()) { + LOG.warn("Not enough {} bookies are available to form an ensemble : {}.", + ensembleSize, bookieList); + return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); + } + return PlacementResult.of(bookieList, + isEnsembleAdheringToPlacementPolicy( + bookieList, writeQuorumSize, ackQuorumSize)); + } finally { + rwLock.readLock().unlock(); + } + } + + private BookieNode replaceToAdherePlacementPolicyInternal( + String netPath, Set excludeBookies, Predicate predicate, + Ensemble ensemble, List provisionalEnsembleNodes, int ensembleIndex, + int ensembleSize, int minNumRacksPerWriteQuorumForThisEnsemble) throws BKNotEnoughBookiesException { + final BookieNode currentNode = provisionalEnsembleNodes.get(ensembleIndex); + // if the current bookie could be applied to the ensemble, apply it to minify the number of bookies replaced + if (!excludeBookies.contains(currentNode) && predicate.apply(currentNode, ensemble)) { + if (ensemble.addNode(currentNode)) { + // add the candidate to exclude set + excludeBookies.add(currentNode); + } + return currentNode; + } + + final List>> conditionList = new ArrayList<>(); + final Set preExcludeRacks = new HashSet<>(); + final Set postExcludeRacks = new HashSet<>(); + for (int i = 0; i < minNumRacksPerWriteQuorumForThisEnsemble - 1; i++) { + preExcludeRacks.add(provisionalEnsembleNodes.get(Math.floorMod((ensembleIndex - i - 1), ensembleSize)) + .getNetworkLocation()); + postExcludeRacks.add(provisionalEnsembleNodes.get(Math.floorMod((ensembleIndex + i + 1), ensembleSize)) + .getNetworkLocation()); + } + // adhere minNumRacksPerWriteQuorum by preExcludeRacks + // avoid additional replace from write quorum candidates by preExcludeRacks and postExcludeRacks + // avoid to use first candidate bookies for election by provisionalEnsembleNodes + conditionList.add(Pair.of( + "~" + String.join(",", + Stream.concat(preExcludeRacks.stream(), postExcludeRacks.stream()).collect(Collectors.toSet())), + provisionalEnsembleNodes + )); + // avoid to use same rack between previous index by netPath + // avoid to use first candidate bookies for election by provisionalEnsembleNodes + conditionList.add(Pair.of(netPath, provisionalEnsembleNodes)); + // avoid to use same rack between previous index by netPath + conditionList.add(Pair.of(netPath, Collections.emptyList())); + + for (Pair> condition : conditionList) { + WeightedRandomSelection wRSelection = null; + + final List leaves = new ArrayList<>(topology.getLeaves(condition.getLeft())); + if (!isWeighted) { + Collections.shuffle(leaves); + } else { + if (CollectionUtils.subtract(leaves, excludeBookies).size() < 1) { + throw new BKNotEnoughBookiesException(); + } + wRSelection = prepareForWeightedSelection(leaves); + if (wRSelection == null) { + throw new BKNotEnoughBookiesException(); + } + } + + final Iterator it = leaves.iterator(); + final Set bookiesSeenSoFar = new HashSet<>(); + while (true) { + Node n; + if (isWeighted) { + if (bookiesSeenSoFar.size() == leaves.size()) { + // Don't loop infinitely. + break; + } + n = wRSelection.getNextRandom(); + bookiesSeenSoFar.add(n); + } else { + if (it.hasNext()) { + n = it.next(); + } else { + break; + } + } + if (excludeBookies.contains(n)) { + continue; + } + if (!(n instanceof BookieNode) || !predicate.apply((BookieNode) n, ensemble)) { + continue; + } + // additional excludeBookies + if (condition.getRight().contains(n)) { + continue; + } + BookieNode bn = (BookieNode) n; + // got a good candidate + if (ensemble.addNode(bn)) { + // add the candidate to exclude set + excludeBookies.add(bn); + } + return bn; + } + } + + throw new BKNotEnoughBookiesException(); + } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index c79cbda1399..08e7472bf55 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -926,12 +926,16 @@ protected String resolveNetworkLocation(BookieId addr) { protected Set convertBookiesToNodes(Collection bookies) { Set nodes = new HashSet(); for (BookieId addr : bookies) { - BookieNode bn = knownBookies.get(addr); - if (null == bn) { - bn = createBookieNode(addr); - } - nodes.add(bn); + nodes.add(convertBookieToNode(addr)); } return nodes; } + + protected BookieNode convertBookieToNode(BookieId addr) { + BookieNode bn = knownBookies.get(addr); + if (null == bn) { + bn = createBookieNode(addr); + } + return bn; + } } From c54b1681ac426b785ca3ade0d147013b7b4db705 Mon Sep 17 00:00:00 2001 From: Yuri Mizushima Date: Thu, 17 Mar 2022 15:55:13 +0800 Subject: [PATCH 25/35] feat: add relocate placement command # Conflicts: # bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java --- .../RackawareEnsemblePlacementPolicyImpl.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 34bbab44e38..df0091cd4a5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -1139,6 +1139,13 @@ public PlacementResult> replaceToAdherePlacementPolicy( prevNode = replaceToAdherePlacementPolicyInternal( curRack, excludeNodes, ensemble, ensemble, provisionalEnsembleNodes, i, ensembleSize, minNumRacksPerWriteQuorumForThisEnsemble); + // got a good candidate + if (ensemble.addNode(prevNode)) { + // add the candidate to exclude set + excludeNodes.add(prevNode); + } else { + throw new BKNotEnoughBookiesException(); + } // replace to newer node provisionalEnsembleNodes.set(i, prevNode); } catch (BKNotEnoughBookiesException e) { @@ -1167,10 +1174,6 @@ private BookieNode replaceToAdherePlacementPolicyInternal( final BookieNode currentNode = provisionalEnsembleNodes.get(ensembleIndex); // if the current bookie could be applied to the ensemble, apply it to minify the number of bookies replaced if (!excludeBookies.contains(currentNode) && predicate.apply(currentNode, ensemble)) { - if (ensemble.addNode(currentNode)) { - // add the candidate to exclude set - excludeBookies.add(currentNode); - } return currentNode; } @@ -1242,11 +1245,6 @@ private BookieNode replaceToAdherePlacementPolicyInternal( continue; } BookieNode bn = (BookieNode) n; - // got a good candidate - if (ensemble.addNode(bn)) { - // add the candidate to exclude set - excludeBookies.add(bn); - } return bn; } } From afc9a2c94dc1edda1abfac5d873986a8387ff619 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Fri, 5 Aug 2022 16:48:33 +0800 Subject: [PATCH 26/35] enhance replaceToAdherePlacementPolicy. --- .../bookkeeper/client/BookKeeperAdmin.java | 2 +- .../bookkeeper/client/BookieWatcherImpl.java | 8 +- .../client/EnsemblePlacementPolicy.java | 2 +- .../RackawareEnsemblePlacementPolicy.java | 2 +- .../RackawareEnsemblePlacementPolicyImpl.java | 77 +++++++++++++++---- .../TestRackawareEnsemblePlacementPolicy.java | 44 +++++------ .../TestZoneawareEnsemblePlacementPolicy.java | 28 +++---- 7 files changed, 104 insertions(+), 59 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java index f3987f556ce..415d29da703 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java @@ -1096,7 +1096,7 @@ private Map getReplacementBookiesByIndexes( oldBookie, bookiesToExclude); BookieId newBookie = replaceBookieResponse.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.getAdheringToPolicy(); if (isEnsembleAdheringToPlacementPolicy == PlacementPolicyAdherence.FAIL) { if (LOG.isDebugEnabled()) { LOG.debug( diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java index c2806877aca..652fbcc7733 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java @@ -269,7 +269,7 @@ public List newEnsemble(int ensembleSize, int writeQuorumSize, newEnsembleResponse = placementPolicy.newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, customMetadata, new HashSet(quarantinedBookiesSet)); socketAddresses = newEnsembleResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = newEnsembleResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = newEnsembleResponse.getAdheringToPolicy(); if (isEnsembleAdheringToPlacementPolicy == PlacementPolicyAdherence.FAIL) { ensembleNotAdheringToPlacementPolicy.inc(); if (ensembleSize > 1) { @@ -286,7 +286,7 @@ public List newEnsemble(int ensembleSize, int writeQuorumSize, newEnsembleResponse = placementPolicy.newEnsemble( ensembleSize, writeQuorumSize, ackQuorumSize, customMetadata, new HashSet<>()); socketAddresses = newEnsembleResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = newEnsembleResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = newEnsembleResponse.getAdheringToPolicy(); if (isEnsembleAdheringToPlacementPolicy == PlacementPolicyAdherence.FAIL) { ensembleNotAdheringToPlacementPolicy.inc(); log.warn("New ensemble: {} is not adhering to Placement Policy", socketAddresses); @@ -317,7 +317,7 @@ public BookieId replaceBookie(int ensembleSize, int writeQuorumSize, int ackQuor ensembleSize, writeQuorumSize, ackQuorumSize, customMetadata, existingBookies, addr, excludedBookiesAndQuarantinedBookies); socketAddress = replaceBookieResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.getAdheringToPolicy(); if (isEnsembleAdheringToPlacementPolicy == PlacementPolicyAdherence.FAIL) { ensembleNotAdheringToPlacementPolicy.inc(); log.warn( @@ -333,7 +333,7 @@ public BookieId replaceBookie(int ensembleSize, int writeQuorumSize, int ackQuor replaceBookieResponse = placementPolicy.replaceBookie(ensembleSize, writeQuorumSize, ackQuorumSize, customMetadata, existingBookies, addr, excludeBookies); socketAddress = replaceBookieResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.getAdheringToPolicy(); if (isEnsembleAdheringToPlacementPolicy == PlacementPolicyAdherence.FAIL) { ensembleNotAdheringToPlacementPolicy.inc(); log.warn( diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index 140f9be173e..35097e954a5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -521,7 +521,7 @@ public T getResult() { return result; } - public PlacementPolicyAdherence isAdheringToPolicy() { + public PlacementPolicyAdherence getAdheringToPolicy() { return policyAdherence; } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicy.java index a841fd966bd..72858f188f7 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicy.java @@ -245,7 +245,7 @@ public PlacementResult> replaceToAdherePlacementPolicy( final PlacementResult> placementResult = super.replaceToAdherePlacementPolicy(ensembleSize, writeQuorumSize, ackQuorumSize, excludeBookies, currentEnsemble); - if (placementResult.isAdheringToPolicy() != PlacementPolicyAdherence.FAIL) { + if (placementResult.getAdheringToPolicy() != PlacementPolicyAdherence.FAIL) { return placementResult; } else { if (slave == null) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index df0091cd4a5..bff6c698039 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -30,6 +30,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import io.netty.util.HashedWheelTimer; + import java.io.IOException; import java.net.InetAddress; import java.util.ArrayList; @@ -1090,6 +1091,52 @@ public PlacementResult> replaceToAdherePlacementPolicy( int ackQuorumSize, Set excludeBookies, List currentEnsemble) { + PlacementPolicyAdherence currentPlacementAdherence = isEnsembleAdheringToPlacementPolicy( + currentEnsemble, writeQuorumSize, ackQuorumSize); + if (PlacementPolicyAdherence.FAIL != currentPlacementAdherence) { + return PlacementResult.of(new ArrayList<>(currentEnsemble), currentPlacementAdherence); + } + PlacementResult> placementResult = PlacementResult.of(new ArrayList<>(currentEnsemble), + currentPlacementAdherence); + int minDiffer = Integer.MAX_VALUE; + for (int i = 0; i < currentEnsemble.size(); i++) { + PlacementResult> result = doReplaceToAdherePlacementPolicy(ensembleSize, + writeQuorumSize, ackQuorumSize, excludeBookies, currentEnsemble, i); + if (PlacementPolicyAdherence.FAIL == result.getAdheringToPolicy()) { + continue; + } + int differ = differBetweenBookies(currentEnsemble, result.getResult()); + if (differ < minDiffer) { + minDiffer = differ; + placementResult = result; + } + } + return placementResult; + } + + private int differBetweenBookies(List bookiesA, List bookiesB) { + if (CollectionUtils.isEmpty(bookiesA) || CollectionUtils.isEmpty(bookiesB)) { + return Integer.MAX_VALUE; + } + if (bookiesA.size() != bookiesB.size()) { + return Integer.MAX_VALUE; + } + int differ = 0; + for (int i = 0; i < bookiesA.size(); i++) { + if (bookiesA.get(i).equals(bookiesB.get(i))) { + differ++; + } + } + return differ; + } + + private PlacementResult> doReplaceToAdherePlacementPolicy( + int ensembleSize, + int writeQuorumSize, + int ackQuorumSize, + Set excludeBookies, + List currentEnsemble, + int startIndex) { rwLock.readLock().lock(); try { final List provisionalEnsembleNodes = currentEnsemble.stream() @@ -1106,24 +1153,22 @@ public PlacementResult> replaceToAdherePlacementPolicy( null, null, minNumRacksPerWriteQuorumForThisEnsemble); - int numRacks = topology.getNumOfRacks(); // only one rack or less than minNumRacksPerWriteQuorumForThisEnsemble, stop calculation to skip relocation if (numRacks < 2 || numRacks < minNumRacksPerWriteQuorumForThisEnsemble) { LOG.warn("Skip ensemble relocation because the cluster has only {} rack.", numRacks); return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); } - BookieNode prevNode = null; - final BookieNode firstNode = provisionalEnsembleNodes.get(0); + final BookieNode firstNode = provisionalEnsembleNodes.get(startIndex); // use same bookie at first to reduce ledger replication if (!excludeNodes.contains(firstNode) && ensemble.apply(firstNode, ensemble) && ensemble.addNode(firstNode)) { excludeNodes.add(firstNode); prevNode = firstNode; } - for (int i = prevNode == null ? 0 : 1; i < ensembleSize; i++) { + int index = i % ensembleSize; final String curRack; if (null == prevNode) { if ((null == localNode) || defaultRack.equals(localNode.getNetworkLocation())) { @@ -1134,11 +1179,10 @@ public PlacementResult> replaceToAdherePlacementPolicy( } else { curRack = "~" + prevNode.getNetworkLocation(); } - try { prevNode = replaceToAdherePlacementPolicyInternal( curRack, excludeNodes, ensemble, ensemble, - provisionalEnsembleNodes, i, ensembleSize, minNumRacksPerWriteQuorumForThisEnsemble); + provisionalEnsembleNodes, index, ensembleSize, minNumRacksPerWriteQuorumForThisEnsemble); // got a good candidate if (ensemble.addNode(prevNode)) { // add the candidate to exclude set @@ -1147,7 +1191,7 @@ public PlacementResult> replaceToAdherePlacementPolicy( throw new BKNotEnoughBookiesException(); } // replace to newer node - provisionalEnsembleNodes.set(i, prevNode); + provisionalEnsembleNodes.set(index, prevNode); } catch (BKNotEnoughBookiesException e) { LOG.warn("Skip ensemble relocation because the cluster has not enough bookies."); return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); @@ -1159,14 +1203,17 @@ public PlacementResult> replaceToAdherePlacementPolicy( ensembleSize, bookieList); return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); } - return PlacementResult.of(bookieList, - isEnsembleAdheringToPlacementPolicy( - bookieList, writeQuorumSize, ackQuorumSize)); + PlacementPolicyAdherence placementPolicyAdherence = isEnsembleAdheringToPlacementPolicy(bookieList, + writeQuorumSize, ackQuorumSize); + if (PlacementPolicyAdherence.FAIL == placementPolicyAdherence) { + return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); + } + return PlacementResult.of(bookieList, placementPolicyAdherence); } finally { rwLock.readLock().unlock(); } } - + private BookieNode replaceToAdherePlacementPolicyInternal( String netPath, Set excludeBookies, Predicate predicate, Ensemble ensemble, List provisionalEnsembleNodes, int ensembleIndex, @@ -1176,7 +1223,6 @@ private BookieNode replaceToAdherePlacementPolicyInternal( if (!excludeBookies.contains(currentNode) && predicate.apply(currentNode, ensemble)) { return currentNode; } - final List>> conditionList = new ArrayList<>(); final Set preExcludeRacks = new HashSet<>(); final Set postExcludeRacks = new HashSet<>(); @@ -1193,16 +1239,16 @@ private BookieNode replaceToAdherePlacementPolicyInternal( "~" + String.join(",", Stream.concat(preExcludeRacks.stream(), postExcludeRacks.stream()).collect(Collectors.toSet())), provisionalEnsembleNodes - )); + )); // avoid to use same rack between previous index by netPath // avoid to use first candidate bookies for election by provisionalEnsembleNodes conditionList.add(Pair.of(netPath, provisionalEnsembleNodes)); // avoid to use same rack between previous index by netPath conditionList.add(Pair.of(netPath, Collections.emptyList())); - + for (Pair> condition : conditionList) { WeightedRandomSelection wRSelection = null; - + final List leaves = new ArrayList<>(topology.getLeaves(condition.getLeft())); if (!isWeighted) { Collections.shuffle(leaves); @@ -1248,7 +1294,6 @@ private BookieNode replaceToAdherePlacementPolicyInternal( return bn; } } - throw new BKNotEnoughBookiesException(); } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index ccfccd2a223..9c04210842f 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -650,7 +650,7 @@ public void testReplaceBookieWithEnoughBookiesInSameRack() throws Exception { EnsemblePlacementPolicy.PlacementResult replaceBookieResponse = repp.replaceBookie(1, 1, 1, null, new ArrayList<>(), addr2.toBookieId(), new HashSet<>()); BookieId replacedBookie = replaceBookieResponse.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.getAdheringToPolicy(); assertEquals(addr3.toBookieId(), replacedBookie); assertEquals(PlacementPolicyAdherence.MEETS_STRICT, isEnsembleAdheringToPlacementPolicy); } @@ -679,7 +679,7 @@ public void testReplaceBookieWithEnoughBookiesInDifferentRack() throws Exception EnsemblePlacementPolicy.PlacementResult replaceBookieResponse = repp.replaceBookie(1, 1, 1, null, new ArrayList<>(), addr2.toBookieId(), excludedAddrs); BookieId replacedBookie = replaceBookieResponse.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.getAdheringToPolicy(); assertFalse(addr1.toBookieId().equals(replacedBookie)); assertTrue(addr3.toBookieId().equals(replacedBookie) || addr4.toBookieId().equals(replacedBookie)); assertEquals(PlacementPolicyAdherence.MEETS_STRICT, isEnsembleAdheringToPlacementPolicy); @@ -744,7 +744,7 @@ public void testReplaceBookieWithEnoughBookiesInSameRackAsEnsemble() throws Exce addr4.toBookieId(), new HashSet<>()); BookieId replacedBookie = replaceBookieResponse.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.getAdheringToPolicy(); assertEquals(addr1.toBookieId(), replacedBookie); assertEquals(PlacementPolicyAdherence.MEETS_STRICT, isEnsembleAdheringToPlacementPolicy); } @@ -766,14 +766,14 @@ public void testNewEnsembleWithSingleRack() throws Exception { EnsemblePlacementPolicy.PlacementResult> ensembleResponse; ensembleResponse = repp.newEnsemble(3, 2, 2, null, new HashSet<>()); List ensemble = ensembleResponse.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assertEquals(0, getNumCoveredWriteQuorums(ensemble, 2, conf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver)); assertEquals(PlacementPolicyAdherence.FAIL, isEnsembleAdheringToPlacementPolicy); EnsemblePlacementPolicy.PlacementResult> ensembleResponse2; ensembleResponse2 = repp.newEnsemble(4, 2, 2, null, new HashSet<>()); List ensemble2 = ensembleResponse2.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy2 = ensembleResponse2.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy2 = ensembleResponse2.getAdheringToPolicy(); assertEquals(0, getNumCoveredWriteQuorums(ensemble2, 2, conf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver)); assertEquals(PlacementPolicyAdherence.FAIL, isEnsembleAdheringToPlacementPolicy); @@ -898,7 +898,7 @@ public void testNewEnsembleWithEnforceMinNumRacks() throws Exception { ensembleResponse = repp.newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, null, new HashSet<>()); ensemble = ensembleResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = ensembleResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assertEquals("Number of writeQuorum sets covered", ensembleSize, getNumCoveredWriteQuorums(ensemble, writeQuorumSize, clientConf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver)); @@ -908,7 +908,7 @@ public void testNewEnsembleWithEnforceMinNumRacks() throws Exception { new HashSet<>(defaultRackBookiesList), EnsembleForReplacementWithNoConstraints.INSTANCE, TruePredicate.INSTANCE); ensemble = ensembleResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = ensembleResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assertEquals("Number of writeQuorum sets covered", ensembleSize, getNumCoveredWriteQuorums(ensemble, writeQuorumSize, clientConf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver)); @@ -962,7 +962,7 @@ public void testNewEnsembleWithSufficientRacksAndEnforceMinNumRacks() throws Exc for (int ensembleSize = effectiveMinNumRacksPerWriteQuorum; ensembleSize < 40; ensembleSize++) { ensembleResponse = repp.newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, null, new HashSet<>()); ensemble = ensembleResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = ensembleResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assertEquals("Number of writeQuorum sets covered", ensembleSize, getNumCoveredWriteQuorums(ensemble, writeQuorumSize, clientConf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver)); @@ -971,7 +971,7 @@ public void testNewEnsembleWithSufficientRacksAndEnforceMinNumRacks() throws Exc ensembleResponse = repp.newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, new HashSet<>(), EnsembleForReplacementWithNoConstraints.INSTANCE, TruePredicate.INSTANCE); ensemble = ensembleResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = ensembleResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assertEquals("Number of writeQuorum sets covered", ensembleSize, getNumCoveredWriteQuorums(ensemble, writeQuorumSize, clientConf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver)); @@ -1098,7 +1098,7 @@ public void testReplaceBookieWithEnforceMinNumRacks() throws Exception { replaceBookieResponse = repp.replaceBookie(ensembleSize, writeQuorumSize, ackQuorumSize, null, ensemble, bookieInEnsembleToBeReplaced, new HashSet<>()); replacedBookieAddress = replaceBookieResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.getAdheringToPolicy(); assertEquals("It should be newBookieAddress2", newBookieAddress2.toBookieId(), replacedBookieAddress); assertEquals(PlacementPolicyAdherence.MEETS_STRICT, isEnsembleAdheringToPlacementPolicy); @@ -1138,7 +1138,7 @@ public void testReplaceBookieWithEnforceMinNumRacks() throws Exception { replaceBookieResponse = repp.replaceBookie(ensembleSize, writeQuorumSize, ackQuorumSize, null, ensemble, bookieInEnsembleToBeReplaced, bookiesToExclude); replacedBookieAddress = replaceBookieResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.getAdheringToPolicy(); assertEquals("It should be newBookieAddress3", newBookieAddress3.toBookieId(), replacedBookieAddress); assertEquals(PlacementPolicyAdherence.MEETS_STRICT, isEnsembleAdheringToPlacementPolicy); } @@ -1657,7 +1657,7 @@ public void testNewEnsembleWithMultipleRacks() throws Exception { repp.newEnsemble(ensembleSize, writeQuorumSize, acqQuorumSize, null, new HashSet<>()); List ensemble = ensembleResponse.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); int numCovered = getNumCoveredWriteQuorums(ensemble, writeQuorumSize, conf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver); assertTrue(numCovered >= 1 && numCovered < 3); @@ -1667,7 +1667,7 @@ public void testNewEnsembleWithMultipleRacks() throws Exception { repp.newEnsemble(ensembleSize, writeQuorumSize, acqQuorumSize, null, new HashSet<>()); List ensemble2 = ensembleResponse2.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy2 = ensembleResponse2.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy2 = ensembleResponse2.getAdheringToPolicy(); numCovered = getNumCoveredWriteQuorums(ensemble2, writeQuorumSize, conf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver); assertTrue(numCovered >= 1 && numCovered < 3); @@ -1745,7 +1745,7 @@ void validateNumOfWriteQuorumsCoveredInEnsembleCreation(Set addrs, repp.newEnsemble(ensembleSize, writeQuorumSize, writeQuorumSize, null, new HashSet<>()); List ensemble = ensembleResponse.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); int numCovered = getNumCoveredWriteQuorums(ensemble, writeQuorumSize, minNumRacksPerWriteQuorum, repp.bookieAddressResolver); assertEquals("minimum number of racks covered for writequorum ensemble: " + ensemble, ensembleSize, numCovered); @@ -1791,7 +1791,7 @@ public void testNewEnsembleWithEnoughRacks() throws Exception { repp.newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, null, new HashSet<>()); List ensemble1 = ensembleResponse.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy1 = ensembleResponse.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy1 = ensembleResponse.getAdheringToPolicy(); assertEquals(ensembleSize, getNumCoveredWriteQuorums(ensemble1, writeQuorumSize, conf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver)); @@ -1801,7 +1801,7 @@ public void testNewEnsembleWithEnoughRacks() throws Exception { EnsemblePlacementPolicy.PlacementResult> ensembleResponse2 = repp.newEnsemble(ensembleSize, writeQuorumSize, 2, null, new HashSet<>()); List ensemble2 = ensembleResponse2.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy2 = ensembleResponse2.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy2 = ensembleResponse2.getAdheringToPolicy(); assertEquals(ensembleSize, getNumCoveredWriteQuorums(ensemble2, writeQuorumSize, conf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver)); @@ -1885,7 +1885,7 @@ public void testWeightedPlacementAndReplaceBookieWithEnoughBookiesInSameRack() t replaceBookieResponse = repp.replaceBookie(1, 1, 1, null, new ArrayList<>(), addr2.toBookieId(), new HashSet<>()); replacedBookie = replaceBookieResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.getAdheringToPolicy(); assertTrue("replaced : " + replacedBookie, addr3.toBookieId().equals(replacedBookie) || addr4.toBookieId().equals(replacedBookie)); assertEquals(PlacementPolicyAdherence.MEETS_STRICT, isEnsembleAdheringToPlacementPolicy); @@ -1956,7 +1956,7 @@ public void testWeightedPlacementAndReplaceBookieWithoutEnoughBookiesInSameRack( replaceBookieResponse = repp.replaceBookie(1, 1, 1, null, new ArrayList<>(), addr2.toBookieId(), new HashSet<>()); replacedBookie = replaceBookieResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = replaceBookieResponse.getAdheringToPolicy(); assertTrue(addr0.toBookieId().equals(replacedBookie) || addr1.toBookieId().equals(replacedBookie) || addr3.toBookieId().equals(replacedBookie) @@ -2238,7 +2238,7 @@ public void testPlacementOnStabilizeNetworkTopology() throws Exception { EnsemblePlacementPolicy.PlacementResult> ensembleResponse = repp.newEnsemble(3, 2, 2, null, new HashSet()); List ensemble = ensembleResponse.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assertFalse(ensemble.contains(addr4.toBookieId())); assertEquals(PlacementPolicyAdherence.FAIL, isEnsembleAdheringToPlacementPolicy); } @@ -2247,7 +2247,7 @@ public void testPlacementOnStabilizeNetworkTopology() throws Exception { EnsemblePlacementPolicy.PlacementResult> ensembleResponse = repp.newEnsemble(4, 2, 2, null, new HashSet()); List ensemble = ensembleResponse.getResult(); - PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.isAdheringToPolicy(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assertEquals(PlacementPolicyAdherence.FAIL, isEnsembleAdheringToPlacementPolicy); assertTrue(ensemble.contains(addr4.toBookieId())); } @@ -2541,7 +2541,7 @@ public void testNewEnsembleExcludesDefaultRackBookiesEnforceMinNumRacks() throws for (int ensembleSize = effectiveMinNumRacksPerWriteQuorum; ensembleSize < 40; ensembleSize++) { ensembleResponse = repp.newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, null, new HashSet<>()); ensemble = ensembleResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = ensembleResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assertEquals("Number of writeQuorum sets covered", ensembleSize, getNumCoveredWriteQuorums(ensemble, writeQuorumSize, clientConf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver)); @@ -2549,7 +2549,7 @@ public void testNewEnsembleExcludesDefaultRackBookiesEnforceMinNumRacks() throws ensembleResponse = repp.newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, null, new HashSet<>()); ensemble = ensembleResponse.getResult(); - isEnsembleAdheringToPlacementPolicy = ensembleResponse.isAdheringToPolicy(); + isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assertEquals("Number of writeQuorum sets covered", ensembleSize, getNumCoveredWriteQuorums(ensemble, writeQuorumSize, clientConf.getMinNumRacksPerWriteQuorum(), repp.bookieAddressResolver)); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java index be66c78ba08..91f2e7ef210 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java @@ -241,7 +241,7 @@ public void testEnoughRWBookies() throws Exception { newEnsemblePlacementResult.getResult()); assertTrue("New ensemble should contain all 6 rw bookies", newEnsembleSet.containsAll(rwAddrs)); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_STRICT, - newEnsemblePlacementResult.isAdheringToPolicy()); + newEnsemblePlacementResult.getAdheringToPolicy()); /* * there are enough bookies so newEnsemble should succeed. @@ -251,7 +251,7 @@ public void testEnoughRWBookies() throws Exception { assertTrue("New ensemble should contain 3 rw bookies", (newEnsembleSet.size() == 3) && (rwAddrs.containsAll(newEnsembleSet))); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_STRICT, - newEnsemblePlacementResult.isAdheringToPolicy()); + newEnsemblePlacementResult.getAdheringToPolicy()); } @Test @@ -309,7 +309,7 @@ public void testWithDefaultBookies() throws Exception { assertTrue("Bookie from default faultDomain shouldn't be part of ensemble", Collections.disjoint(newEnsembleSet, bookiesInDefaultFaultDomain)); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_STRICT, - newEnsemblePlacementResult.isAdheringToPolicy()); + newEnsemblePlacementResult.getAdheringToPolicy()); } } @@ -373,7 +373,7 @@ public void testMinZonesPerWriteQuorum() throws Exception { assertTrue("Bookie from default faultDomain shouldn't be part of ensemble", Collections.disjoint(newEnsembleSet, bookiesInDefaultFaultDomain)); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_SOFT, - newEnsemblePlacementResult.isAdheringToPolicy()); + newEnsemblePlacementResult.getAdheringToPolicy()); try { /* @@ -404,7 +404,7 @@ public void testMinZonesPerWriteQuorum() throws Exception { newEnsembleSet = new HashSet(newEnsemblePlacementResult.getResult()); assertTrue("New ensemble should contain 4 different bookies", newEnsembleSet.size() == 4); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, - newEnsemblePlacementResult.isAdheringToPolicy()); + newEnsemblePlacementResult.getAdheringToPolicy()); } @Test @@ -479,7 +479,7 @@ public void testMinUDsNotAvailable() throws Exception { */ newEnsemblePlacementResult = zepp.newEnsemble(ensSize, writeQuorum, 2, null, new HashSet<>()); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_STRICT, - newEnsemblePlacementResult.isAdheringToPolicy()); + newEnsemblePlacementResult.getAdheringToPolicy()); List newEnsemble = newEnsemblePlacementResult.getResult(); Set newEnsembleSet = new HashSet(newEnsemble); assertTrue("New ensemble should contain all 6 rw bookies in non-default fault domains", @@ -563,7 +563,7 @@ public void testUniqueUds() throws Exception { assertTrue("New ensemble should contain 6 rw bookies in non-default fault domains", rwAddrs.containsAll(newEnsembleSet) && (newEnsembleSet.size() == 6)); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_SOFT, - newEnsemblePlacementResult.isAdheringToPolicy()); + newEnsemblePlacementResult.getAdheringToPolicy()); Set bookiesNetworkLocations = new HashSet(); for (BookieId bookieAddr : newEnsembleSet) { @@ -654,7 +654,7 @@ public void testNewBookieUniformDistributionWithMinZoneAndMinUDs() throws Except excludedBookies); List newEnsembleList = newEnsemblePlacementResult.getResult(); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_SOFT, - newEnsemblePlacementResult.isAdheringToPolicy()); + newEnsemblePlacementResult.getAdheringToPolicy()); Set newEnsembleSet = new HashSet(newEnsembleList); Set bookiesNetworkLocationsSet = new HashSet(); List bookiesNodeLocationList = new ArrayList(); @@ -761,7 +761,7 @@ public void testReplaceBookie() throws Exception { BookieId replacedBookie = replacePlacementResult.getResult(); assertEquals("replaced bookie", addr8.toBookieId(), replacedBookie); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_STRICT, - replacePlacementResult.isAdheringToPolicy()); + replacePlacementResult.getAdheringToPolicy()); excludedBookies.add(addr8.toBookieId()); /* @@ -854,7 +854,7 @@ public void testReplaceBookieMinUDs() throws Exception { BookieId replacedBookie = replaceResponse.getResult(); assertEquals("replaced bookie", "/zone3/ud2", zepp.resolveNetworkLocation(replacedBookie)); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_SOFT, - replaceResponse.isAdheringToPolicy()); + replaceResponse.getAdheringToPolicy()); } @Test @@ -1074,7 +1074,7 @@ public void testPlacementOnStabilizeNetworkTopology() throws Exception { List ensemble = ensembleResponse.getResult(); assertFalse(ensemble.contains(addr4.toBookieId())); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_STRICT, - ensembleResponse.isAdheringToPolicy()); + ensembleResponse.getAdheringToPolicy()); } // we could still use addr4 for urgent allocation if it is just bookie @@ -1084,7 +1084,7 @@ public void testPlacementOnStabilizeNetworkTopology() throws Exception { List ensemble = ensembleResponse.getResult(); assertTrue(ensemble.contains(addr4.toBookieId())); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_STRICT, - ensembleResponse.isAdheringToPolicy()); + ensembleResponse.getAdheringToPolicy()); } @Test @@ -1135,7 +1135,7 @@ public void testCreateNewEnsembleRandomly() throws Exception { assertFalse("excludeBookie should not be included in the ensemble", newEnsembleSet.contains(addr5.toBookieId())); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, - newEnsemblePlacementResult.isAdheringToPolicy()); + newEnsemblePlacementResult.getAdheringToPolicy()); rwAddrs.remove(addr4.toBookieId()); roAddrs.add(addr4.toBookieId()); @@ -1209,7 +1209,7 @@ public void testReplaceBookieRandomly() throws Exception { */ assertEquals("ReplaceBookie candidate", addr7.toBookieId(), replaceBookie); assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, - replaceResponse.isAdheringToPolicy()); + replaceResponse.getAdheringToPolicy()); rwAddrs.remove(addr7.toBookieId()); excludeBookies.add(addr7.toBookieId()); From 0b9180534c272b7709edcbcee70c953c901b53a4 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Fri, 5 Aug 2022 18:43:13 +0800 Subject: [PATCH 27/35] enhance replaceToAdherePlacementPolicy. --- .../RackawareEnsemblePlacementPolicyImpl.java | 195 ++++++++++-------- 1 file changed, 108 insertions(+), 87 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index bff6c698039..0356a9de8bb 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -26,6 +26,7 @@ import static org.apache.bookkeeper.client.BookKeeperClientStats.READ_REQUESTS_REORDERED; import static org.apache.bookkeeper.client.RegionAwareEnsemblePlacementPolicy.UNKNOWN_REGION; +import com.beust.jcommander.internal.Lists; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -1091,27 +1092,40 @@ public PlacementResult> replaceToAdherePlacementPolicy( int ackQuorumSize, Set excludeBookies, List currentEnsemble) { - PlacementPolicyAdherence currentPlacementAdherence = isEnsembleAdheringToPlacementPolicy( - currentEnsemble, writeQuorumSize, ackQuorumSize); - if (PlacementPolicyAdherence.FAIL != currentPlacementAdherence) { - return PlacementResult.of(new ArrayList<>(currentEnsemble), currentPlacementAdherence); - } - PlacementResult> placementResult = PlacementResult.of(new ArrayList<>(currentEnsemble), - currentPlacementAdherence); - int minDiffer = Integer.MAX_VALUE; - for (int i = 0; i < currentEnsemble.size(); i++) { - PlacementResult> result = doReplaceToAdherePlacementPolicy(ensembleSize, - writeQuorumSize, ackQuorumSize, excludeBookies, currentEnsemble, i); - if (PlacementPolicyAdherence.FAIL == result.getAdheringToPolicy()) { - continue; + rwLock.readLock().lock(); + try { + PlacementPolicyAdherence currentPlacementAdherence = isEnsembleAdheringToPlacementPolicy( + currentEnsemble, writeQuorumSize, ackQuorumSize); + if (PlacementPolicyAdherence.FAIL != currentPlacementAdherence) { + return PlacementResult.of(new ArrayList<>(currentEnsemble), currentPlacementAdherence); } - int differ = differBetweenBookies(currentEnsemble, result.getResult()); - if (differ < minDiffer) { - minDiffer = differ; - placementResult = result; + for (BookieId bookieId : currentEnsemble) { + if (!knownBookies.containsKey(bookieId)) { + excludeBookies.add(bookieId); + } } + PlacementResult> placementResult = PlacementResult.of(new ArrayList<>(currentEnsemble), + currentPlacementAdherence); + int minDiffer = Integer.MAX_VALUE; + for (int i = 0; i < currentEnsemble.size(); i++) { + PlacementResult> result = doReplaceToAdherePlacementPolicy(ensembleSize, + writeQuorumSize, ackQuorumSize, excludeBookies, currentEnsemble, i); + if (PlacementPolicyAdherence.FAIL == result.getAdheringToPolicy()) { + continue; + } + int differ = differBetweenBookies(currentEnsemble, result.getResult()); + if (differ < minDiffer) { + minDiffer = differ; + placementResult = result; + if (minDiffer == 1) { + break; + } + } + } + return placementResult; + } finally { + rwLock.readLock().unlock(); } - return placementResult; } private int differBetweenBookies(List bookiesA, List bookiesB) { @@ -1123,7 +1137,7 @@ private int differBetweenBookies(List bookiesA, List bookies } int differ = 0; for (int i = 0; i < bookiesA.size(); i++) { - if (bookiesA.get(i).equals(bookiesB.get(i))) { + if (!bookiesA.get(i).equals(bookiesB.get(i))) { differ++; } } @@ -1137,81 +1151,88 @@ private PlacementResult> doReplaceToAdherePlacementPolicy( Set excludeBookies, List currentEnsemble, int startIndex) { - rwLock.readLock().lock(); - try { - final List provisionalEnsembleNodes = currentEnsemble.stream() - .map(this::convertBookieToNode).collect(Collectors.toList()); - final Set excludeNodes = convertBookiesToNodes( - addDefaultRackBookiesIfMinNumRacksIsEnforced(excludeBookies)); - int minNumRacksPerWriteQuorumForThisEnsemble = Math.min(writeQuorumSize, minNumRacksPerWriteQuorum); - final RRTopologyAwareCoverageEnsemble ensemble = - new RRTopologyAwareCoverageEnsemble( - ensembleSize, - writeQuorumSize, - ackQuorumSize, - RACKNAME_DISTANCE_FROM_LEAVES, - null, - null, - minNumRacksPerWriteQuorumForThisEnsemble); - int numRacks = topology.getNumOfRacks(); - // only one rack or less than minNumRacksPerWriteQuorumForThisEnsemble, stop calculation to skip relocation - if (numRacks < 2 || numRacks < minNumRacksPerWriteQuorumForThisEnsemble) { - LOG.warn("Skip ensemble relocation because the cluster has only {} rack.", numRacks); - return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); - } - BookieNode prevNode = null; - final BookieNode firstNode = provisionalEnsembleNodes.get(startIndex); - // use same bookie at first to reduce ledger replication - if (!excludeNodes.contains(firstNode) && ensemble.apply(firstNode, ensemble) - && ensemble.addNode(firstNode)) { - excludeNodes.add(firstNode); - prevNode = firstNode; - } - for (int i = prevNode == null ? 0 : 1; i < ensembleSize; i++) { - int index = i % ensembleSize; - final String curRack; - if (null == prevNode) { - if ((null == localNode) || defaultRack.equals(localNode.getNetworkLocation())) { - curRack = NodeBase.ROOT; - } else { - curRack = localNode.getNetworkLocation(); - } + final List provisionalEnsembleNodes = currentEnsemble.stream() + .map(this::convertBookieToNode).collect(Collectors.toList()); + final Set excludeNodes = convertBookiesToNodes( + addDefaultRackBookiesIfMinNumRacksIsEnforced(excludeBookies)); + int minNumRacksPerWriteQuorumForThisEnsemble = Math.min(writeQuorumSize, minNumRacksPerWriteQuorum); + final RRTopologyAwareCoverageEnsemble ensemble = + new RRTopologyAwareCoverageEnsemble( + ensembleSize, + writeQuorumSize, + ackQuorumSize, + RACKNAME_DISTANCE_FROM_LEAVES, + null, + null, + minNumRacksPerWriteQuorumForThisEnsemble); + int numRacks = topology.getNumOfRacks(); + // only one rack or less than minNumRacksPerWriteQuorumForThisEnsemble, stop calculation to skip relocation + if (numRacks < 2 || numRacks < minNumRacksPerWriteQuorumForThisEnsemble) { + LOG.warn("Skip ensemble relocation because the cluster has only {} rack.", numRacks); + return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); + } + BookieNode prevNode = null; + final BookieNode firstNode = provisionalEnsembleNodes.get(startIndex); + // use same bookie at first to reduce ledger replication + if (!excludeNodes.contains(firstNode) && ensemble.apply(firstNode, ensemble) + && ensemble.addNode(firstNode)) { + excludeNodes.add(firstNode); + prevNode = firstNode; + } + for (int i = prevNode == null ? 0 : 1; i < ensembleSize; i++) { + int index = (startIndex + i) % ensembleSize; + final String curRack; + if (null == prevNode) { + if ((null == localNode) || defaultRack.equals(localNode.getNetworkLocation())) { + curRack = NodeBase.ROOT; } else { - curRack = "~" + prevNode.getNetworkLocation(); - } - try { - prevNode = replaceToAdherePlacementPolicyInternal( - curRack, excludeNodes, ensemble, ensemble, - provisionalEnsembleNodes, index, ensembleSize, minNumRacksPerWriteQuorumForThisEnsemble); - // got a good candidate - if (ensemble.addNode(prevNode)) { - // add the candidate to exclude set - excludeNodes.add(prevNode); - } else { - throw new BKNotEnoughBookiesException(); - } - // replace to newer node - provisionalEnsembleNodes.set(index, prevNode); - } catch (BKNotEnoughBookiesException e) { - LOG.warn("Skip ensemble relocation because the cluster has not enough bookies."); - return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); + curRack = localNode.getNetworkLocation(); } + } else { + curRack = "~" + prevNode.getNetworkLocation(); } - List bookieList = ensemble.toList(); - if (ensembleSize != bookieList.size()) { - LOG.warn("Not enough {} bookies are available to form an ensemble : {}.", - ensembleSize, bookieList); + try { + prevNode = replaceToAdherePlacementPolicyInternal( + curRack, excludeNodes, ensemble, ensemble, + provisionalEnsembleNodes, index, ensembleSize, minNumRacksPerWriteQuorumForThisEnsemble); + // got a good candidate + if (ensemble.addNode(prevNode)) { + // add the candidate to exclude set + excludeNodes.add(prevNode); + } else { + throw new BKNotEnoughBookiesException(); + } + // replace to newer node + provisionalEnsembleNodes.set(index, prevNode); + } catch (BKNotEnoughBookiesException e) { + LOG.warn("Skip ensemble relocation because the cluster has not enough bookies."); return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); } - PlacementPolicyAdherence placementPolicyAdherence = isEnsembleAdheringToPlacementPolicy(bookieList, - writeQuorumSize, ackQuorumSize); - if (PlacementPolicyAdherence.FAIL == placementPolicyAdherence) { - return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); + } + List bookieList = ensemble.toList(); + if (ensembleSize != bookieList.size()) { + LOG.warn("Not enough {} bookies are available to form an ensemble : {}.", + ensembleSize, bookieList); + return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); + } + PlacementPolicyAdherence placementPolicyAdherence = isEnsembleAdheringToPlacementPolicy(bookieList, + writeQuorumSize, ackQuorumSize); + if (PlacementPolicyAdherence.FAIL == placementPolicyAdherence) { + return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); + } + + return PlacementResult.of(revertBookieListByIndex(bookieList, startIndex), placementPolicyAdherence); + } + + private List revertBookieListByIndex(List bookies, int startIndex) { + BookieId[] bookieIds = new BookieId[bookies.size()]; + for (int i = 0; i < bookies.size(); i++) { + if (startIndex == bookies.size()) { + startIndex = 0; } - return PlacementResult.of(bookieList, placementPolicyAdherence); - } finally { - rwLock.readLock().unlock(); + bookieIds[startIndex++] = bookies.get(i); } + return Lists.newArrayList(bookieIds); } private BookieNode replaceToAdherePlacementPolicyInternal( From aef4e3ec01939de8aefbfaa018105783def94e29 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Sun, 7 Aug 2022 02:33:40 +0800 Subject: [PATCH 28/35] use replaceToAdherePlacementPolicy to find target bookie. --- .../bookkeeper/client/BookKeeperAdmin.java | 26 +- .../client/EnsemblePlacementPolicy.java | 14 - .../RackawareEnsemblePlacementPolicyImpl.java | 4 +- .../TopologyAwareEnsemblePlacementPolicy.java | 108 ---- .../TestRackawareEnsemblePlacementPolicy.java | 553 ++++++++++++------ .../TestZoneawareEnsemblePlacementPolicy.java | 193 ------ .../replication/TestReplicationWorker.java | 5 - site3/website/docs/reference/config.md | 22 +- 8 files changed, 419 insertions(+), 506 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java index 415d29da703..52357b2346e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java @@ -33,8 +33,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -1139,8 +1141,7 @@ public void replicateLedgerFragment(LedgerHandle lh, final LedgerFragment ledger ledgerFragment.getBookiesIndexes(), excludedBookies); } else if (LedgerFragment.ReplicateType.DATA_NOT_ADHERING_PLACEMENT == ledgerFragment.getReplicateType()) { targetBookieAddresses = replaceNotAdheringPlacementPolicyBookie(ledgerFragment.getEnsemble(), - lh.getLedgerMetadata().getWriteQuorumSize(), lh.getLedgerMetadata().getAckQuorumSize(), - lh.getLedgerMetadata().getCustomMetadata()); + lh.getLedgerMetadata().getWriteQuorumSize(), lh.getLedgerMetadata().getAckQuorumSize()); ledgerFragment.getBookiesIndexes().addAll(targetBookieAddresses.keySet()); } if (MapUtils.isEmpty(targetBookieAddresses)) { @@ -1789,10 +1790,23 @@ public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List replaceNotAdheringPlacementPolicyBookie(List ensembleBookiesList, - int writeQuorumSize, int ackQuorumSize, Map customMetadata) { - return bkc.getPlacementPolicy() - .replaceNotAdheringPlacementPolicyBookie(ensembleBookiesList, writeQuorumSize, ackQuorumSize, - customMetadata); + int writeQuorumSize, int ackQuorumSize) { + EnsemblePlacementPolicy.PlacementResult> placementResult = bkc.getPlacementPolicy() + .replaceToAdherePlacementPolicy(ensembleBookiesList.size(), writeQuorumSize, ackQuorumSize, + new HashSet<>(), ensembleBookiesList); + if (PlacementPolicyAdherence.FAIL != placementResult.getAdheringToPolicy()) { + Map targetMap = new HashMap<>(); + List newEnsembles = placementResult.getResult(); + for (int i = 0; i < ensembleBookiesList.size(); i++) { + BookieId originBookie = ensembleBookiesList.get(i); + BookieId newBookie = newEnsembles.get(i); + if (!originBookie.equals(newBookie)) { + targetMap.put(i, newBookie); + } + } + return targetMap; + } + return Collections.emptyMap(); } /** diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index 35097e954a5..8aade830d0a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -364,20 +364,6 @@ DistributionSchedule.WriteSet reorderReadLACSequence( default void updateBookieInfo(Map bookieInfoMap) { } - /** - * Replace some bookie to adhere placement policy. If the all kinds of replacement - * didn't adhere placement policy, return empty map. - * - * @param ensemble which would be take place. - * @param writeQuorumSize - * @param ackQuorumSize - * @return Map: key means ensemble index, value means target replace bookieId. - */ - default Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, - int ackQuorumSize, Map customMetadata) { - return Collections.emptyMap(); - } - /** * Select one bookie to the "sticky" bookie where all reads for a particular * ledger will be directed to. diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 0356a9de8bb..634f7d9b811 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -1104,8 +1104,8 @@ public PlacementResult> replaceToAdherePlacementPolicy( excludeBookies.add(bookieId); } } - PlacementResult> placementResult = PlacementResult.of(new ArrayList<>(currentEnsemble), - currentPlacementAdherence); + PlacementResult> placementResult = PlacementResult.of(Collections.emptyList(), + PlacementPolicyAdherence.FAIL); int minDiffer = Integer.MAX_VALUE; for (int i = 0; i < currentEnsemble.size(); i++) { PlacementResult> result = doReplaceToAdherePlacementPolicy(ensembleSize, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index 08e7472bf55..c58e3c11aa5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -791,114 +791,6 @@ public void updateBookieInfo(Map bookieInfoMap) { } } - @Override - public Map replaceNotAdheringPlacementPolicyBookie(List ensemble, int writeQuorumSize, - int ackQuorumSize, Map customMetadata) { - rwLock.readLock().lock(); - try { - if (CollectionUtils.isEmpty(ensemble)) { - return Collections.emptyMap(); - } - PlacementPolicyAdherence ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(ensemble, - writeQuorumSize, ackQuorumSize); - if (PlacementPolicyAdherence.FAIL != ensembleAdheringToPlacementPolicy) { - return Collections.emptyMap(); - } - Map bookieIndex = new HashMap<>(); - for (int i = 0; i < ensemble.size(); i++) { - bookieIndex.put(ensemble.get(i), i); - } - Map clone = new HashMap<>(knownBookies); - - Map> toPlaceGroup = new HashMap<>(); - for (BookieId bookieId : ensemble) { - //When ReplicationWorker.getUnderreplicatedFragments, the bookie is alive, so the fragment is not - // data_loss. When find other rack bookie to replace, the bookie maybe shutdown, so here we should pick - // the shutdown bookies. If the bookieId shutdown, put it to inactive. When do replace, we should - // replace inactive bookie firstly. - BookieNode bookieNode = clone.get(bookieId); - if (bookieNode == null) { - List list = toPlaceGroup.computeIfAbsent(NetworkTopology.INACTIVE, - k -> new ArrayList<>()); - list.add(new BookieNode(bookieId, NetworkTopology.INACTIVE)); - } else { - List list = toPlaceGroup.computeIfAbsent(bookieNode.getNetworkLocation(), - k -> new ArrayList<>()); - list.add(bookieNode); - } - } - for (List bookieNodes : toPlaceGroup.values()) { - Collections.shuffle(bookieNodes); - } - - Map> knownRackToBookies = clone.values().stream() - .collect(Collectors.groupingBy(NodeBase::getNetworkLocation)); - HashSet knownRacks = new HashSet<>(knownRackToBookies.keySet()); - - Set excludesBookies = new HashSet<>(); - - for (String key : toPlaceGroup.keySet()) { - List sameRack = knownRackToBookies.get(key); - if (!CollectionUtils.isEmpty(sameRack)) { - excludesBookies.addAll(sameRack.stream().map(BookieNode::getAddr).collect(Collectors.toSet())); - } - } - - Map targetBookieAddresses = new HashMap<>(); - boolean placeSucceed = false; - while (knownRacks.size() > 0) { - BookieNode beReplaceNode = getBeReplaceNode(toPlaceGroup); - if (beReplaceNode == null) { - break; - } - Integer index = bookieIndex.get(beReplaceNode.getAddr()); - try { - PlacementResult placementResult = replaceBookie(ensemble.size(), writeQuorumSize, - ackQuorumSize, customMetadata, ensemble, beReplaceNode.getAddr(), excludesBookies); - BookieNode replaceNode = clone.get(placementResult.getResult()); - String replaceNodeNetwork = replaceNode.getNetworkLocation(); - knownRacks.remove(replaceNodeNetwork); - targetBookieAddresses.put(index, replaceNode.getAddr()); - List bookieNodes = knownRackToBookies.get(replaceNodeNetwork); - if (!CollectionUtils.isEmpty(bookieNodes)) { - excludesBookies.addAll( - bookieNodes.stream().map(BookieNode::getAddr).collect(Collectors.toSet())); - } - } catch (BKException.BKNotEnoughBookiesException e) { - LOG.warn("Didn't find replaced bookie to adhere placement policy.", e); - break; - } - - List toCheckEnsemble = new ArrayList<>(ensemble); - targetBookieAddresses.forEach(toCheckEnsemble::set); - - ensembleAdheringToPlacementPolicy = isEnsembleAdheringToPlacementPolicy(toCheckEnsemble, - writeQuorumSize, ackQuorumSize); - if (PlacementPolicyAdherence.FAIL != ensembleAdheringToPlacementPolicy) { - placeSucceed = true; - break; - } - } - return placeSucceed ? targetBookieAddresses : Collections.emptyMap(); - } finally { - rwLock.readLock().unlock(); - } - } - - private BookieNode getBeReplaceNode(Map> toPlaceGroup) { - List inactiveNodes = toPlaceGroup.get(NetworkTopology.INACTIVE); - if (!CollectionUtils.isEmpty(inactiveNodes)) { - return inactiveNodes.remove(inactiveNodes.size() - 1); - } - Optional>> toPlaceEntry = toPlaceGroup.entrySet().stream() - .filter(ele -> ele.getValue().size() > 1).findAny(); - if (!toPlaceEntry.isPresent()) { - return null; - } - Map.Entry> entry = toPlaceEntry.get(); - return entry.getValue().remove(entry.getValue().size() - 1); - } - protected BookieNode createBookieNode(BookieId addr) { return new BookieNode(addr, resolveNetworkLocation(addr)); } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index 9c04210842f..e696bc4b2ab 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -21,6 +21,9 @@ import static org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl.shuffleWithMask; import static org.apache.bookkeeper.client.RoundRobinDistributionSchedule.writeSetFromValues; import static org.apache.bookkeeper.feature.SettableFeatureProvider.DISABLE_ALL; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.netty.util.HashedWheelTimer; @@ -36,6 +39,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import junit.framework.TestCase; import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; import org.apache.bookkeeper.client.BookieInfoReader.BookieInfo; @@ -60,6 +64,10 @@ import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; import org.apache.bookkeeper.util.StaticDNSResolver; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1464,173 +1472,7 @@ public boolean validate() { // this is expected } } - - @Test - public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { - repp.uninitalize(); - - int minNumRacksPerWriteQuorum = 3; - ClientConfiguration clientConf = new ClientConfiguration(conf); - clientConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorum); - repp = new RackawareEnsemblePlacementPolicy(); - repp.initialize(clientConf, Optional. empty(), timer, DISABLE_ALL, - NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); - repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); - - int numOfRacks = 3; - int numOfBookiesPerRack = 3; - String[] rackLocationNames = new String[numOfRacks]; - List bookieSocketAddresses = new ArrayList(); - Map bookieRackMap = new HashMap(); - BookieId bookieAddress; - - for (int i = 0; i < numOfRacks; i++) { - rackLocationNames[i] = "/default-region/r" + i; - for (int j = 0; j < numOfBookiesPerRack; j++) { - int index = i * numOfBookiesPerRack + j; - bookieAddress = new BookieSocketAddress("128.0.0." + index, 3181).toBookieId(); - StaticDNSResolver.addNodeToRack("128.0.0." + index, rackLocationNames[i]); - bookieSocketAddresses.add(bookieAddress); - bookieRackMap.put(bookieAddress, rackLocationNames[i]); - } - } - - repp.onClusterChanged(new HashSet(bookieSocketAddresses), new HashSet()); - - int writeQuorum = 3; - int ackQuorum = 3; - - //test three knows bookie - List knowsEnsemble = new ArrayList<>(); - knowsEnsemble.add(BookieId.parse("128.0.0.0:3181")); - knowsEnsemble.add(BookieId.parse("128.0.0.1:3181")); - knowsEnsemble.add(BookieId.parse("128.0.0.2:3181")); - - PlacementPolicyAdherence placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( - knowsEnsemble, writeQuorum, ackQuorum); - assertEquals(placementPolicyAdherence, PlacementPolicyAdherence.FAIL); - - Map targetBookie = - repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, writeQuorum, ackQuorum, - Collections.emptyMap()); - //should replace two bookie - assertEquals(2, targetBookie.size()); - - for (Map.Entry entry : targetBookie.entrySet()) { - knowsEnsemble.set(entry.getKey(), entry.getValue()); - } - - placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( - knowsEnsemble, writeQuorum, ackQuorum); - assertEquals(PlacementPolicyAdherence.MEETS_STRICT, placementPolicyAdherence); - - //test three unknowns bookie - List unknownEnsembles = new ArrayList<>(); - unknownEnsembles.add(BookieId.parse("128.0.0.100:3181")); - unknownEnsembles.add(BookieId.parse("128.0.0.101:3181")); - unknownEnsembles.add(BookieId.parse("128.0.0.102:3181")); - - placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( - unknownEnsembles, writeQuorum, ackQuorum); - assertEquals(PlacementPolicyAdherence.FAIL, placementPolicyAdherence); - - //should replace three bookie - targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, writeQuorum, ackQuorum, - Collections.emptyMap()); - assertEquals(targetBookie.size(), 3); - - for (Map.Entry entry : targetBookie.entrySet()) { - unknownEnsembles.set(entry.getKey(), entry.getValue()); - } - - placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( - unknownEnsembles, writeQuorum, ackQuorum); - assertEquals(PlacementPolicyAdherence.MEETS_STRICT, placementPolicyAdherence); - - //test three knows bookie - List adheringEnsemble = new ArrayList<>(); - adheringEnsemble.add(BookieId.parse("128.0.0.0:3181")); - adheringEnsemble.add(BookieId.parse("128.0.0.3:3181")); - adheringEnsemble.add(BookieId.parse("128.0.0.6:3181")); - - placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( - adheringEnsemble, writeQuorum, ackQuorum); - assertEquals(PlacementPolicyAdherence.MEETS_STRICT, placementPolicyAdherence); - - //shouldn't replace already meets_strict ensemble - targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(adheringEnsemble, writeQuorum, ackQuorum, - Collections.emptyMap()); - assertEquals(0, targetBookie.size()); - } - - @Test - public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() throws Exception { - repp.uninitalize(); - - int minNumRacksPerWriteQuorum = 3; - ClientConfiguration clientConf = new ClientConfiguration(conf); - clientConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorum); - repp = new RackawareEnsemblePlacementPolicy(); - repp.initialize(clientConf, Optional. empty(), timer, DISABLE_ALL, - NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); - repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); - - int numOfRacks = 2; - int numOfBookiesPerRack = 3; - String[] rackLocationNames = new String[numOfRacks]; - List bookieSocketAddresses = new ArrayList(); - Map bookieRackMap = new HashMap(); - BookieId bookieAddress; - - for (int i = 0; i < numOfRacks; i++) { - rackLocationNames[i] = "/default-region/r" + i; - for (int j = 0; j < numOfBookiesPerRack; j++) { - int index = i * numOfBookiesPerRack + j; - bookieAddress = new BookieSocketAddress("128.0.0." + index, 3181).toBookieId(); - StaticDNSResolver.addNodeToRack("128.0.0." + index, rackLocationNames[i]); - bookieSocketAddresses.add(bookieAddress); - bookieRackMap.put(bookieAddress, rackLocationNames[i]); - } - } - - repp.onClusterChanged(new HashSet(bookieSocketAddresses), - new HashSet()); - - int writeQuorum = 3; - int ackQuorum = 3; - - //test three knows bookie - List knowsEnsemble = new ArrayList<>(); - knowsEnsemble.add(BookieId.parse("128.0.0.0:3181")); - knowsEnsemble.add(BookieId.parse("128.0.0.1:3181")); - knowsEnsemble.add(BookieId.parse("128.0.0.2:3181")); - - PlacementPolicyAdherence placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( - knowsEnsemble, writeQuorum, ackQuorum); - assertEquals(PlacementPolicyAdherence.FAIL, placementPolicyAdherence); - - Map targetBookie = - repp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, writeQuorum, ackQuorum, - Collections.emptyMap()); - //should not replace - assertEquals(0, targetBookie.size()); - - //test three unknowns bookie - List unknownEnsembles = new ArrayList<>(); - unknownEnsembles.add(BookieId.parse("128.0.0.100:3181")); - unknownEnsembles.add(BookieId.parse("128.0.0.101:3181")); - unknownEnsembles.add(BookieId.parse("128.0.0.102:3181")); - - placementPolicyAdherence = repp.isEnsembleAdheringToPlacementPolicy( - unknownEnsembles, writeQuorum, ackQuorum); - assertEquals(PlacementPolicyAdherence.FAIL, placementPolicyAdherence); - - //should not replace - targetBookie = repp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, writeQuorum, ackQuorum, - Collections.emptyMap()); - assertEquals(0, targetBookie.size()); - } - + @Test public void testNewEnsembleWithMultipleRacks() throws Exception { BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); @@ -2650,4 +2492,381 @@ public void testAreAckedBookiesAdheringToPlacementPolicy() throws Exception { testAreAckedBookiesAdheringToPlacementPolicyHelper(4, 6, 3, 2, 6, 3, 3); testAreAckedBookiesAdheringToPlacementPolicyHelper(5, 7, 5, 3, 7, 5, 2); } + + @SuppressWarnings("unchecked") + @Test + public void testReplaceToAdherePlacementPolicy() throws Exception { + final BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); + final BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181); + final BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181); + final BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); + final BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.5", 3181); + final BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.6", 3181); + final BookieSocketAddress addr7 = new BookieSocketAddress("127.0.0.7", 3181); + final BookieSocketAddress addr8 = new BookieSocketAddress("127.0.0.8", 3181); + final BookieSocketAddress addr9 = new BookieSocketAddress("127.0.0.9", 3181); + + final String rackName1 = NetworkTopology.DEFAULT_REGION + "/r1"; + final String rackName2 = NetworkTopology.DEFAULT_REGION + "/r2"; + final String rackName3 = NetworkTopology.DEFAULT_REGION + "/r3"; + + // update dns mapping + StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr3.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), rackName2); + StaticDNSResolver.addNodeToRack(addr5.getSocketAddress().getAddress().getHostAddress(), rackName2); + StaticDNSResolver.addNodeToRack(addr6.getSocketAddress().getAddress().getHostAddress(), rackName2); + StaticDNSResolver.addNodeToRack(addr7.getSocketAddress().getAddress().getHostAddress(), rackName3); + StaticDNSResolver.addNodeToRack(addr8.getSocketAddress().getAddress().getHostAddress(), rackName3); + StaticDNSResolver.addNodeToRack(addr9.getSocketAddress().getAddress().getHostAddress(), rackName3); + + // Update cluster + final Set addrs = new HashSet<>(); + addrs.add(addr1.toBookieId()); + addrs.add(addr2.toBookieId()); + addrs.add(addr3.toBookieId()); + addrs.add(addr4.toBookieId()); + addrs.add(addr5.toBookieId()); + addrs.add(addr6.toBookieId()); + addrs.add(addr7.toBookieId()); + addrs.add(addr8.toBookieId()); + addrs.add(addr9.toBookieId()); + + final ClientConfiguration newConf = new ClientConfiguration(conf); + newConf.setDiskWeightBasedPlacementEnabled(false); + newConf.setMinNumRacksPerWriteQuorum(2); + newConf.setEnforceMinNumRacksPerWriteQuorum(true); + + repp.initialize(newConf, Optional.empty(), timer, + DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); + + repp.onClusterChanged(addrs, new HashSet<>()); + final Map bookieInfoMap = new HashMap<>(); + bookieInfoMap.put(addr1.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr2.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr3.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr4.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr5.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr6.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr7.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr8.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr9.toBookieId(), new BookieInfo(100L, 100L)); + + repp.updateBookieInfo(bookieInfoMap); + + final Set excludeList = new HashSet<>(); + final int ensembleSize = 7; + final int writeQuorumSize = 2; + final int ackQuorumSize = 2; + + final BookieRackMatcher rack1 = new BookieRackMatcher(rackName1); + final BookieRackMatcher rack2 = new BookieRackMatcher(rackName2); + final BookieRackMatcher rack3 = new BookieRackMatcher(rackName3); + final BookieRackMatcher rack12 = new BookieRackMatcher(rackName1, rackName2); + final BookieRackMatcher rack13 = new BookieRackMatcher(rackName1, rackName3); + final BookieRackMatcher rack23 = new BookieRackMatcher(rackName2, rackName3); + final BookieRackMatcher rack123 = new BookieRackMatcher(rackName1, rackName2, rackName3); + final Consumer, Matcher>>> test = (pair) -> { + // RackawareEnsemblePlacementPolicyImpl#isEnsembleAdheringToPlacementPolicy + // is not scope of this test case. So, use the method in assertion for convenience. + assertEquals(PlacementPolicyAdherence.FAIL, + repp.isEnsembleAdheringToPlacementPolicy(pair.getLeft(), writeQuorumSize, ackQuorumSize)); + final EnsemblePlacementPolicy.PlacementResult> result = + repp.replaceToAdherePlacementPolicy(ensembleSize, writeQuorumSize, ackQuorumSize, + excludeList, pair.getLeft()); + if (LOG.isDebugEnabled()) { + LOG.debug("input: {}, result: {}", pair.getLeft(), result.getResult()); + } + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, result.getAdheringToPolicy()); + assertThat(result.getResult(), pair.getRight()); + }; + + for (int i = 0; i < 1000; i++) { + test.accept(Pair.of(Arrays.asList(addr1.toBookieId(), addr4.toBookieId(), addr7.toBookieId(), + addr2.toBookieId(), addr5.toBookieId(), addr8.toBookieId(), addr9.toBookieId()), + // first, same, same, same, same, same, condition[0] + contains(is(addr1.toBookieId()), is(addr4.toBookieId()), is(addr7.toBookieId()), + is(addr2.toBookieId()), is(addr5.toBookieId()), is(addr8.toBookieId()), + is(addr6.toBookieId())))); + + test.accept(Pair.of(Arrays.asList(addr6.toBookieId(), addr4.toBookieId(), addr7.toBookieId(), + addr2.toBookieId(), addr5.toBookieId(), addr8.toBookieId(), addr3.toBookieId()), + // first, condition[0], same, same, same, same, same + contains(is(addr6.toBookieId()), is(addr1.toBookieId()), is(addr7.toBookieId()), + is(addr2.toBookieId()), is(addr5.toBookieId()), is(addr8.toBookieId()), + is(addr3.toBookieId())))); + + test.accept(Pair.of(Arrays.asList(addr1.toBookieId(), addr2.toBookieId(), addr3.toBookieId(), + addr4.toBookieId(), addr5.toBookieId(), addr6.toBookieId(), addr7.toBookieId()), + // first, candidate[0], same, same, candidate[0], same, same + contains(is(addr1.toBookieId()), is(rack3), is(addr3.toBookieId()), + is(addr4.toBookieId()), is(rack13), is(addr6.toBookieId()), is(addr7.toBookieId())))); + + test.accept(Pair.of(Arrays.asList(addr1.toBookieId(), addr2.toBookieId(), addr4.toBookieId(), + addr5.toBookieId(), addr7.toBookieId(), addr8.toBookieId(), addr9.toBookieId()), + contains(is(addr1.toBookieId()), is(rack23), is(rack123), is(rack123), + is(rack123), is(rack123), is(rack23)))); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testReplaceToAdherePlacementPolicyWithOutOfOrder() throws Exception { + final BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); + final BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181); + final BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181); + final BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); + final BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.5", 3181); + final BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.6", 3181); + + final String rackName1 = NetworkTopology.DEFAULT_REGION + "/r1"; + final String rackName2 = NetworkTopology.DEFAULT_REGION + "/r2"; + + // update dns mapping + StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr3.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), rackName2); + StaticDNSResolver.addNodeToRack(addr5.getSocketAddress().getAddress().getHostAddress(), rackName2); + StaticDNSResolver.addNodeToRack(addr6.getSocketAddress().getAddress().getHostAddress(), rackName2); + + // Update cluster + final Set addrs = new HashSet<>(); + addrs.add(addr1.toBookieId()); + addrs.add(addr2.toBookieId()); + addrs.add(addr3.toBookieId()); + addrs.add(addr4.toBookieId()); + addrs.add(addr5.toBookieId()); + addrs.add(addr6.toBookieId()); + + final ClientConfiguration newConf = new ClientConfiguration(conf); + newConf.setDiskWeightBasedPlacementEnabled(false); + newConf.setMinNumRacksPerWriteQuorum(2); + newConf.setEnforceMinNumRacksPerWriteQuorum(true); + + repp.initialize(newConf, Optional.empty(), timer, + DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); + + repp.onClusterChanged(addrs, new HashSet<>()); + final Map bookieInfoMap = new HashMap<>(); + bookieInfoMap.put(addr1.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr2.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr3.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr4.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr5.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr6.toBookieId(), new BookieInfo(100L, 100L)); + + repp.updateBookieInfo(bookieInfoMap); + + final Set excludeList = new HashSet<>(); + final int ensembleSize = 6; + final int writeQuorumSize = 2; + final int ackQuorumSize = 2; + + final Consumer, Matcher>>> test = (pair) -> { + // RackawareEnsemblePlacementPolicyImpl#isEnsembleAdheringToPlacementPolicy + // is not scope of this test case. So, use the method in assertion for convenience. + assertEquals(PlacementPolicyAdherence.FAIL, + repp.isEnsembleAdheringToPlacementPolicy(pair.getLeft(), writeQuorumSize, ackQuorumSize)); + final EnsemblePlacementPolicy.PlacementResult> result = + repp.replaceToAdherePlacementPolicy(ensembleSize, writeQuorumSize, ackQuorumSize, + excludeList, pair.getLeft()); + if (LOG.isDebugEnabled()) { + LOG.debug("input: {}, result: {}", pair.getLeft(), result.getResult()); + } + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, result.getAdheringToPolicy()); + }; + + for (int i = 0; i < 1000; i++) { + //All bookies already in the ensemble, the bookie order not adhere the placement policy. + test.accept(Pair.of(Arrays.asList(addr1.toBookieId(), addr2.toBookieId(), addr3.toBookieId(), + addr4.toBookieId(), addr5.toBookieId(), addr6.toBookieId()), + //The result is not predict. We know the best min replace place is 2. + //1,2,3,4,5,6 => 1,5,3,4,2,6 + //But maybe the final result is 1,6,3,4,2,5. + //When we from index 0 to replace, the first bookie(1) is /rack1, we only pick /rack2 bookie + //for the second bookie, so we can choose 4,5,6, the choice is random. If we pick 6 for the second, + // the final result is 1,6,3,4,2,5. If we pick 5 for the second, the final result is 1,5,3,4,2,6 + null)); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testReplaceToAdherePlacementPolicyWithNoMoreRackBookie() throws Exception { + final BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); + final BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181); + final BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181); + final BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); + final BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.5", 3181); + final BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.6", 3181); + + final String rackName1 = NetworkTopology.DEFAULT_REGION + "/r1"; + final String rackName2 = NetworkTopology.DEFAULT_REGION + "/r2"; + + // update dns mapping + StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr3.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), rackName2); + StaticDNSResolver.addNodeToRack(addr5.getSocketAddress().getAddress().getHostAddress(), rackName2); + StaticDNSResolver.addNodeToRack(addr6.getSocketAddress().getAddress().getHostAddress(), rackName2); + + // Update cluster + final Set addrs = new HashSet<>(); + addrs.add(addr1.toBookieId()); + addrs.add(addr2.toBookieId()); + addrs.add(addr3.toBookieId()); + addrs.add(addr4.toBookieId()); + addrs.add(addr5.toBookieId()); + addrs.add(addr6.toBookieId()); + + final ClientConfiguration newConf = new ClientConfiguration(conf); + newConf.setDiskWeightBasedPlacementEnabled(false); + newConf.setMinNumRacksPerWriteQuorum(2); + newConf.setEnforceMinNumRacksPerWriteQuorum(true); + + repp.initialize(newConf, Optional.empty(), timer, + DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); + + repp.onClusterChanged(addrs, new HashSet<>()); + final Map bookieInfoMap = new HashMap<>(); + bookieInfoMap.put(addr1.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr2.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr3.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr4.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr5.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr6.toBookieId(), new BookieInfo(100L, 100L)); + + repp.updateBookieInfo(bookieInfoMap); + + final Set excludeList = new HashSet<>(); + final int ensembleSize = 3; + final int writeQuorumSize = 2; + final int ackQuorumSize = 2; + + final Consumer, Matcher>>> test = (pair) -> { + // RackawareEnsemblePlacementPolicyImpl#isEnsembleAdheringToPlacementPolicy + // is not scope of this test case. So, use the method in assertion for convenience. + assertEquals(PlacementPolicyAdherence.FAIL, + repp.isEnsembleAdheringToPlacementPolicy(pair.getLeft(), writeQuorumSize, ackQuorumSize)); + final EnsemblePlacementPolicy.PlacementResult> result = + repp.replaceToAdherePlacementPolicy(ensembleSize, writeQuorumSize, ackQuorumSize, + excludeList, pair.getLeft()); + if (LOG.isDebugEnabled()) { + LOG.debug("input: {}, result: {}", pair.getLeft(), result.getResult()); + } + assertEquals(PlacementPolicyAdherence.FAIL, result.getAdheringToPolicy()); + assertEquals(0, result.getResult().size()); + }; + + for (int i = 0; i < 1000; i++) { + test.accept(Pair.of(Arrays.asList(addr1.toBookieId(), addr2.toBookieId(), addr4.toBookieId()), + null)); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testReplaceToAdherePlacementPolicyWithUnknowBookie() throws Exception { + final BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); + final BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181); + final BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181); + final BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); + final BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.5", 3181); + final BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.6", 3181); + + final String rackName1 = NetworkTopology.DEFAULT_REGION + "/r1"; + final String rackName2 = NetworkTopology.DEFAULT_REGION + "/r2"; + + // update dns mapping + StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr3.getSocketAddress().getAddress().getHostAddress(), rackName1); + StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), rackName2); + StaticDNSResolver.addNodeToRack(addr5.getSocketAddress().getAddress().getHostAddress(), rackName2); + StaticDNSResolver.addNodeToRack(addr6.getSocketAddress().getAddress().getHostAddress(), rackName2); + + // Update cluster + final Set addrs = new HashSet<>(); + addrs.add(addr1.toBookieId()); + addrs.add(addr2.toBookieId()); + addrs.add(addr3.toBookieId()); + addrs.add(addr4.toBookieId()); + addrs.add(addr5.toBookieId()); + addrs.add(addr6.toBookieId()); + + final ClientConfiguration newConf = new ClientConfiguration(conf); + newConf.setDiskWeightBasedPlacementEnabled(false); + newConf.setMinNumRacksPerWriteQuorum(2); + newConf.setEnforceMinNumRacksPerWriteQuorum(true); + + repp.initialize(newConf, Optional.empty(), timer, + DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); + + repp.onClusterChanged(addrs, new HashSet<>()); + final Map bookieInfoMap = new HashMap<>(); + bookieInfoMap.put(addr1.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr2.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr3.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr4.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr5.toBookieId(), new BookieInfo(100L, 100L)); + bookieInfoMap.put(addr6.toBookieId(), new BookieInfo(100L, 100L)); + + repp.updateBookieInfo(bookieInfoMap); + + final Set excludeList = new HashSet<>(); + final int ensembleSize = 6; + final int writeQuorumSize = 2; + final int ackQuorumSize = 2; + + final BookieRackMatcher rack1 = new BookieRackMatcher(rackName1); + + final Consumer, Matcher>>> test = (pair) -> { + // RackawareEnsemblePlacementPolicyImpl#isEnsembleAdheringToPlacementPolicy + // is not scope of this test case. So, use the method in assertion for convenience. + assertEquals(PlacementPolicyAdherence.FAIL, + repp.isEnsembleAdheringToPlacementPolicy(pair.getLeft(), writeQuorumSize, ackQuorumSize)); + final EnsemblePlacementPolicy.PlacementResult> result = + repp.replaceToAdherePlacementPolicy(ensembleSize, writeQuorumSize, ackQuorumSize, + excludeList, pair.getLeft()); + if (LOG.isDebugEnabled()) { + LOG.debug("input: {}, result: {}", pair.getLeft(), result.getResult()); + } + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, result.getAdheringToPolicy()); + assertThat(result.getResult(), pair.getRight()); + }; + + for (int i = 0; i < 1000; i++) { + test.accept(Pair.of(Arrays.asList(BookieId.parse("127.0.0.10:3181"), BookieId.parse("127.0.0.11:3181"), + addr3.toBookieId(), + addr4.toBookieId(), addr5.toBookieId(), addr6.toBookieId()), + // first, same, same, same, same, same, condition[0] + contains(is(rack1), is(addr5.toBookieId()), is(addr3.toBookieId()), + is(addr4.toBookieId()), is(rack1), is(addr6.toBookieId())))); + } + } + + private static class BookieRackMatcher extends TypeSafeMatcher { + final List expectedRacks; + + public BookieRackMatcher(String... expectedRacks) { + this.expectedRacks = Arrays.asList(expectedRacks); + } + + @Override + protected boolean matchesSafely(BookieId bookieId) { + return expectedRacks.contains(StaticDNSResolver.getRack(bookieId.toString().split(":")[0])); + } + + @Override + public void describeTo(Description description) { + description.appendText("expected racks " + expectedRacks); + } + } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java index 91f2e7ef210..b191225c834 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestZoneawareEnsemblePlacementPolicy.java @@ -1365,197 +1365,4 @@ public void testIsEnsembleAdheringToPlacementPolicy() throws Exception { assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_SOFT, zepp.isEnsembleAdheringToPlacementPolicy(ensemble, 3, 2)); } - - @Test - public void testReplaceNotAdheringPlacementPolicyBookie() throws Exception { - zepp.uninitalize(); - updateMyUpgradeDomain(NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); - - // Update cluster - BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.6", 3181); - BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.7", 3181); - BookieSocketAddress addr7 = new BookieSocketAddress("127.0.0.8", 3181); - BookieSocketAddress addr8 = new BookieSocketAddress("127.0.0.9", 3181); - BookieSocketAddress addr9 = new BookieSocketAddress("127.0.0.10", 3181); - BookieSocketAddress addr10 = new BookieSocketAddress("127.0.0.11", 3181); - - // update dns mapping - StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/zone1/ud1"); - StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/zone1/ud2"); - StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/zone1/ud2"); - StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/zone2/ud1"); - StaticDNSResolver.addNodeToRack(addr5.getHostName(), "/zone2/ud2"); - StaticDNSResolver.addNodeToRack(addr6.getHostName(), "/zone2/ud2"); - StaticDNSResolver.addNodeToRack(addr7.getHostName(), "/zone3/ud1"); - StaticDNSResolver.addNodeToRack(addr8.getHostName(), "/zone3/ud2"); - StaticDNSResolver.addNodeToRack(addr9.getHostName(), "/zone3/ud2"); - StaticDNSResolver.addNodeToRack(addr10.getHostName(), NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); - - zepp = new ZoneawareEnsemblePlacementPolicy(); - ClientConfiguration confLocal = new ClientConfiguration(); - confLocal.addConfiguration(conf); - confLocal.setMinNumZonesPerWriteQuorum(2); - confLocal.setDesiredNumZonesPerWriteQuorum(3); - zepp.initialize(confLocal, Optional. empty(), timer, DISABLE_ALL, - NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); - zepp.withDefaultFaultDomain(NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); - - Set writableBookies = new HashSet<>(); - writableBookies.add(addr1.toBookieId()); - writableBookies.add(addr2.toBookieId()); - writableBookies.add(addr3.toBookieId()); - writableBookies.add(addr4.toBookieId()); - writableBookies.add(addr5.toBookieId()); - writableBookies.add(addr6.toBookieId()); - writableBookies.add(addr7.toBookieId()); - writableBookies.add(addr8.toBookieId()); - writableBookies.add(addr9.toBookieId()); - - zepp.onClusterChanged(new HashSet(writableBookies), new HashSet()); - - //all bookie in /zone1 - List knowsEnsemble = new ArrayList(); - - knowsEnsemble.add(BookieId.parse("127.0.0.2:3181")); // /zone1/ud1 - knowsEnsemble.add(BookieId.parse("127.0.0.3:3181")); // /zone1/ud2 - knowsEnsemble.add(BookieId.parse("127.0.0.4:3181")); // /zone1/ud2 - - assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, - zepp.isEnsembleAdheringToPlacementPolicy(knowsEnsemble, 3, 2)); - - Map targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, 3, 2, - Collections.emptyMap()); - - assertEquals(1, targetBookie.size()); - - for (Map.Entry entry : targetBookie.entrySet()) { - knowsEnsemble.set(entry.getKey(), entry.getValue()); - } - - assertNotEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, - zepp.isEnsembleAdheringToPlacementPolicy(knowsEnsemble, 3, 2)); - - List unknownEnsembles = new ArrayList(); - unknownEnsembles.add(BookieId.parse("127.0.0.100:3181")); // /inactive - unknownEnsembles.add(BookieId.parse("127.0.0.101:3181")); // /inactive - unknownEnsembles.add(BookieId.parse("127.0.0.102:3181")); // /inactive - - assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, - zepp.isEnsembleAdheringToPlacementPolicy(unknownEnsembles, 3, 2)); - - targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, 3, 2, - Collections.emptyMap()); - - assertEquals(3, targetBookie.size()); - - for (Map.Entry entry : targetBookie.entrySet()) { - unknownEnsembles.set(entry.getKey(), entry.getValue()); - } - - assertNotEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, - zepp.isEnsembleAdheringToPlacementPolicy(unknownEnsembles, 3, 2)); - - //test three knows bookie - List adheringStrictEnsemble = new ArrayList<>(); - adheringStrictEnsemble.add(BookieId.parse("127.0.0.2:3181")); // /zone1/ud1 - adheringStrictEnsemble.add(BookieId.parse("127.0.0.6:3181")); // /zone2/ud2 - adheringStrictEnsemble.add(BookieId.parse("127.0.0.10:3181")); // /zone3/ud2 - - assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_STRICT, - zepp.isEnsembleAdheringToPlacementPolicy(adheringStrictEnsemble, 3, 2)); - - //shouldn't replace already meets_strict ensemble - targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(adheringStrictEnsemble, 3, 2, - Collections.emptyMap()); - assertEquals(0, targetBookie.size()); - - //test three knows bookie - List adheringSoftEnsemble = new ArrayList<>(); - adheringSoftEnsemble.add(BookieId.parse("127.0.0.2:3181")); // /zone1/ud1 - adheringSoftEnsemble.add(BookieId.parse("127.0.0.8:3181")); // /zone3/ud1 - adheringSoftEnsemble.add(BookieId.parse("127.0.0.9:3181")); // /zone3/ud2 - - assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.MEETS_SOFT, - zepp.isEnsembleAdheringToPlacementPolicy(adheringSoftEnsemble, 3, 2)); - - //shouldn't replace already meets_soft ensemble - targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(adheringSoftEnsemble, 3, 2, - Collections.emptyMap()); - assertEquals(0, targetBookie.size()); - } - - @Test - public void testReplaceNotAdheringPlacementPolicyBookieWithNoMoreRackBookie() throws Exception { - zepp.uninitalize(); - updateMyUpgradeDomain(NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); - - // Update cluster - BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.6", 3181); - BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.7", 3181); - BookieSocketAddress addr7 = new BookieSocketAddress("127.0.0.8", 3181); - BookieSocketAddress addr8 = new BookieSocketAddress("127.0.0.9", 3181); - BookieSocketAddress addr9 = new BookieSocketAddress("127.0.0.10", 3181); - BookieSocketAddress addr10 = new BookieSocketAddress("127.0.0.11", 3181); - - // update dns mapping - StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/zone1/ud1"); - StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/zone1/ud2"); - StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/zone1/ud3"); - StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/zone1/ud4"); - StaticDNSResolver.addNodeToRack(addr5.getHostName(), "/zone1/ud5"); - StaticDNSResolver.addNodeToRack(addr6.getHostName(), "/zone1/ud6"); - StaticDNSResolver.addNodeToRack(addr7.getHostName(), "/zone1/ud7"); - StaticDNSResolver.addNodeToRack(addr8.getHostName(), "/zone1/ud8"); - StaticDNSResolver.addNodeToRack(addr9.getHostName(), "/zone1/ud9"); - StaticDNSResolver.addNodeToRack(addr10.getHostName(), NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); - - zepp = new ZoneawareEnsemblePlacementPolicy(); - ClientConfiguration confLocal = new ClientConfiguration(); - confLocal.addConfiguration(conf); - confLocal.setMinNumZonesPerWriteQuorum(2); - confLocal.setDesiredNumZonesPerWriteQuorum(3); - zepp.initialize(confLocal, Optional. empty(), timer, DISABLE_ALL, - NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); - zepp.withDefaultFaultDomain(NetworkTopology.DEFAULT_ZONE_AND_UPGRADEDOMAIN); - - Set writableBookies = new HashSet<>(); - writableBookies.add(addr1.toBookieId()); - writableBookies.add(addr2.toBookieId()); - writableBookies.add(addr3.toBookieId()); - writableBookies.add(addr4.toBookieId()); - writableBookies.add(addr5.toBookieId()); - writableBookies.add(addr6.toBookieId()); - writableBookies.add(addr7.toBookieId()); - writableBookies.add(addr8.toBookieId()); - writableBookies.add(addr9.toBookieId()); - - zepp.onClusterChanged(new HashSet(writableBookies), new HashSet()); - - //all bookie in /zone1 - List knowsEnsemble = new ArrayList(); - knowsEnsemble.add(BookieId.parse("127.0.0.2:3181")); // /zone1/ud1 - knowsEnsemble.add(BookieId.parse("127.0.0.3:3181")); // /zone1/ud2 - knowsEnsemble.add(BookieId.parse("127.0.0.4:3181")); // /zone1/ud3 - - assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, - zepp.isEnsembleAdheringToPlacementPolicy(knowsEnsemble, 3, 2)); - - Map targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(knowsEnsemble, 3, 2, - Collections.emptyMap()); - - assertEquals(0, targetBookie.size()); - - List unknownEnsembles = new ArrayList(); - unknownEnsembles.add(BookieId.parse("127.0.0.100:3181")); // /inactive - unknownEnsembles.add(BookieId.parse("127.0.0.101:3181")); // /inactive - unknownEnsembles.add(BookieId.parse("127.0.0.102:3181")); // /inactive - - assertEquals("PlacementPolicyAdherence", PlacementPolicyAdherence.FAIL, - zepp.isEnsembleAdheringToPlacementPolicy(unknownEnsembles, 3, 2)); - //no more bookie can be replaced - targetBookie = zepp.replaceNotAdheringPlacementPolicyBookie(unknownEnsembles, 3, 2, - Collections.emptyMap()); - - assertEquals(0, targetBookie.size()); - } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java index 70042191a67..510d02f8db6 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -1154,11 +1154,6 @@ public void testRWShutDownInTheCaseOfZKOperationFailures() throws Exception { } } - @Test - public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsOnZone() throws Exception { - testRepairedNotAdheringPlacementPolicyLedgerFragments(ZoneawareEnsemblePlacementPolicy.class); - } - @Test public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsOnRack() throws Exception { testRepairedNotAdheringPlacementPolicyLedgerFragments(RackawareEnsemblePlacementPolicy.class); diff --git a/site3/website/docs/reference/config.md b/site3/website/docs/reference/config.md index 63bd279a4e5..232100a9701 100644 --- a/site3/website/docs/reference/config.md +++ b/site3/website/docs/reference/config.md @@ -325,17 +325,17 @@ The table below lists parameters that you can set to configure bookies. All conf ## AutoRecovery auditor settings -| Parameter | Description | Default -| --------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| -| auditorPeriodicBookieCheckInterval | The time interval between auditor bookie checks, in seconds. The auditor bookie check checks ledger metadata to see which bookies should contain entries for each ledger. If a bookie that should contain entries is unavailable, then the ledger containing that entry is marked for recovery. Setting this to 0 disables the periodic check. Bookie checks will still run when a bookie fails. The default is once per day. | 86400 | -| auditorPeriodicCheckInterval | The time interval, in seconds, at which the auditor will check all ledgers in the cluster. By default this runs once a week.

Set this to 0 to disable the periodic check completely. Note that periodic checking will put extra load on the cluster, so it should not be run more frequently than once a day.
| 604800 | -| auditorPeriodicPlacementPolicyCheckInterval | The time interval between auditor placement policy checks, in seconds. The auditor placement policy check validates if the ensemble of segments of all the closed ledgers is adhering to the placement policy. It is just monitoring scrutiny but doesn't take any corrective measure other than logging error and reporting metrics. By default, it is disabled. | 0 | -| repairedPlacementPolicyNotAdheringBookieEnabled | In Auditor, it combines with auditorPeriodicPlacementPolicyCheckInterval, to control is marked ledger id to under replication managed when found a ledger ensemble not adhere to placement policy. In ReplicationWorker, to control is to repair the ledger which the ensemble does not adhere to the placement policy. By default, it is disabled. If you want to enable this feature, there maybe lots of ledger will be mark underreplicated. The replicationWorker will replicate lots of ledger, it will increase read request and write request in bookie server. You should set a suitable rereplicationEntryBatchSize to avoid bookie server pressure. | false | -| auditorLedgerVerificationPercentage | The percentage of a ledger (fragment)'s entries will be verified before claiming a fragment as missing. If it is 0, it only verifies the first and last entries of a given fragment.
| | -| lostBookieRecoveryDelay | How long to wait, in seconds, before starting autorecovery of a lost bookie. | | -| storeSystemTimeAsLedgerUnderreplicatedMarkTime | Enable the Auditor to use system time as underreplicated ledger mark time. If this is enabled, Auditor will write a ctime field into the underreplicated ledger znode. | true | -| underreplicatedLedgerRecoveryGracePeriod | The grace period (in seconds) for underreplicated ledgers recovery. If ledger is marked underreplicated for more than this period then it will be reported by placementPolicyCheck in Auditor. Setting this to 0 will disable this check. | | -| auditorReplicasCheckInterval | Sets the regularity/interval at which the auditor will run a replicas check of all ledgers, which are closed. This should not be run very often since it validates availability of replicas of all ledgers by querying bookies. Setting this to 0 will completely disable the periodic replicas check. By default it is disabled. | | +| Parameter | Description | Default +| --------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| auditorPeriodicBookieCheckInterval | The time interval between auditor bookie checks, in seconds. The auditor bookie check checks ledger metadata to see which bookies should contain entries for each ledger. If a bookie that should contain entries is unavailable, then the ledger containing that entry is marked for recovery. Setting this to 0 disables the periodic check. Bookie checks will still run when a bookie fails. The default is once per day. | 86400 | +| auditorPeriodicCheckInterval | The time interval, in seconds, at which the auditor will check all ledgers in the cluster. By default this runs once a week.

Set this to 0 to disable the periodic check completely. Note that periodic checking will put extra load on the cluster, so it should not be run more frequently than once a day.
| 604800 | +| auditorPeriodicPlacementPolicyCheckInterval | The time interval between auditor placement policy checks, in seconds. The auditor placement policy check validates if the ensemble of segments of all the closed ledgers is adhering to the placement policy. It is just monitoring scrutiny but doesn't take any corrective measure other than logging error and reporting metrics. By default, it is disabled. | 0 | +| repairedPlacementPolicyNotAdheringBookieEnabled | In Auditor, it combines with auditorPeriodicPlacementPolicyCheckInterval, to control is marked ledger id to under replication managed when found a ledger ensemble not adhere to placement policy. In ReplicationWorker, to control is to repair the ledger which the ensemble does not adhere to the placement policy. By default, it is disabled. If you want to enable this feature, consider two factors. 1:Must config RackawareEnsemblePlacementPolicy. 2:There maybe lots of ledger will be mark underreplicated. The replicationWorker will replicate lots of ledger, it will increase read request and write request in bookie server. You should set a suitable rereplicationEntryBatchSize to avoid bookie server pressure. | false | +| auditorLedgerVerificationPercentage | The percentage of a ledger (fragment)'s entries will be verified before claiming a fragment as missing. If it is 0, it only verifies the first and last entries of a given fragment.
| | +| lostBookieRecoveryDelay | How long to wait, in seconds, before starting autorecovery of a lost bookie. | | +| storeSystemTimeAsLedgerUnderreplicatedMarkTime | Enable the Auditor to use system time as underreplicated ledger mark time. If this is enabled, Auditor will write a ctime field into the underreplicated ledger znode. | true | +| underreplicatedLedgerRecoveryGracePeriod | The grace period (in seconds) for underreplicated ledgers recovery. If ledger is marked underreplicated for more than this period then it will be reported by placementPolicyCheck in Auditor. Setting this to 0 will disable this check. | | +| auditorReplicasCheckInterval | Sets the regularity/interval at which the auditor will run a replicas check of all ledgers, which are closed. This should not be run very often since it validates availability of replicas of all ledgers by querying bookies. Setting this to 0 will completely disable the periodic replicas check. By default it is disabled. | | ## AutoRecovery replication worker settings From 98147d7ab05d48bb2eb7da734c1be41d527e1bac Mon Sep 17 00:00:00 2001 From: horizonzy Date: Sun, 7 Aug 2022 02:43:26 +0800 Subject: [PATCH 29/35] fix checkstyle. --- .../client/EnsemblePlacementPolicy.java | 1 - .../RackawareEnsemblePlacementPolicyImpl.java | 12 +-- .../TopologyAwareEnsemblePlacementPolicy.java | 3 - .../TestRackawareEnsemblePlacementPolicy.java | 99 +++++++++---------- 4 files changed, 54 insertions(+), 61 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index 8aade830d0a..12e56d717cf 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -18,7 +18,6 @@ package org.apache.bookkeeper.client; import io.netty.util.HashedWheelTimer; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 634f7d9b811..4cd08ed0795 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -31,7 +31,6 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import io.netty.util.HashedWheelTimer; - import java.io.IOException; import java.net.InetAddress; import java.util.ArrayList; @@ -1127,7 +1126,7 @@ public PlacementResult> replaceToAdherePlacementPolicy( rwLock.readLock().unlock(); } } - + private int differBetweenBookies(List bookiesA, List bookiesB) { if (CollectionUtils.isEmpty(bookiesA) || CollectionUtils.isEmpty(bookiesB)) { return Integer.MAX_VALUE; @@ -1143,7 +1142,7 @@ private int differBetweenBookies(List bookiesA, List bookies } return differ; } - + private PlacementResult> doReplaceToAdherePlacementPolicy( int ensembleSize, int writeQuorumSize, @@ -1220,10 +1219,9 @@ private PlacementResult> doReplaceToAdherePlacementPolicy( if (PlacementPolicyAdherence.FAIL == placementPolicyAdherence) { return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); } - return PlacementResult.of(revertBookieListByIndex(bookieList, startIndex), placementPolicyAdherence); } - + private List revertBookieListByIndex(List bookies, int startIndex) { BookieId[] bookieIds = new BookieId[bookies.size()]; for (int i = 0; i < bookies.size(); i++) { @@ -1234,7 +1232,7 @@ private List revertBookieListByIndex(List bookies, int start } return Lists.newArrayList(bookieIds); } - + private BookieNode replaceToAdherePlacementPolicyInternal( String netPath, Set excludeBookies, Predicate predicate, Ensemble ensemble, List provisionalEnsembleNodes, int ensembleIndex, @@ -1266,7 +1264,7 @@ private BookieNode replaceToAdherePlacementPolicyInternal( conditionList.add(Pair.of(netPath, provisionalEnsembleNodes)); // avoid to use same rack between previous index by netPath conditionList.add(Pair.of(netPath, Collections.emptyList())); - + for (Pair> condition : conditionList) { WeightedRandomSelection wRSelection = null; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index c58e3c11aa5..58a971e682e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -33,12 +33,10 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; -import java.util.stream.Collectors; import org.apache.bookkeeper.client.BookieInfoReader.BookieInfo; import org.apache.bookkeeper.client.WeightedRandomSelection.WeightedObject; import org.apache.bookkeeper.net.BookieId; @@ -54,7 +52,6 @@ import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.OpStatsLogger; import org.apache.bookkeeper.stats.annotations.StatsDoc; -import org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index e696bc4b2ab..ac3974ffe5d 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -2492,7 +2492,7 @@ public void testAreAckedBookiesAdheringToPlacementPolicy() throws Exception { testAreAckedBookiesAdheringToPlacementPolicyHelper(4, 6, 3, 2, 6, 3, 3); testAreAckedBookiesAdheringToPlacementPolicyHelper(5, 7, 5, 3, 7, 5, 2); } - + @SuppressWarnings("unchecked") @Test public void testReplaceToAdherePlacementPolicy() throws Exception { @@ -2505,11 +2505,11 @@ public void testReplaceToAdherePlacementPolicy() throws Exception { final BookieSocketAddress addr7 = new BookieSocketAddress("127.0.0.7", 3181); final BookieSocketAddress addr8 = new BookieSocketAddress("127.0.0.8", 3181); final BookieSocketAddress addr9 = new BookieSocketAddress("127.0.0.9", 3181); - + final String rackName1 = NetworkTopology.DEFAULT_REGION + "/r1"; final String rackName2 = NetworkTopology.DEFAULT_REGION + "/r2"; final String rackName3 = NetworkTopology.DEFAULT_REGION + "/r3"; - + // update dns mapping StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), rackName1); StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), rackName1); @@ -2520,7 +2520,7 @@ public void testReplaceToAdherePlacementPolicy() throws Exception { StaticDNSResolver.addNodeToRack(addr7.getSocketAddress().getAddress().getHostAddress(), rackName3); StaticDNSResolver.addNodeToRack(addr8.getSocketAddress().getAddress().getHostAddress(), rackName3); StaticDNSResolver.addNodeToRack(addr9.getSocketAddress().getAddress().getHostAddress(), rackName3); - + // Update cluster final Set addrs = new HashSet<>(); addrs.add(addr1.toBookieId()); @@ -2532,16 +2532,16 @@ public void testReplaceToAdherePlacementPolicy() throws Exception { addrs.add(addr7.toBookieId()); addrs.add(addr8.toBookieId()); addrs.add(addr9.toBookieId()); - + final ClientConfiguration newConf = new ClientConfiguration(conf); newConf.setDiskWeightBasedPlacementEnabled(false); newConf.setMinNumRacksPerWriteQuorum(2); newConf.setEnforceMinNumRacksPerWriteQuorum(true); - + repp.initialize(newConf, Optional.empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); - + repp.onClusterChanged(addrs, new HashSet<>()); final Map bookieInfoMap = new HashMap<>(); bookieInfoMap.put(addr1.toBookieId(), new BookieInfo(100L, 100L)); @@ -2553,14 +2553,14 @@ public void testReplaceToAdherePlacementPolicy() throws Exception { bookieInfoMap.put(addr7.toBookieId(), new BookieInfo(100L, 100L)); bookieInfoMap.put(addr8.toBookieId(), new BookieInfo(100L, 100L)); bookieInfoMap.put(addr9.toBookieId(), new BookieInfo(100L, 100L)); - + repp.updateBookieInfo(bookieInfoMap); - + final Set excludeList = new HashSet<>(); final int ensembleSize = 7; final int writeQuorumSize = 2; final int ackQuorumSize = 2; - + final BookieRackMatcher rack1 = new BookieRackMatcher(rackName1); final BookieRackMatcher rack2 = new BookieRackMatcher(rackName2); final BookieRackMatcher rack3 = new BookieRackMatcher(rackName3); @@ -2582,7 +2582,7 @@ public void testReplaceToAdherePlacementPolicy() throws Exception { assertEquals(PlacementPolicyAdherence.MEETS_STRICT, result.getAdheringToPolicy()); assertThat(result.getResult(), pair.getRight()); }; - + for (int i = 0; i < 1000; i++) { test.accept(Pair.of(Arrays.asList(addr1.toBookieId(), addr4.toBookieId(), addr7.toBookieId(), addr2.toBookieId(), addr5.toBookieId(), addr8.toBookieId(), addr9.toBookieId()), @@ -2590,27 +2590,27 @@ public void testReplaceToAdherePlacementPolicy() throws Exception { contains(is(addr1.toBookieId()), is(addr4.toBookieId()), is(addr7.toBookieId()), is(addr2.toBookieId()), is(addr5.toBookieId()), is(addr8.toBookieId()), is(addr6.toBookieId())))); - + test.accept(Pair.of(Arrays.asList(addr6.toBookieId(), addr4.toBookieId(), addr7.toBookieId(), addr2.toBookieId(), addr5.toBookieId(), addr8.toBookieId(), addr3.toBookieId()), // first, condition[0], same, same, same, same, same contains(is(addr6.toBookieId()), is(addr1.toBookieId()), is(addr7.toBookieId()), is(addr2.toBookieId()), is(addr5.toBookieId()), is(addr8.toBookieId()), is(addr3.toBookieId())))); - + test.accept(Pair.of(Arrays.asList(addr1.toBookieId(), addr2.toBookieId(), addr3.toBookieId(), addr4.toBookieId(), addr5.toBookieId(), addr6.toBookieId(), addr7.toBookieId()), // first, candidate[0], same, same, candidate[0], same, same contains(is(addr1.toBookieId()), is(rack3), is(addr3.toBookieId()), is(addr4.toBookieId()), is(rack13), is(addr6.toBookieId()), is(addr7.toBookieId())))); - + test.accept(Pair.of(Arrays.asList(addr1.toBookieId(), addr2.toBookieId(), addr4.toBookieId(), addr5.toBookieId(), addr7.toBookieId(), addr8.toBookieId(), addr9.toBookieId()), contains(is(addr1.toBookieId()), is(rack23), is(rack123), is(rack123), is(rack123), is(rack123), is(rack23)))); } } - + @SuppressWarnings("unchecked") @Test public void testReplaceToAdherePlacementPolicyWithOutOfOrder() throws Exception { @@ -2620,10 +2620,10 @@ public void testReplaceToAdherePlacementPolicyWithOutOfOrder() throws Exception final BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); final BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.5", 3181); final BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.6", 3181); - + final String rackName1 = NetworkTopology.DEFAULT_REGION + "/r1"; final String rackName2 = NetworkTopology.DEFAULT_REGION + "/r2"; - + // update dns mapping StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), rackName1); StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), rackName1); @@ -2631,7 +2631,7 @@ public void testReplaceToAdherePlacementPolicyWithOutOfOrder() throws Exception StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), rackName2); StaticDNSResolver.addNodeToRack(addr5.getSocketAddress().getAddress().getHostAddress(), rackName2); StaticDNSResolver.addNodeToRack(addr6.getSocketAddress().getAddress().getHostAddress(), rackName2); - + // Update cluster final Set addrs = new HashSet<>(); addrs.add(addr1.toBookieId()); @@ -2640,16 +2640,16 @@ public void testReplaceToAdherePlacementPolicyWithOutOfOrder() throws Exception addrs.add(addr4.toBookieId()); addrs.add(addr5.toBookieId()); addrs.add(addr6.toBookieId()); - + final ClientConfiguration newConf = new ClientConfiguration(conf); newConf.setDiskWeightBasedPlacementEnabled(false); newConf.setMinNumRacksPerWriteQuorum(2); newConf.setEnforceMinNumRacksPerWriteQuorum(true); - + repp.initialize(newConf, Optional.empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); - + repp.onClusterChanged(addrs, new HashSet<>()); final Map bookieInfoMap = new HashMap<>(); bookieInfoMap.put(addr1.toBookieId(), new BookieInfo(100L, 100L)); @@ -2658,14 +2658,14 @@ public void testReplaceToAdherePlacementPolicyWithOutOfOrder() throws Exception bookieInfoMap.put(addr4.toBookieId(), new BookieInfo(100L, 100L)); bookieInfoMap.put(addr5.toBookieId(), new BookieInfo(100L, 100L)); bookieInfoMap.put(addr6.toBookieId(), new BookieInfo(100L, 100L)); - + repp.updateBookieInfo(bookieInfoMap); - + final Set excludeList = new HashSet<>(); final int ensembleSize = 6; final int writeQuorumSize = 2; final int ackQuorumSize = 2; - + final Consumer, Matcher>>> test = (pair) -> { // RackawareEnsemblePlacementPolicyImpl#isEnsembleAdheringToPlacementPolicy // is not scope of this test case. So, use the method in assertion for convenience. @@ -2679,7 +2679,7 @@ public void testReplaceToAdherePlacementPolicyWithOutOfOrder() throws Exception } assertEquals(PlacementPolicyAdherence.MEETS_STRICT, result.getAdheringToPolicy()); }; - + for (int i = 0; i < 1000; i++) { //All bookies already in the ensemble, the bookie order not adhere the placement policy. test.accept(Pair.of(Arrays.asList(addr1.toBookieId(), addr2.toBookieId(), addr3.toBookieId(), @@ -2693,7 +2693,7 @@ public void testReplaceToAdherePlacementPolicyWithOutOfOrder() throws Exception null)); } } - + @SuppressWarnings("unchecked") @Test public void testReplaceToAdherePlacementPolicyWithNoMoreRackBookie() throws Exception { @@ -2703,10 +2703,10 @@ public void testReplaceToAdherePlacementPolicyWithNoMoreRackBookie() throws Exce final BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); final BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.5", 3181); final BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.6", 3181); - + final String rackName1 = NetworkTopology.DEFAULT_REGION + "/r1"; final String rackName2 = NetworkTopology.DEFAULT_REGION + "/r2"; - + // update dns mapping StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), rackName1); StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), rackName1); @@ -2714,7 +2714,7 @@ public void testReplaceToAdherePlacementPolicyWithNoMoreRackBookie() throws Exce StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), rackName2); StaticDNSResolver.addNodeToRack(addr5.getSocketAddress().getAddress().getHostAddress(), rackName2); StaticDNSResolver.addNodeToRack(addr6.getSocketAddress().getAddress().getHostAddress(), rackName2); - + // Update cluster final Set addrs = new HashSet<>(); addrs.add(addr1.toBookieId()); @@ -2723,16 +2723,16 @@ public void testReplaceToAdherePlacementPolicyWithNoMoreRackBookie() throws Exce addrs.add(addr4.toBookieId()); addrs.add(addr5.toBookieId()); addrs.add(addr6.toBookieId()); - + final ClientConfiguration newConf = new ClientConfiguration(conf); newConf.setDiskWeightBasedPlacementEnabled(false); newConf.setMinNumRacksPerWriteQuorum(2); newConf.setEnforceMinNumRacksPerWriteQuorum(true); - + repp.initialize(newConf, Optional.empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); - + repp.onClusterChanged(addrs, new HashSet<>()); final Map bookieInfoMap = new HashMap<>(); bookieInfoMap.put(addr1.toBookieId(), new BookieInfo(100L, 100L)); @@ -2741,14 +2741,14 @@ public void testReplaceToAdherePlacementPolicyWithNoMoreRackBookie() throws Exce bookieInfoMap.put(addr4.toBookieId(), new BookieInfo(100L, 100L)); bookieInfoMap.put(addr5.toBookieId(), new BookieInfo(100L, 100L)); bookieInfoMap.put(addr6.toBookieId(), new BookieInfo(100L, 100L)); - + repp.updateBookieInfo(bookieInfoMap); - + final Set excludeList = new HashSet<>(); final int ensembleSize = 3; final int writeQuorumSize = 2; final int ackQuorumSize = 2; - + final Consumer, Matcher>>> test = (pair) -> { // RackawareEnsemblePlacementPolicyImpl#isEnsembleAdheringToPlacementPolicy // is not scope of this test case. So, use the method in assertion for convenience. @@ -2763,13 +2763,13 @@ public void testReplaceToAdherePlacementPolicyWithNoMoreRackBookie() throws Exce assertEquals(PlacementPolicyAdherence.FAIL, result.getAdheringToPolicy()); assertEquals(0, result.getResult().size()); }; - + for (int i = 0; i < 1000; i++) { test.accept(Pair.of(Arrays.asList(addr1.toBookieId(), addr2.toBookieId(), addr4.toBookieId()), null)); } } - + @SuppressWarnings("unchecked") @Test public void testReplaceToAdherePlacementPolicyWithUnknowBookie() throws Exception { @@ -2779,10 +2779,10 @@ public void testReplaceToAdherePlacementPolicyWithUnknowBookie() throws Exceptio final BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); final BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.5", 3181); final BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.6", 3181); - + final String rackName1 = NetworkTopology.DEFAULT_REGION + "/r1"; final String rackName2 = NetworkTopology.DEFAULT_REGION + "/r2"; - + // update dns mapping StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), rackName1); StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), rackName1); @@ -2790,7 +2790,7 @@ public void testReplaceToAdherePlacementPolicyWithUnknowBookie() throws Exceptio StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), rackName2); StaticDNSResolver.addNodeToRack(addr5.getSocketAddress().getAddress().getHostAddress(), rackName2); StaticDNSResolver.addNodeToRack(addr6.getSocketAddress().getAddress().getHostAddress(), rackName2); - + // Update cluster final Set addrs = new HashSet<>(); addrs.add(addr1.toBookieId()); @@ -2799,16 +2799,16 @@ public void testReplaceToAdherePlacementPolicyWithUnknowBookie() throws Exceptio addrs.add(addr4.toBookieId()); addrs.add(addr5.toBookieId()); addrs.add(addr6.toBookieId()); - + final ClientConfiguration newConf = new ClientConfiguration(conf); newConf.setDiskWeightBasedPlacementEnabled(false); newConf.setMinNumRacksPerWriteQuorum(2); newConf.setEnforceMinNumRacksPerWriteQuorum(true); - + repp.initialize(newConf, Optional.empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); - + repp.onClusterChanged(addrs, new HashSet<>()); final Map bookieInfoMap = new HashMap<>(); bookieInfoMap.put(addr1.toBookieId(), new BookieInfo(100L, 100L)); @@ -2817,16 +2817,16 @@ public void testReplaceToAdherePlacementPolicyWithUnknowBookie() throws Exceptio bookieInfoMap.put(addr4.toBookieId(), new BookieInfo(100L, 100L)); bookieInfoMap.put(addr5.toBookieId(), new BookieInfo(100L, 100L)); bookieInfoMap.put(addr6.toBookieId(), new BookieInfo(100L, 100L)); - + repp.updateBookieInfo(bookieInfoMap); - + final Set excludeList = new HashSet<>(); final int ensembleSize = 6; final int writeQuorumSize = 2; final int ackQuorumSize = 2; - + final BookieRackMatcher rack1 = new BookieRackMatcher(rackName1); - + final Consumer, Matcher>>> test = (pair) -> { // RackawareEnsemblePlacementPolicyImpl#isEnsembleAdheringToPlacementPolicy // is not scope of this test case. So, use the method in assertion for convenience. @@ -2841,17 +2841,16 @@ public void testReplaceToAdherePlacementPolicyWithUnknowBookie() throws Exceptio assertEquals(PlacementPolicyAdherence.MEETS_STRICT, result.getAdheringToPolicy()); assertThat(result.getResult(), pair.getRight()); }; - + for (int i = 0; i < 1000; i++) { test.accept(Pair.of(Arrays.asList(BookieId.parse("127.0.0.10:3181"), BookieId.parse("127.0.0.11:3181"), addr3.toBookieId(), addr4.toBookieId(), addr5.toBookieId(), addr6.toBookieId()), - // first, same, same, same, same, same, condition[0] contains(is(rack1), is(addr5.toBookieId()), is(addr3.toBookieId()), is(addr4.toBookieId()), is(rack1), is(addr6.toBookieId())))); } } - + private static class BookieRackMatcher extends TypeSafeMatcher { final List expectedRacks; From a2de1350891e9e1d3d26c4f9083b5051b48c1ece Mon Sep 17 00:00:00 2001 From: horizonzy Date: Sun, 7 Aug 2022 10:33:27 +0800 Subject: [PATCH 30/35] fix checkstyle. --- .../client/RackawareEnsemblePlacementPolicyImpl.java | 2 +- .../client/TestRackawareEnsemblePlacementPolicy.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 4cd08ed0795..9a9864156ad 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -1267,7 +1267,7 @@ private BookieNode replaceToAdherePlacementPolicyInternal( for (Pair> condition : conditionList) { WeightedRandomSelection wRSelection = null; - + final List leaves = new ArrayList<>(topology.getLeaves(condition.getLeft())); if (!isWeighted) { Collections.shuffle(leaves); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index ac3974ffe5d..06f5e30d5e5 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -1472,7 +1472,7 @@ public boolean validate() { // this is expected } } - + @Test public void testNewEnsembleWithMultipleRacks() throws Exception { BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); @@ -2853,16 +2853,16 @@ public void testReplaceToAdherePlacementPolicyWithUnknowBookie() throws Exceptio private static class BookieRackMatcher extends TypeSafeMatcher { final List expectedRacks; - + public BookieRackMatcher(String... expectedRacks) { this.expectedRacks = Arrays.asList(expectedRacks); } - + @Override protected boolean matchesSafely(BookieId bookieId) { return expectedRacks.contains(StaticDNSResolver.getRack(bookieId.toString().split(":")[0])); } - + @Override public void describeTo(Description description) { description.appendText("expected racks " + expectedRacks); From 465f986f348b82e0f8fe16411f360c687dce8a10 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Mon, 8 Aug 2022 00:47:13 +0800 Subject: [PATCH 31/35] address comment. --- .../bookkeeper/client/BookKeeperAdmin.java | 29 +++++++++++-------- .../RackawareEnsemblePlacementPolicyImpl.java | 16 ---------- .../TopologyAwareEnsemblePlacementPolicy.java | 17 +++++++++++ 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java index 52357b2346e..2e95ab5a4cb 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java @@ -1791,20 +1791,25 @@ public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List replaceNotAdheringPlacementPolicyBookie(List ensembleBookiesList, int writeQuorumSize, int ackQuorumSize) { - EnsemblePlacementPolicy.PlacementResult> placementResult = bkc.getPlacementPolicy() - .replaceToAdherePlacementPolicy(ensembleBookiesList.size(), writeQuorumSize, ackQuorumSize, - new HashSet<>(), ensembleBookiesList); - if (PlacementPolicyAdherence.FAIL != placementResult.getAdheringToPolicy()) { - Map targetMap = new HashMap<>(); - List newEnsembles = placementResult.getResult(); - for (int i = 0; i < ensembleBookiesList.size(); i++) { - BookieId originBookie = ensembleBookiesList.get(i); - BookieId newBookie = newEnsembles.get(i); - if (!originBookie.equals(newBookie)) { - targetMap.put(i, newBookie); + try { + EnsemblePlacementPolicy.PlacementResult> placementResult = bkc.getPlacementPolicy() + .replaceToAdherePlacementPolicy(ensembleBookiesList.size(), writeQuorumSize, ackQuorumSize, + new HashSet<>(), ensembleBookiesList); + if (PlacementPolicyAdherence.FAIL != placementResult.getAdheringToPolicy()) { + Map targetMap = new HashMap<>(); + List newEnsembles = placementResult.getResult(); + for (int i = 0; i < ensembleBookiesList.size(); i++) { + BookieId originBookie = ensembleBookiesList.get(i); + BookieId newBookie = newEnsembles.get(i); + if (!originBookie.equals(newBookie)) { + targetMap.put(i, newBookie); + } } + return targetMap; } - return targetMap; + } catch (UnsupportedOperationException e) { + LOG.warn("The placement policy: {} didn't support replaceToAdherePlacementPolicy, " + + "ignore replace not adhere bookie.", bkc.getPlacementPolicy().getClass().getName()); } return Collections.emptyMap(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 9a9864156ad..d71bba16d1f 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -1127,22 +1127,6 @@ public PlacementResult> replaceToAdherePlacementPolicy( } } - private int differBetweenBookies(List bookiesA, List bookiesB) { - if (CollectionUtils.isEmpty(bookiesA) || CollectionUtils.isEmpty(bookiesB)) { - return Integer.MAX_VALUE; - } - if (bookiesA.size() != bookiesB.size()) { - return Integer.MAX_VALUE; - } - int differ = 0; - for (int i = 0; i < bookiesA.size(); i++) { - if (!bookiesA.get(i).equals(bookiesB.get(i))) { - differ++; - } - } - return differ; - } - private PlacementResult> doReplaceToAdherePlacementPolicy( int ensembleSize, int writeQuorumSize, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index 58a971e682e..852b7f3e35d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -52,6 +52,7 @@ import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.OpStatsLogger; import org.apache.bookkeeper.stats.annotations.StatsDoc; +import org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -763,6 +764,22 @@ public void onBookieRackChange(List bookieAddressList) { } } + public static int differBetweenBookies(List bookiesA, List bookiesB) { + if (CollectionUtils.isEmpty(bookiesA) || CollectionUtils.isEmpty(bookiesB)) { + return Integer.MAX_VALUE; + } + if (bookiesA.size() != bookiesB.size()) { + return Integer.MAX_VALUE; + } + int differ = 0; + for (int i = 0; i < bookiesA.size(); i++) { + if (!bookiesA.get(i).equals(bookiesB.get(i))) { + differ++; + } + } + return differ; + } + @Override public void updateBookieInfo(Map bookieInfoMap) { if (!isWeighted) { From 965bf96c56c42a5392b022352981c8198dc4656b Mon Sep 17 00:00:00 2001 From: horizonzy Date: Mon, 8 Aug 2022 09:39:48 +0800 Subject: [PATCH 32/35] address comment. --- .../apache/bookkeeper/client/EnsemblePlacementPolicy.java | 8 ++++++++ .../org/apache/bookkeeper/conf/ServerConfiguration.java | 4 +++- .../java/org/apache/bookkeeper/replication/Auditor.java | 2 +- .../apache/bookkeeper/replication/ReplicationWorker.java | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index 12e56d717cf..7a2933c7754 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -506,6 +506,14 @@ public T getResult() { return result; } + /** + * Use {@link #getAdheringToPolicy} + */ + @Deprecated + public PlacementPolicyAdherence isAdheringToPolicy() { + return policyAdherence; + } + public PlacementPolicyAdherence getAdheringToPolicy() { return policyAdherence; } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index a4a212df3c2..1e441609660 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -2622,6 +2622,8 @@ public void setRepairedPlacementPolicyNotAdheringBookieEnable(boolean enabled) { } /** + * Now the feature only support RackawareEnsemblePlacementPolicy. + * * In Auditor, it combines with {@link #getAuditorPeriodicPlacementPolicyCheckInterval}, to control is marked * ledger id to under replication managed when found a ledger ensemble not adhere to placement policy. * In ReplicationWorker, to control is to repair the ledger which the ensemble does not adhere to the placement @@ -2632,7 +2634,7 @@ public void setRepairedPlacementPolicyNotAdheringBookieEnable(boolean enabled) { * server. You should set a suitable rereplicationEntryBatchSize to avoid bookie server pressure. * */ - public boolean getRepairedPlacementPolicyNotAdheringBookieEnable() { + public boolean isRepairedPlacementPolicyNotAdheringBookieEnable() { return getBoolean(REPAIRED_PLACEMENT_POLICY_NOT_ADHERING_BOOKIE_ENABLED, false); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java index a3cacf22d40..243203da0ee 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java @@ -1471,7 +1471,7 @@ public void process(Long ledgerId, AsyncCallback.VoidCallback iterCallback) { if (foundSegmentNotAdheringToPlacementPolicy) { numOfLedgersFoundNotAdheringInPlacementPolicyCheck.incrementAndGet(); //If user enable repaired, mark this ledger to under replication manager. - if (conf.getRepairedPlacementPolicyNotAdheringBookieEnable()) { + if (conf.isRepairedPlacementPolicyNotAdheringBookieEnable()) { ledgerUnderreplicationManager.markLedgerUnderreplicatedAsync(ledgerId, Collections.emptyList()).whenComplete((res, e) -> { if (e != null) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java index 9214a8445e3..e15f5c1407e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java @@ -369,7 +369,7 @@ private boolean tryReadingFaultyEntries(LedgerHandle lh, LedgerFragment ledgerFr } private Set getNeedRepairedPlacementNotAdheringFragments(LedgerHandle lh) { - if (!conf.getRepairedPlacementPolicyNotAdheringBookieEnable()) { + if (!conf.isRepairedPlacementPolicyNotAdheringBookieEnable()) { return Collections.emptySet(); } long ledgerId = lh.getId(); From edefdeb82f816c3910bbc340c1264fc644d3f3a5 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Mon, 8 Aug 2022 09:48:56 +0800 Subject: [PATCH 33/35] fix check style. --- .../org/apache/bookkeeper/client/EnsemblePlacementPolicy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java index 7a2933c7754..05a687b964d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/EnsemblePlacementPolicy.java @@ -507,7 +507,7 @@ public T getResult() { } /** - * Use {@link #getAdheringToPolicy} + * Use {@link #getAdheringToPolicy}. */ @Deprecated public PlacementPolicyAdherence isAdheringToPolicy() { From 6ef1e2aa8ac0780bd8199b360e840506cbc85e5d Mon Sep 17 00:00:00 2001 From: horizonzy Date: Mon, 8 Aug 2022 12:11:38 +0800 Subject: [PATCH 34/35] address comment. --- .../RackawareEnsemblePlacementPolicyImpl.java | 13 +++++++------ .../org/apache/bookkeeper/net/NetworkTopology.java | 1 - 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index d71bba16d1f..d3d557d14be 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -1103,6 +1103,13 @@ public PlacementResult> replaceToAdherePlacementPolicy( excludeBookies.add(bookieId); } } + int minNumRacksPerWriteQuorumForThisEnsemble = Math.min(writeQuorumSize, minNumRacksPerWriteQuorum); + int numRacks = topology.getNumOfRacks(); + // only one rack or less than minNumRacksPerWriteQuorumForThisEnsemble, stop calculation to skip relocation + if (numRacks < 2 || numRacks < minNumRacksPerWriteQuorumForThisEnsemble) { + LOG.warn("Skip ensemble relocation because the cluster has only {} rack.", numRacks); + return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); + } PlacementResult> placementResult = PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); int minDiffer = Integer.MAX_VALUE; @@ -1148,12 +1155,6 @@ private PlacementResult> doReplaceToAdherePlacementPolicy( null, null, minNumRacksPerWriteQuorumForThisEnsemble); - int numRacks = topology.getNumOfRacks(); - // only one rack or less than minNumRacksPerWriteQuorumForThisEnsemble, stop calculation to skip relocation - if (numRacks < 2 || numRacks < minNumRacksPerWriteQuorumForThisEnsemble) { - LOG.warn("Skip ensemble relocation because the cluster has only {} rack.", numRacks); - return PlacementResult.of(Collections.emptyList(), PlacementPolicyAdherence.FAIL); - } BookieNode prevNode = null; final BookieNode firstNode = provisionalEnsembleNodes.get(startIndex); // use same bookie at first to reduce ledger replication diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopology.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopology.java index e599de9938c..70a1032ce83 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopology.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopology.java @@ -29,7 +29,6 @@ public interface NetworkTopology { String DEFAULT_RACK = "/default-rack"; String DEFAULT_ZONE = "/default-zone"; String DEFAULT_UPGRADEDOMAIN = "/default-upgradedomain"; - String INACTIVE = "/inactive"; String DEFAULT_ZONE_AND_UPGRADEDOMAIN = DEFAULT_ZONE + DEFAULT_UPGRADEDOMAIN; String DEFAULT_REGION_AND_RACK = DEFAULT_REGION + DEFAULT_RACK; From 85bd141d5db3ea2e6b5e5b3ee0d99b73d776375d Mon Sep 17 00:00:00 2001 From: horizonzy Date: Thu, 11 Aug 2022 15:39:08 +0800 Subject: [PATCH 35/35] reset StaticDNSResolver after test. --- .../client/TestRackawareEnsemblePlacementPolicy.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index 06f5e30d5e5..396e86d92bb 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -2609,6 +2609,7 @@ public void testReplaceToAdherePlacementPolicy() throws Exception { contains(is(addr1.toBookieId()), is(rack23), is(rack123), is(rack123), is(rack123), is(rack123), is(rack23)))); } + StaticDNSResolver.reset(); } @SuppressWarnings("unchecked") @@ -2692,6 +2693,7 @@ public void testReplaceToAdherePlacementPolicyWithOutOfOrder() throws Exception // the final result is 1,6,3,4,2,5. If we pick 5 for the second, the final result is 1,5,3,4,2,6 null)); } + StaticDNSResolver.reset(); } @SuppressWarnings("unchecked") @@ -2768,6 +2770,7 @@ public void testReplaceToAdherePlacementPolicyWithNoMoreRackBookie() throws Exce test.accept(Pair.of(Arrays.asList(addr1.toBookieId(), addr2.toBookieId(), addr4.toBookieId()), null)); } + StaticDNSResolver.reset(); } @SuppressWarnings("unchecked") @@ -2849,6 +2852,7 @@ public void testReplaceToAdherePlacementPolicyWithUnknowBookie() throws Exceptio contains(is(rack1), is(addr5.toBookieId()), is(addr3.toBookieId()), is(addr4.toBookieId()), is(rack1), is(addr6.toBookieId())))); } + StaticDNSResolver.reset(); } private static class BookieRackMatcher extends TypeSafeMatcher {