Skip to content

Add small files check in garbage collection #3631

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hangc0276 merged 4 commits into from
Mar 16, 2023
Merged

Add small files check in garbage collection #3631

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
46f64ff
add gc small files check
hangc0276 Nov 10, 2022
e8f21c1
format code
hangc0276 Nov 10, 2022
d6a2ccc
address comments
hangc0276 Mar 13, 2023
c48bf69
format code
hangc0276 Mar 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,15 +561,20 @@ void doCompactEntryLogs(double threshold, long maxTimeMillis) throws EntryLogMet
MutableLong timeDiff = new MutableLong(0);

entryLogMetaMap.forEach((entryLogId, meta) -> {
int bucketIndex = calculateUsageIndex(numBuckets, meta.getUsage());
double usage = meta.getUsage();
if (conf.isUseTargetEntryLogSizeForGc() && usage < 1.0d) {
usage = (double) meta.getRemainingSize() / Math.max(meta.getTotalSize(), conf.getEntryLogSizeLimit());
}
int bucketIndex = calculateUsageIndex(numBuckets, usage);
entryLogUsageBuckets[bucketIndex]++;

if (timeDiff.getValue() < maxTimeMillis) {
end.setValue(System.currentTimeMillis());
timeDiff.setValue(end.getValue() - start);
}
if (meta.getUsage() >= threshold || (maxTimeMillis > 0 && timeDiff.getValue() >= maxTimeMillis)
|| !running) {
if ((usage >= threshold
|| (maxTimeMillis > 0 && timeDiff.getValue() >= maxTimeMillis)
|| !running)) {
// We allow the usage limit calculation to continue so that we get an accurate
// report of where the usage was prior to running compaction.
return;
Expand Down
10 changes: 10 additions & 0 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ public class ServerConfiguration extends AbstractConfiguration<ServerConfigurati
protected static final String VERIFY_METADATA_ON_GC = "verifyMetadataOnGC";
protected static final String GC_ENTRYLOGMETADATA_CACHE_ENABLED = "gcEntryLogMetadataCacheEnabled";
protected static final String GC_ENTRYLOG_METADATA_CACHE_PATH = "gcEntryLogMetadataCachePath";
protected static final String USE_TARGET_ENTRYLOG_SIZE_FOR_GC = "useTargetEntryLogSizeForGc";
// Scrub Parameters
protected static final String LOCAL_SCRUB_PERIOD = "localScrubInterval";
protected static final String LOCAL_SCRUB_RATE_LIMIT = "localScrubRateLimit";
Expand Down Expand Up @@ -554,6 +555,15 @@ public ServerConfiguration setGcEntryLogMetadataCachePath(String gcEntrylogMetad
return this;
}

public boolean isUseTargetEntryLogSizeForGc() {
return getBoolean(USE_TARGET_ENTRYLOG_SIZE_FOR_GC, false);
}

public ServerConfiguration setUseTargetEntryLogSizeForGc(boolean useTargetEntryLogSizeForGc) {
this.setProperty(USE_TARGET_ENTRYLOG_SIZE_FOR_GC, useTargetEntryLogSizeForGc);
return this;
}

/**
* Get whether local scrub is enabled.
*
Expand Down
135 changes: 135 additions & 0 deletions bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/GarbageCollectorThreadTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.any;
Expand Down Expand Up @@ -238,4 +239,138 @@ private void testExtractMetaFromEntryLogs(EntryLogger entryLogger, File ledgerDi
assertTrue(entryLogMetaMap.isEmpty());
assertFalse(entryLogger.logExists(logId3));
}

@Test
public void testCompactionWithFileSizeCheck() throws Exception {
File ledgerDir = tmpDirs.createNew("testFileSize", "ledgers");
EntryLogger entryLogger = newLegacyEntryLogger(20000, ledgerDir);

MockLedgerStorage storage = new MockLedgerStorage();
MockLedgerManager lm = new MockLedgerManager();

GarbageCollectorThread gcThread = new GarbageCollectorThread(
TestBKConfiguration.newServerConfiguration().setUseTargetEntryLogSizeForGc(true), lm,
newDirsManager(ledgerDir),
storage, entryLogger, NullStatsLogger.INSTANCE);

// Add entries.
// Ledger 1 is on first entry log
// Ledger 2 spans first, second and third entry log
// Ledger 3 is on the third entry log (which is still active when extract meta)
long loc1 = entryLogger.addEntry(1L, makeEntry(1L, 1L, 5000));
long loc2 = entryLogger.addEntry(2L, makeEntry(2L, 1L, 5000));
assertThat(logIdFromLocation(loc2), equalTo(logIdFromLocation(loc1)));
long loc3 = entryLogger.addEntry(2L, makeEntry(2L, 2L, 15000));
assertThat(logIdFromLocation(loc3), greaterThan(logIdFromLocation(loc2)));
long loc4 = entryLogger.addEntry(2L, makeEntry(2L, 3L, 15000));
assertThat(logIdFromLocation(loc4), greaterThan(logIdFromLocation(loc3)));
long loc5 = entryLogger.addEntry(3L, makeEntry(3L, 1L, 1000));
assertThat(logIdFromLocation(loc5), equalTo(logIdFromLocation(loc4)));
long loc6 = entryLogger.addEntry(3L, makeEntry(3L, 2L, 5000));

long logId1 = logIdFromLocation(loc2);
long logId2 = logIdFromLocation(loc3);
long logId3 = logIdFromLocation(loc5);
long logId4 = logIdFromLocation(loc6);
entryLogger.flush();

storage.setMasterKey(1L, new byte[0]);
storage.setMasterKey(2L, new byte[0]);
storage.setMasterKey(3L, new byte[0]);

assertThat(entryLogger.getFlushedLogIds(), containsInAnyOrder(logId1, logId2, logId3));
assertTrue(entryLogger.logExists(logId1));
assertTrue(entryLogger.logExists(logId2));
assertTrue(entryLogger.logExists(logId3));
assertTrue(entryLogger.logExists(logId4));

// all ledgers exist, nothing should disappear
final EntryLogMetadataMap entryLogMetaMap = gcThread.getEntryLogMetaMap();
gcThread.extractMetaFromEntryLogs();

assertThat(entryLogger.getFlushedLogIds(), containsInAnyOrder(logId1, logId2, logId3));
assertTrue(entryLogMetaMap.containsKey(logId1));
assertTrue(entryLogMetaMap.containsKey(logId2));
assertTrue(entryLogger.logExists(logId3));

storage.deleteLedger(1);
// only logId 1 will be compacted.
gcThread.runWithFlags(true, true, false);

// logId1 and logId2 should be compacted
assertFalse(entryLogger.logExists(logId1));
assertTrue(entryLogger.logExists(logId2));
assertTrue(entryLogger.logExists(logId3));
assertFalse(entryLogMetaMap.containsKey(logId1));
assertTrue(entryLogMetaMap.containsKey(logId2));

assertEquals(1, storage.getUpdatedLocations().size());

EntryLocation location2 = storage.getUpdatedLocations().get(0);
assertEquals(2, location2.getLedger());
assertEquals(1, location2.getEntry());
assertEquals(logIdFromLocation(location2.getLocation()), logId4);
}

@Test
public void testCompactionWithoutFileSizeCheck() throws Exception {
File ledgerDir = tmpDirs.createNew("testFileSize", "ledgers");
EntryLogger entryLogger = newLegacyEntryLogger(20000, ledgerDir);

MockLedgerStorage storage = new MockLedgerStorage();
MockLedgerManager lm = new MockLedgerManager();

GarbageCollectorThread gcThread = new GarbageCollectorThread(
TestBKConfiguration.newServerConfiguration(), lm,
newDirsManager(ledgerDir),
storage, entryLogger, NullStatsLogger.INSTANCE);

// Add entries.
// Ledger 1 is on first entry log
// Ledger 2 spans first, second and third entry log
// Ledger 3 is on the third entry log (which is still active when extract meta)
long loc1 = entryLogger.addEntry(1L, makeEntry(1L, 1L, 5000));
long loc2 = entryLogger.addEntry(2L, makeEntry(2L, 1L, 5000));
assertThat(logIdFromLocation(loc2), equalTo(logIdFromLocation(loc1)));
long loc3 = entryLogger.addEntry(2L, makeEntry(2L, 2L, 15000));
assertThat(logIdFromLocation(loc3), greaterThan(logIdFromLocation(loc2)));
long loc4 = entryLogger.addEntry(2L, makeEntry(2L, 3L, 15000));
assertThat(logIdFromLocation(loc4), greaterThan(logIdFromLocation(loc3)));
long loc5 = entryLogger.addEntry(3L, makeEntry(3L, 1L, 1000));
assertThat(logIdFromLocation(loc5), equalTo(logIdFromLocation(loc4)));

long logId1 = logIdFromLocation(loc2);
long logId2 = logIdFromLocation(loc3);
long logId3 = logIdFromLocation(loc5);
entryLogger.flush();

storage.setMasterKey(1L, new byte[0]);
storage.setMasterKey(2L, new byte[0]);
storage.setMasterKey(3L, new byte[0]);

assertThat(entryLogger.getFlushedLogIds(), containsInAnyOrder(logId1, logId2));
assertTrue(entryLogger.logExists(logId1));
assertTrue(entryLogger.logExists(logId2));
assertTrue(entryLogger.logExists(logId3));

// all ledgers exist, nothing should disappear
final EntryLogMetadataMap entryLogMetaMap = gcThread.getEntryLogMetaMap();
gcThread.extractMetaFromEntryLogs();

assertThat(entryLogger.getFlushedLogIds(), containsInAnyOrder(logId1, logId2));
assertTrue(entryLogMetaMap.containsKey(logId1));
assertTrue(entryLogMetaMap.containsKey(logId2));
assertTrue(entryLogger.logExists(logId3));

gcThread.runWithFlags(true, true, false);

assertTrue(entryLogger.logExists(logId1));
assertTrue(entryLogger.logExists(logId2));
assertTrue(entryLogger.logExists(logId3));
assertTrue(entryLogMetaMap.containsKey(logId1));
assertTrue(entryLogMetaMap.containsKey(logId2));

assertEquals(0, storage.getUpdatedLocations().size());
}

}
9 changes: 9 additions & 0 deletions conf/bk_server.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -621,6 +621,15 @@ gcEntryLogMetadataCacheEnabled=false
# name "entrylogIndexCache"]
# gcEntryLogMetadataCachePath=

# When judging whether an entry log file need to be compacted, we calculate the usage rate of the entry log file based
# on the actual size of the entry log file. However, if an entry log file is 1MB in size and 0.9MB of data is
# being used, this entry log file won't be compacted by garbage collector due to the high usage ratio,
# which will result in many small entry log files.
# We introduced the parameter `useTargetEntryLogSizeForGc` to determine whether to calculate entry log file usage
# based on the configured target entry log file size, which is configured by `logSizeLimit`.
# Default: useTargetEntryLogSizeForGc is false.
# useTargetEntryLogSizeForGc=false

#############################################################################
## Disk utilization
#############################################################################
Expand Down