From ba0448069fce8482945e9cbbec1b9d479f55b08c Mon Sep 17 00:00:00 2001 From: dao-jun Date: Mon, 10 Jun 2024 12:46:20 +0800 Subject: [PATCH 01/22] Fix code --- pulsar-common/pom.xml | 5 + .../ConcurrentOpenLongPairRangeSet.java | 89 ++++--- .../collections/ConcurrentRoaringBitmap.java | 240 ++++++++++++++++++ 3 files changed, 289 insertions(+), 45 deletions(-) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index cdc30dac2897d..2d85480492e70 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -248,6 +248,11 @@ awaitility test + + + org.roaringbitmap + RoaringBitmap + diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java index 72215d7296cc3..9181bbe93e517 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java @@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang.mutable.MutableInt; +import org.roaringbitmap.RoaringBitmap; /** * A Concurrent set comprising zero or more ranges of type {@link LongPair}. This can be alternative of @@ -43,7 +44,7 @@ */ public class ConcurrentOpenLongPairRangeSet> implements LongPairRangeSet { - protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); + protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); private boolean threadSafe = true; private final int bitSetSize; private final LongPairConsumer consumer; @@ -82,29 +83,26 @@ public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, lon if (lowerKey != upperKey) { // (1) set lower to last in lowerRange.getKey() if (isValid(lowerKey, lowerValue)) { - BitSet rangeBitSet = rangeBitSetMap.get(lowerKey); + RoaringBitmap rangeBitSet = rangeBitSetMap.get(lowerKey); // if lower and upper has different key/ledger then set ranges for lower-key only if // a. bitSet already exist and given value is not the last value in the bitset. // it will prevent setting up values which are not actually expected to set // eg: (2:10..4:10] in this case, don't set any value for 2:10 and set [4:0..4:10] - if (rangeBitSet != null && (rangeBitSet.previousSetBit(rangeBitSet.size()) > lowerValueOpen)) { - int lastValue = rangeBitSet.previousSetBit(rangeBitSet.size()); - rangeBitSet.set((int) lowerValue, (int) Math.max(lastValue, lowerValue) + 1); + if (rangeBitSet != null && (rangeBitSet.last() > lowerValueOpen)) { + int lastValue = (int) rangeBitSet.previousValue(rangeBitSet.last()); + rangeBitSet.add((int) lowerValue, (int) Math.max(lastValue, lowerValue) + 1); } } // (2) set 0th-index to upper-index in upperRange.getKey() if (isValid(upperKey, upperValue)) { - BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); - if (rangeBitSet != null) { - rangeBitSet.set(0, (int) upperValue + 1); - } + RoaringBitmap rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); + rangeBitSet.add(0, upperValue + 1); } // No-op if values are not valid eg: if lower == LongPair.earliest or upper == LongPair.latest then nothing // to set } else { - long key = lowerKey; - BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(key, (k) -> createNewBitSet()); - rangeBitSet.set((int) lowerValue, (int) upperValue + 1); + RoaringBitmap rangeBitSet = rangeBitSetMap.computeIfAbsent(lowerKey, (k) -> createNewBitSet()); + rangeBitSet.add(lowerValue, upperValue + 1); } updatedAfterCachedForSize = true; updatedAfterCachedForToString = true; @@ -118,25 +116,25 @@ private boolean isValid(long key, long value) { @Override public boolean contains(long key, long value) { - BitSet rangeBitSet = rangeBitSetMap.get(key); + RoaringBitmap rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { - return rangeBitSet.get(getSafeEntry(value)); + return rangeBitSet.contains(getSafeEntry(value)); } return false; } @Override public Range rangeContaining(long key, long value) { - BitSet rangeBitSet = rangeBitSetMap.get(key); + RoaringBitmap rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { - if (!rangeBitSet.get(getSafeEntry(value))) { + if (!rangeBitSet.contains(getSafeEntry(value))) { // if position is not part of any range then return null return null; } - int lowerValue = rangeBitSet.previousClearBit(getSafeEntry(value)) + 1; + int lowerValue = (int) (rangeBitSet.previousAbsentValue(getSafeEntry(value)) + 1); final T lower = consumer.apply(key, lowerValue); final T upper = consumer.apply(key, - Math.max(rangeBitSet.nextClearBit(getSafeEntry(value)) - 1, lowerValue)); + Math.max(rangeBitSet.nextAbsentValue(getSafeEntry(value)) - 1, lowerValue)); return Range.closed(lower, upper); } return null; @@ -152,7 +150,7 @@ public boolean isEmpty() { if (rangeBitSetMap.isEmpty()) { return true; } - for (BitSet rangeBitSet : rangeBitSetMap.values()) { + for (RoaringBitmap rangeBitSet : rangeBitSetMap.values()) { if (!rangeBitSet.isEmpty()) { return false; } @@ -172,10 +170,10 @@ public Range span() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); - Entry lastSet = rangeBitSetMap.lastEntry(); - int first = firstSet.getValue().nextSetBit(0); - int last = lastSet.getValue().previousSetBit(lastSet.getValue().size()); + Entry firstSet = rangeBitSetMap.firstEntry(); + Entry lastSet = rangeBitSetMap.lastEntry(); + int first = (int) firstSet.getValue().nextValue(0); + int last = (int) lastSet.getValue().previousValue(lastSet.getValue().last()); return Range.openClosed(consumer.apply(firstSet.getKey(), first - 1), consumer.apply(lastSet.getKey(), last)); } @@ -215,17 +213,17 @@ public void forEachRawRange(RawRangeProcessor processor) { if (set.isEmpty()) { return; } - int first = set.nextSetBit(0); - int last = set.previousSetBit(set.size()); + int first = (int) set.nextValue(0); + int last = (int) set.previousValue(set.last()); int currentClosedMark = first; while (currentClosedMark != -1 && currentClosedMark <= last) { - int nextOpenMark = set.nextClearBit(currentClosedMark); + int nextOpenMark = (int) set.nextAbsentValue(currentClosedMark); if (!processor.processRawRange(key, currentClosedMark - 1, key, nextOpenMark - 1)) { completed.set(true); break; } - currentClosedMark = set.nextSetBit(nextOpenMark); + currentClosedMark = (int) set.nextValue(nextOpenMark); } }); } @@ -236,9 +234,9 @@ public Range firstRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); - int lower = firstSet.getValue().nextSetBit(0); - int upper = Math.max(lower, firstSet.getValue().nextClearBit(lower) - 1); + Entry firstSet = rangeBitSetMap.firstEntry(); + int lower = (int) firstSet.getValue().nextValue(0); + int upper = (int) Math.max(lower, firstSet.getValue().nextAbsentValue(lower) - 1); return Range.openClosed(consumer.apply(firstSet.getKey(), lower - 1), consumer.apply(firstSet.getKey(), upper)); } @@ -247,30 +245,30 @@ public Range lastRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry lastSet = rangeBitSetMap.lastEntry(); - int upper = lastSet.getValue().previousSetBit(lastSet.getValue().size()); - int lower = Math.min(lastSet.getValue().previousClearBit(upper), upper); + Entry lastSet = rangeBitSetMap.lastEntry(); + int upper = (int) lastSet.getValue().previousValue(lastSet.getValue().last()); + int lower = (int) Math.min(lastSet.getValue().previousAbsentValue(upper), upper); return Range.openClosed(consumer.apply(lastSet.getKey(), lower), consumer.apply(lastSet.getKey(), upper)); } @Override public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { - NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); + NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); MutableInt v = new MutableInt(0); subMap.forEach((key, bitset) -> { if (key == lowerKey || key == upperKey) { - BitSet temp = (BitSet) bitset.clone(); + RoaringBitmap temp = bitset.clone(); // Trim the bitset index which < lowerValue if (key == lowerKey) { - temp.clear(0, (int) Math.max(0, lowerValue)); + temp.remove(0, Math.max(0, lowerValue)); } // Trim the bitset index which > upperValue if (key == upperKey) { - temp.clear((int) Math.min(upperValue + 1, temp.length()), temp.length()); + temp.remove(Math.min(upperValue, temp.last()), temp.last()); } - v.add(temp.cardinality()); + v.add(temp.getCardinality()); } else { - v.add(bitset.cardinality()); + v.add(bitset.getCardinality()); } }); return v.intValue(); @@ -339,7 +337,7 @@ public void add(Range range) { // #addOpenClosed doesn't create bitSet for lower-key because it avoids setting up values for non-exist items // into the key-ledger. so, create bitSet and initialize so, it can't be ignored at #addOpenClosed rangeBitSetMap.computeIfAbsent(lowerEndpoint.getKey(), (key) -> createNewBitSet()) - .set((int) lowerValueOpen + 1); + .add((int) lowerValueOpen + 1); this.addOpenClosed(lowerEndpoint.getKey(), lowerValueOpen, upperEndpoint.getKey(), upperValueClosed); } @@ -382,15 +380,15 @@ public void remove(Range range) { // remove all the keys between two endpoint keys rangeBitSetMap.forEach((key, set) -> { if (lowerEndpoint.getKey() == upperEndpoint.getKey() && key == upperEndpoint.getKey()) { - set.clear((int) lower, (int) upper + 1); + set.remove(lower, upper + 1); } else { // eg: remove-range: [(3,5) - (5,5)] -> Delete all items from 3,6->3,N,4.*,5,0->5,5 if (key == lowerEndpoint.getKey()) { // remove all entries from given position to last position - set.clear((int) lower, set.previousSetBit(set.size())); + set.remove((int) lower, set.previousValue(set.last())); } else if (key == upperEndpoint.getKey()) { // remove all entries from 0 to given position - set.clear(0, (int) upper + 1); + set.remove(0, (int) upper + 1); } else if (key > lowerEndpoint.getKey() && key < upperEndpoint.getKey()) { rangeBitSetMap.remove(key); } @@ -413,8 +411,9 @@ private int getSafeEntry(long value) { return (int) Math.max(value, -1); } - private BitSet createNewBitSet() { - return this.threadSafe ? new ConcurrentBitSet(bitSetSize) : new BitSet(bitSetSize); + private RoaringBitmap createNewBitSet() { + return new RoaringBitmap(); +// return this.threadSafe ? new ConcurrentBitSet(bitSetSize) : new BitSet(bitSetSize); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java new file mode 100644 index 0000000000000..61e1d12255966 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java @@ -0,0 +1,240 @@ +package org.apache.pulsar.common.util.collections; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.concurrent.locks.StampedLock; + +public class ConcurrentRoaringBitmap extends RoaringBitmap { + + private final StampedLock lock = new StampedLock(); + + @Override + public void add(int x) { + long stamp = lock.writeLock(); + try { + super.add(x); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public boolean contains(int x) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + if (super.contains(x)) { + return true; + } + if (!lock.validate(stamp)) { + stamp = lock.readLock(); + try { + return super.contains(x); + } finally { + lock.unlockRead(stamp); + } + } + } else { + stamp = lock.readLock(); + try { + return super.contains(x); + } finally { + lock.unlockRead(stamp); + } + } + return false; + } + + @Override + public void remove(int x) { + long stamp = lock.writeLock(); + try { + super.remove(x); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void clear() { + long stamp = lock.writeLock(); + try { + super.clear(); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public int getSizeInBytes() { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + int size = super.getSizeInBytes(); + if (!lock.validate(stamp)) { + stamp = lock.readLock(); + try { + return super.getSizeInBytes(); + } finally { + lock.unlockRead(stamp); + } + } + return size; + } else { + stamp = lock.readLock(); + try { + return super.getSizeInBytes(); + } finally { + lock.unlockRead(stamp); + } + } + } + + @Override + public ConcurrentRoaringBitmap clone() { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + ConcurrentRoaringBitmap clone = (ConcurrentRoaringBitmap) super.clone(); + if (!lock.validate(stamp)) { + stamp = lock.readLock(); + try { + return (ConcurrentRoaringBitmap) super.clone(); + } finally { + lock.unlockRead(stamp); + } + } + return clone; + } else { + stamp = lock.readLock(); + try { + return (ConcurrentRoaringBitmap) super.clone(); + } finally { + lock.unlockRead(stamp); + } + } + } + + @Override + public void and(RoaringBitmap x2) { + long stamp = lock.writeLock(); + try { + super.and(x2); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void or(RoaringBitmap x2) { + long stamp = lock.writeLock(); + try { + super.or(x2); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void xor(RoaringBitmap x2) { + long stamp = lock.writeLock(); + try { + super.xor(x2); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void andNot(RoaringBitmap x2) { + long stamp = lock.writeLock(); + try { + super.andNot(x2); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void flip(int rangeStart, int rangeEnd) { + long stamp = lock.writeLock(); + try { + super.flip(rangeStart, rangeEnd); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void flip(int x) { + long stamp = lock.writeLock(); + try { + super.flip(x); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void serialize(java.io.DataOutput out) throws java.io.IOException { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + super.serialize(out); + if (!lock.validate(stamp)) { + stamp = lock.readLock(); + try { + super.serialize(out); + } finally { + lock.unlockRead(stamp); + } + } + } else { + stamp = lock.readLock(); + try { + super.serialize(out); + } finally { + lock.unlockRead(stamp); + } + } + } + + @Override + public void deserialize(java.io.DataInput in) throws java.io.IOException { + long stamp = lock.writeLock(); + try { + super.deserialize(in); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void readExternal(java.io.ObjectInput in) throws java.io.IOException { + long stamp = lock.writeLock(); + try { + super.readExternal(in); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void writeExternal(java.io.ObjectOutput out) throws java.io.IOException { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + super.writeExternal(out); + if (!lock.validate(stamp)) { + stamp = lock.readLock(); + try { + super.writeExternal(out); + } finally { + lock.unlockRead(stamp); + } + } + } else { + stamp = lock.readLock(); + try { + super.writeExternal(out); + } finally { + lock.unlockRead(stamp); + } + } + } +} From 9d09de1195e0381197ec063cce219fa361fe40d1 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Sat, 15 Jun 2024 23:26:43 +0800 Subject: [PATCH 02/22] Add ConcurrentRoaringBitmap --- .../ConcurrentOpenLongPairRangeSet.java | 3 +- .../collections/ConcurrentRoaringBitmap.java | 733 +++++++++++++++--- 2 files changed, 634 insertions(+), 102 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java index 9181bbe93e517..8e7da30a4615e 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java @@ -412,8 +412,7 @@ private int getSafeEntry(long value) { } private RoaringBitmap createNewBitSet() { - return new RoaringBitmap(); -// return this.threadSafe ? new ConcurrentBitSet(bitSetSize) : new BitSet(bitSetSize); + return threadSafe ? new ConcurrentRoaringBitmap() : new RoaringBitmap(); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java index 61e1d12255966..1a898e46fd949 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java @@ -1,7 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.common.util.collections; +import org.roaringbitmap.Container; +import org.roaringbitmap.IntConsumer; +import org.roaringbitmap.RelativeRangeConsumer; import org.roaringbitmap.RoaringBitmap; - +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.nio.ByteBuffer; import java.util.concurrent.locks.StampedLock; public class ConcurrentRoaringBitmap extends RoaringBitmap { @@ -18,37 +44,99 @@ public void add(int x) { } } + @Override - public boolean contains(int x) { + public void addN(int[] dat, int offset, int n) { + long stamp = lock.writeLock(); + try { + super.addN(dat, offset, n); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void add(long rangeStart, long rangeEnd) { + long stamp = lock.writeLock(); + try { + super.add(rangeStart, rangeEnd); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void add(int rangeStart, int rangeEnd) { + long stamp = lock.writeLock(); + try { + super.add(rangeStart, rangeEnd); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public boolean intersects(long minimum, long supremum) { long stamp = lock.tryOptimisticRead(); if (stamp != 0) { - if (super.contains(x)) { - return true; - } - if (!lock.validate(stamp)) { - stamp = lock.readLock(); - try { - return super.contains(x); - } finally { - lock.unlockRead(stamp); - } - } - } else { - stamp = lock.readLock(); - try { - return super.contains(x); - } finally { - lock.unlockRead(stamp); + boolean intersects = super.intersects(minimum, supremum); + if (lock.validate(stamp)) { + return intersects; } } - return false; + stamp = lock.readLock(); + try { + return super.intersects(minimum, supremum); + } finally { + lock.unlockRead(stamp); + } + } + + + public void and(final RoaringBitmap x2) { + long stamp = lock.writeLock(); + try { + super.and(x2); + } finally { + lock.unlockWrite(stamp); + } } @Override - public void remove(int x) { + public void andNot(RoaringBitmap x2) { long stamp = lock.writeLock(); try { - super.remove(x); + super.andNot(x2); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void orNot(RoaringBitmap other, long rangeEnd) { + long stamp = lock.writeLock(); + try { + super.orNot(other, rangeEnd); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public boolean checkedAdd(int x) { + long stamp = lock.writeLock(); + try { + return super.checkedAdd(x); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public boolean checkedRemove(int x) { + long stamp = lock.writeLock(); + try { + return super.checkedRemove(x); } finally { lock.unlockWrite(stamp); } @@ -65,95 +153,132 @@ public void clear() { } @Override - public int getSizeInBytes() { + public RoaringBitmap clone() { long stamp = lock.tryOptimisticRead(); if (stamp != 0) { - int size = super.getSizeInBytes(); - if (!lock.validate(stamp)) { - stamp = lock.readLock(); - try { - return super.getSizeInBytes(); - } finally { - lock.unlockRead(stamp); - } - } - return size; - } else { - stamp = lock.readLock(); - try { - return super.getSizeInBytes(); - } finally { - lock.unlockRead(stamp); + RoaringBitmap clone = super.clone(); + if (lock.validate(stamp)) { + return clone; } } + stamp = lock.readLock(); + try { + return super.clone(); + } finally { + lock.unlockRead(stamp); + } } @Override - public ConcurrentRoaringBitmap clone() { + public boolean contains(int x) { long stamp = lock.tryOptimisticRead(); if (stamp != 0) { - ConcurrentRoaringBitmap clone = (ConcurrentRoaringBitmap) super.clone(); - if (!lock.validate(stamp)) { - stamp = lock.readLock(); - try { - return (ConcurrentRoaringBitmap) super.clone(); - } finally { - lock.unlockRead(stamp); - } + boolean contains = super.contains(x); + if (lock.validate(stamp)) { + return contains; } - return clone; - } else { - stamp = lock.readLock(); - try { - return (ConcurrentRoaringBitmap) super.clone(); - } finally { - lock.unlockRead(stamp); + } + stamp = lock.readLock(); + try { + return super.contains(x); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public boolean contains(long minimum, long supremum) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + boolean contains = super.contains(minimum, supremum); + if (lock.validate(stamp)) { + return contains; } } + stamp = lock.readLock(); + try { + return super.contains(minimum, supremum); + } finally { + lock.unlockRead(stamp); + } } @Override - public void and(RoaringBitmap x2) { + public void deserialize(DataInput in, byte[] buffer) throws IOException { long stamp = lock.writeLock(); try { - super.and(x2); + super.deserialize(in, buffer); } finally { lock.unlockWrite(stamp); } } @Override - public void or(RoaringBitmap x2) { + public void deserialize(DataInput in) throws IOException { long stamp = lock.writeLock(); try { - super.or(x2); + super.deserialize(in); } finally { lock.unlockWrite(stamp); } } @Override - public void xor(RoaringBitmap x2) { + public void deserialize(ByteBuffer bbf) throws IOException { long stamp = lock.writeLock(); try { - super.xor(x2); + super.deserialize(bbf); } finally { lock.unlockWrite(stamp); } } @Override - public void andNot(RoaringBitmap x2) { + public boolean equals(Object o) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + boolean equals = super.equals(o); + if (lock.validate(stamp)) { + return equals; + } + } + stamp = lock.readLock(); + try { + return super.equals(o); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public boolean isHammingSimilar(RoaringBitmap other, int tolerance) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + boolean isHammingSimilar = super.isHammingSimilar(other, tolerance); + if (lock.validate(stamp)) { + return isHammingSimilar; + } + } + stamp = lock.readLock(); + try { + return super.isHammingSimilar(other, tolerance); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public void flip(int x) { long stamp = lock.writeLock(); try { - super.andNot(x2); + super.flip(x); } finally { lock.unlockWrite(stamp); } } @Override - public void flip(int rangeStart, int rangeEnd) { + public void flip(long rangeStart, long rangeEnd) { long stamp = lock.writeLock(); try { super.flip(rangeStart, rangeEnd); @@ -163,50 +288,214 @@ public void flip(int rangeStart, int rangeEnd) { } @Override - public void flip(int x) { + public void flip(int rangeStart, int rangeEnd) { long stamp = lock.writeLock(); try { - super.flip(x); + super.flip(rangeStart, rangeEnd); } finally { lock.unlockWrite(stamp); } } @Override - public void serialize(java.io.DataOutput out) throws java.io.IOException { + public long getLongCardinality() { long stamp = lock.tryOptimisticRead(); if (stamp != 0) { - super.serialize(out); - if (!lock.validate(stamp)) { - stamp = lock.readLock(); - try { - super.serialize(out); - } finally { - lock.unlockRead(stamp); - } + long longCardinality = super.getLongCardinality(); + if (lock.validate(stamp)) { + return longCardinality; + } + } + stamp = lock.readLock(); + try { + return super.getLongCardinality(); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public boolean cardinalityExceeds(long threshold) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + boolean cardinalityExceeds = super.cardinalityExceeds(threshold); + if (lock.validate(stamp)) { + return cardinalityExceeds; + } + } + stamp = lock.readLock(); + try { + return super.cardinalityExceeds(threshold); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public void forEach(IntConsumer ic) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + super.forEach(ic); + if (lock.validate(stamp)) { + return; + } + } + stamp = lock.readLock(); + try { + super.forEach(ic); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public void forAllInRange(int uStart, int length, RelativeRangeConsumer rrc) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + super.forAllInRange(uStart, length, rrc); + if (lock.validate(stamp)) { + return; + } + } + stamp = lock.readLock(); + try { + super.forAllInRange(uStart, length, rrc); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public long getLongSizeInBytes() { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + long longSizeInBytes = super.getLongSizeInBytes(); + if (lock.validate(stamp)) { + return longSizeInBytes; + } + } + stamp = lock.readLock(); + try { + return super.getLongSizeInBytes(); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public int hashCode() { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + int hashCode = super.hashCode(); + if (lock.validate(stamp)) { + return hashCode; + } + } + stamp = lock.readLock(); + try { + return super.hashCode(); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public boolean hasRunCompression() { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + boolean hasRunCompression = super.hasRunCompression(); + if (lock.validate(stamp)) { + return hasRunCompression; } - } else { - stamp = lock.readLock(); - try { - super.serialize(out); - } finally { - lock.unlockRead(stamp); + } + stamp = lock.readLock(); + try { + return super.hasRunCompression(); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public boolean isEmpty() { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + boolean isEmpty = super.isEmpty(); + if (lock.validate(stamp)) { + return isEmpty; + } + } + stamp = lock.readLock(); + try { + return super.isEmpty(); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public RoaringBitmap limit(int maxcardinality) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + RoaringBitmap limit = super.limit(maxcardinality); + if (lock.validate(stamp)) { + return limit; } } + stamp = lock.readLock(); + try { + return super.limit(maxcardinality); + } finally { + lock.unlockRead(stamp); + } } @Override - public void deserialize(java.io.DataInput in) throws java.io.IOException { + public void or(RoaringBitmap x2) { long stamp = lock.writeLock(); try { - super.deserialize(in); + super.or(x2); } finally { lock.unlockWrite(stamp); } } @Override - public void readExternal(java.io.ObjectInput in) throws java.io.IOException { + public long rankLong(int x) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + long rankLong = super.rankLong(x); + if (lock.validate(stamp)) { + return rankLong; + } + } + stamp = lock.readLock(); + try { + return super.rankLong(x); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public long rangeCardinality(long start, long end) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + long rangeCardinality = super.rangeCardinality(start, end); + if (lock.validate(stamp)) { + return rangeCardinality; + } + } + stamp = lock.readLock(); + try { + return super.rangeCardinality(start, end); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public void readExternal(ObjectInput in) throws IOException { long stamp = lock.writeLock(); try { super.readExternal(in); @@ -216,25 +505,269 @@ public void readExternal(java.io.ObjectInput in) throws java.io.IOException { } @Override - public void writeExternal(java.io.ObjectOutput out) throws java.io.IOException { + public void remove(int x) { + long stamp = lock.writeLock(); + try { + super.remove(x); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void remove(long rangeStart, long rangeEnd) { + long stamp = lock.writeLock(); + try { + super.remove(rangeStart, rangeEnd); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public boolean removeRunCompression() { + long stamp = lock.writeLock(); + try { + return super.removeRunCompression(); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public boolean runOptimize() { + long stamp = lock.writeLock(); + try { + return super.runOptimize(); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public boolean contains(RoaringBitmap subset) { long stamp = lock.tryOptimisticRead(); if (stamp != 0) { - super.writeExternal(out); - if (!lock.validate(stamp)) { - stamp = lock.readLock(); - try { - super.writeExternal(out); - } finally { - lock.unlockRead(stamp); - } - } - } else { - stamp = lock.readLock(); - try { - super.writeExternal(out); - } finally { - lock.unlockRead(stamp); + boolean contains = super.contains(subset); + if (lock.validate(stamp)) { + return contains; + } + } + stamp = lock.readLock(); + try { + return super.contains(subset); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public int select(int j) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + int select = super.select(j); + if (lock.validate(stamp)) { + return select; + } + } + stamp = lock.readLock(); + try { + return super.select(j); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public long nextValue(int fromValue) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + long nextValue = super.nextValue(fromValue); + if (lock.validate(stamp)) { + return nextValue; + } + } + stamp = lock.readLock(); + try { + return super.nextValue(fromValue); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public long previousValue(int fromValue) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + long previousValue = super.previousValue(fromValue); + if (lock.validate(stamp)) { + return previousValue; + } + } + stamp = lock.readLock(); + try { + return super.previousValue(fromValue); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public long nextAbsentValue(int fromValue) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + long nextAbsentValue = super.nextAbsentValue(fromValue); + if (lock.validate(stamp)) { + return nextAbsentValue; } } + stamp = lock.readLock(); + try { + return super.nextAbsentValue(fromValue); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public long previousAbsentValue(int fromValue) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + long previousAbsentValue = super.previousAbsentValue(fromValue); + if (lock.validate(stamp)) { + return previousAbsentValue; + } + } + stamp = lock.readLock(); + try { + return super.previousAbsentValue(fromValue); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public int first() { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + int first = super.first(); + if (lock.validate(stamp)) { + return first; + } + } + stamp = lock.readLock(); + try { + return super.first(); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public int last() { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + int last = super.last(); + if (lock.validate(stamp)) { + return last; + } + } + stamp = lock.readLock(); + try { + return super.last(); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public void serialize(DataOutput out) throws IOException { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + super.serialize(out); + if (lock.validate(stamp)) { + return; + } + } + stamp = lock.readLock(); + try { + super.serialize(out); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public void serialize(ByteBuffer buffer) { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + super.serialize(buffer); + if (lock.validate(stamp)) { + return; + } + } + stamp = lock.readLock(); + try { + super.serialize(buffer); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public int[] toArray() { + long stamp = lock.tryOptimisticRead(); + if (stamp != 0) { + int[] toArray = super.toArray(); + if (lock.validate(stamp)) { + return toArray; + } + } + stamp = lock.readLock(); + try { + return super.toArray(); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public void append(char key, Container container) { + long stamp = lock.writeLock(); + try { + super.append(key, container); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void trim() { + long stamp = lock.writeLock(); + try { + super.trim(); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + long stamp = lock.writeLock(); + try { + super.writeExternal(out); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void xor(RoaringBitmap x2) { + long stamp = lock.writeLock(); + try { + super.xor(x2); + } finally { + lock.unlockWrite(stamp); + } } } From fbfcbf4e9b20d39c741671f1073bd84b12f33f30 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Sun, 16 Jun 2024 12:48:03 +0800 Subject: [PATCH 03/22] fix checkstyle --- .../common/util/collections/ConcurrentRoaringBitmap.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java index 1a898e46fd949..3b8f033e95d52 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java @@ -18,10 +18,6 @@ */ package org.apache.pulsar.common.util.collections; -import org.roaringbitmap.Container; -import org.roaringbitmap.IntConsumer; -import org.roaringbitmap.RelativeRangeConsumer; -import org.roaringbitmap.RoaringBitmap; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -29,6 +25,10 @@ import java.io.ObjectOutput; import java.nio.ByteBuffer; import java.util.concurrent.locks.StampedLock; +import org.roaringbitmap.Container; +import org.roaringbitmap.IntConsumer; +import org.roaringbitmap.RelativeRangeConsumer; +import org.roaringbitmap.RoaringBitmap; public class ConcurrentRoaringBitmap extends RoaringBitmap { From 7719be3462fb03be23b4f0a9356dfe6ebca43462 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Sun, 16 Jun 2024 22:21:22 +0800 Subject: [PATCH 04/22] improve code --- .../ConcurrentOpenLongPairRangeSet.java | 90 +- .../collections/ConcurrentRoaringBitSet.java | 432 ++++++++++ .../collections/ConcurrentRoaringBitmap.java | 773 ------------------ .../util/collections/RoaringBitSet.java | 255 ++++++ 4 files changed, 733 insertions(+), 817 deletions(-) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java delete mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java index 8e7da30a4615e..da15b78a60fc9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java @@ -29,7 +29,6 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang.mutable.MutableInt; -import org.roaringbitmap.RoaringBitmap; /** * A Concurrent set comprising zero or more ranges of type {@link LongPair}. This can be alternative of @@ -44,7 +43,7 @@ */ public class ConcurrentOpenLongPairRangeSet> implements LongPairRangeSet { - protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); + protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); private boolean threadSafe = true; private final int bitSetSize; private final LongPairConsumer consumer; @@ -83,26 +82,29 @@ public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, lon if (lowerKey != upperKey) { // (1) set lower to last in lowerRange.getKey() if (isValid(lowerKey, lowerValue)) { - RoaringBitmap rangeBitSet = rangeBitSetMap.get(lowerKey); + BitSet rangeBitSet = rangeBitSetMap.get(lowerKey); // if lower and upper has different key/ledger then set ranges for lower-key only if // a. bitSet already exist and given value is not the last value in the bitset. // it will prevent setting up values which are not actually expected to set // eg: (2:10..4:10] in this case, don't set any value for 2:10 and set [4:0..4:10] - if (rangeBitSet != null && (rangeBitSet.last() > lowerValueOpen)) { - int lastValue = (int) rangeBitSet.previousValue(rangeBitSet.last()); - rangeBitSet.add((int) lowerValue, (int) Math.max(lastValue, lowerValue) + 1); + if (rangeBitSet != null && (rangeBitSet.previousSetBit(rangeBitSet.size()) > lowerValueOpen)) { + int lastValue = rangeBitSet.previousSetBit(rangeBitSet.size()); + rangeBitSet.set((int) lowerValue, (int) Math.max(lastValue, lowerValue) + 1); } } // (2) set 0th-index to upper-index in upperRange.getKey() if (isValid(upperKey, upperValue)) { - RoaringBitmap rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); - rangeBitSet.add(0, upperValue + 1); + BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); + if (rangeBitSet != null) { + rangeBitSet.set(0, (int) upperValue + 1); + } } // No-op if values are not valid eg: if lower == LongPair.earliest or upper == LongPair.latest then nothing // to set } else { - RoaringBitmap rangeBitSet = rangeBitSetMap.computeIfAbsent(lowerKey, (k) -> createNewBitSet()); - rangeBitSet.add(lowerValue, upperValue + 1); + long key = lowerKey; + BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(key, (k) -> createNewBitSet()); + rangeBitSet.set((int) lowerValue, (int) upperValue + 1); } updatedAfterCachedForSize = true; updatedAfterCachedForToString = true; @@ -116,25 +118,25 @@ private boolean isValid(long key, long value) { @Override public boolean contains(long key, long value) { - RoaringBitmap rangeBitSet = rangeBitSetMap.get(key); + BitSet rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { - return rangeBitSet.contains(getSafeEntry(value)); + return rangeBitSet.get(getSafeEntry(value)); } return false; } @Override public Range rangeContaining(long key, long value) { - RoaringBitmap rangeBitSet = rangeBitSetMap.get(key); + BitSet rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { - if (!rangeBitSet.contains(getSafeEntry(value))) { + if (!rangeBitSet.get(getSafeEntry(value))) { // if position is not part of any range then return null return null; } - int lowerValue = (int) (rangeBitSet.previousAbsentValue(getSafeEntry(value)) + 1); + int lowerValue = rangeBitSet.previousClearBit(getSafeEntry(value)) + 1; final T lower = consumer.apply(key, lowerValue); final T upper = consumer.apply(key, - Math.max(rangeBitSet.nextAbsentValue(getSafeEntry(value)) - 1, lowerValue)); + Math.max(rangeBitSet.nextClearBit(getSafeEntry(value)) - 1, lowerValue)); return Range.closed(lower, upper); } return null; @@ -150,7 +152,7 @@ public boolean isEmpty() { if (rangeBitSetMap.isEmpty()) { return true; } - for (RoaringBitmap rangeBitSet : rangeBitSetMap.values()) { + for (BitSet rangeBitSet : rangeBitSetMap.values()) { if (!rangeBitSet.isEmpty()) { return false; } @@ -170,10 +172,10 @@ public Range span() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); - Entry lastSet = rangeBitSetMap.lastEntry(); - int first = (int) firstSet.getValue().nextValue(0); - int last = (int) lastSet.getValue().previousValue(lastSet.getValue().last()); + Entry firstSet = rangeBitSetMap.firstEntry(); + Entry lastSet = rangeBitSetMap.lastEntry(); + int first = firstSet.getValue().nextSetBit(0); + int last = lastSet.getValue().previousSetBit(lastSet.getValue().size()); return Range.openClosed(consumer.apply(firstSet.getKey(), first - 1), consumer.apply(lastSet.getKey(), last)); } @@ -213,17 +215,17 @@ public void forEachRawRange(RawRangeProcessor processor) { if (set.isEmpty()) { return; } - int first = (int) set.nextValue(0); - int last = (int) set.previousValue(set.last()); + int first = set.nextSetBit(0); + int last = set.previousSetBit(set.size()); int currentClosedMark = first; while (currentClosedMark != -1 && currentClosedMark <= last) { - int nextOpenMark = (int) set.nextAbsentValue(currentClosedMark); + int nextOpenMark = set.nextClearBit(currentClosedMark); if (!processor.processRawRange(key, currentClosedMark - 1, key, nextOpenMark - 1)) { completed.set(true); break; } - currentClosedMark = (int) set.nextValue(nextOpenMark); + currentClosedMark = set.nextSetBit(nextOpenMark); } }); } @@ -234,9 +236,9 @@ public Range firstRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); - int lower = (int) firstSet.getValue().nextValue(0); - int upper = (int) Math.max(lower, firstSet.getValue().nextAbsentValue(lower) - 1); + Entry firstSet = rangeBitSetMap.firstEntry(); + int lower = firstSet.getValue().nextSetBit(0); + int upper = Math.max(lower, firstSet.getValue().nextClearBit(lower) - 1); return Range.openClosed(consumer.apply(firstSet.getKey(), lower - 1), consumer.apply(firstSet.getKey(), upper)); } @@ -245,30 +247,30 @@ public Range lastRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry lastSet = rangeBitSetMap.lastEntry(); - int upper = (int) lastSet.getValue().previousValue(lastSet.getValue().last()); - int lower = (int) Math.min(lastSet.getValue().previousAbsentValue(upper), upper); + Entry lastSet = rangeBitSetMap.lastEntry(); + int upper = lastSet.getValue().previousSetBit(lastSet.getValue().size()); + int lower = Math.min(lastSet.getValue().previousClearBit(upper), upper); return Range.openClosed(consumer.apply(lastSet.getKey(), lower), consumer.apply(lastSet.getKey(), upper)); } @Override public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { - NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); + NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); MutableInt v = new MutableInt(0); subMap.forEach((key, bitset) -> { if (key == lowerKey || key == upperKey) { - RoaringBitmap temp = bitset.clone(); + BitSet temp = (BitSet) bitset.clone(); // Trim the bitset index which < lowerValue if (key == lowerKey) { - temp.remove(0, Math.max(0, lowerValue)); + temp.clear(0, (int) Math.max(0, lowerValue)); } // Trim the bitset index which > upperValue if (key == upperKey) { - temp.remove(Math.min(upperValue, temp.last()), temp.last()); + temp.clear((int) Math.min(upperValue + 1, temp.length()), temp.length()); } - v.add(temp.getCardinality()); + v.add(temp.cardinality()); } else { - v.add(bitset.getCardinality()); + v.add(bitset.cardinality()); } }); return v.intValue(); @@ -337,7 +339,7 @@ public void add(Range range) { // #addOpenClosed doesn't create bitSet for lower-key because it avoids setting up values for non-exist items // into the key-ledger. so, create bitSet and initialize so, it can't be ignored at #addOpenClosed rangeBitSetMap.computeIfAbsent(lowerEndpoint.getKey(), (key) -> createNewBitSet()) - .add((int) lowerValueOpen + 1); + .set((int) lowerValueOpen + 1); this.addOpenClosed(lowerEndpoint.getKey(), lowerValueOpen, upperEndpoint.getKey(), upperValueClosed); } @@ -380,15 +382,15 @@ public void remove(Range range) { // remove all the keys between two endpoint keys rangeBitSetMap.forEach((key, set) -> { if (lowerEndpoint.getKey() == upperEndpoint.getKey() && key == upperEndpoint.getKey()) { - set.remove(lower, upper + 1); + set.clear((int) lower, (int) upper + 1); } else { // eg: remove-range: [(3,5) - (5,5)] -> Delete all items from 3,6->3,N,4.*,5,0->5,5 if (key == lowerEndpoint.getKey()) { // remove all entries from given position to last position - set.remove((int) lower, set.previousValue(set.last())); + set.clear((int) lower, set.previousSetBit(set.size())); } else if (key == upperEndpoint.getKey()) { // remove all entries from 0 to given position - set.remove(0, (int) upper + 1); + set.clear(0, (int) upper + 1); } else if (key > lowerEndpoint.getKey() && key < upperEndpoint.getKey()) { rangeBitSetMap.remove(key); } @@ -411,8 +413,8 @@ private int getSafeEntry(long value) { return (int) Math.max(value, -1); } - private RoaringBitmap createNewBitSet() { - return threadSafe ? new ConcurrentRoaringBitmap() : new RoaringBitmap(); + private BitSet createNewBitSet() { + return this.threadSafe ? new ConcurrentRoaringBitSet() : new RoaringBitSet(); } -} +} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java new file mode 100644 index 0000000000000..ec7003d180669 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java @@ -0,0 +1,432 @@ +package org.apache.pulsar.common.util.collections; + +import java.util.BitSet; +import java.util.concurrent.locks.StampedLock; +import java.util.stream.IntStream; + +public class ConcurrentRoaringBitSet extends RoaringBitSet { + private final StampedLock rwLock = new StampedLock(); + + public ConcurrentRoaringBitSet() { + super(); + } + + @Override + public boolean get(int bitIndex) { + long stamp = rwLock.tryOptimisticRead(); + boolean isSet = super.get(bitIndex); + if (!rwLock.validate(stamp)) { + stamp = rwLock.readLock(); + try { + isSet = super.get(bitIndex); + } finally { + rwLock.unlockRead(stamp); + } + } + return isSet; + } + + @Override + public void set(int bitIndex) { + long stamp = rwLock.writeLock(); + try { + super.set(bitIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void clear(int bitIndex) { + long stamp = rwLock.writeLock(); + try { + super.clear(bitIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void set(int fromIndex, int toIndex) { + long stamp = rwLock.writeLock(); + try { + super.set(fromIndex, toIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void clear(int fromIndex, int toIndex) { + long stamp = rwLock.writeLock(); + try { + super.clear(fromIndex, toIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void clear() { + long stamp = rwLock.writeLock(); + try { + super.clear(); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public int nextSetBit(int fromIndex) { + long stamp = rwLock.tryOptimisticRead(); + int nextSetBit = super.nextSetBit(fromIndex); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + nextSetBit = super.nextSetBit(fromIndex); + } finally { + rwLock.unlockRead(stamp); + } + } + return nextSetBit; + } + + @Override + public int nextClearBit(int fromIndex) { + long stamp = rwLock.tryOptimisticRead(); + int nextClearBit = super.nextClearBit(fromIndex); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + nextClearBit = super.nextClearBit(fromIndex); + } finally { + rwLock.unlockRead(stamp); + } + } + return nextClearBit; + } + + @Override + public int previousSetBit(int fromIndex) { + long stamp = rwLock.tryOptimisticRead(); + int previousSetBit = super.previousSetBit(fromIndex); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + previousSetBit = super.previousSetBit(fromIndex); + } finally { + rwLock.unlockRead(stamp); + } + } + return previousSetBit; + } + + @Override + public int previousClearBit(int fromIndex) { + long stamp = rwLock.tryOptimisticRead(); + int previousClearBit = super.previousClearBit(fromIndex); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + previousClearBit = super.previousClearBit(fromIndex); + } finally { + rwLock.unlockRead(stamp); + } + } + return previousClearBit; + } + + @Override + public boolean isEmpty() { + long stamp = rwLock.tryOptimisticRead(); + boolean isEmpty = super.isEmpty(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + isEmpty = super.isEmpty(); + } finally { + rwLock.unlockRead(stamp); + } + } + return isEmpty; + } + + @Override + public int cardinality() { + long stamp = rwLock.tryOptimisticRead(); + int cardinality = super.cardinality(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + cardinality = super.cardinality(); + } finally { + rwLock.unlockRead(stamp); + } + } + return cardinality; + } + + @Override + public int size() { + long stamp = rwLock.tryOptimisticRead(); + int size = super.size(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + size = super.size(); + } finally { + rwLock.unlockRead(stamp); + } + } + return size; + } + + @Override + public byte[] toByteArray() { + long stamp = rwLock.tryOptimisticRead(); + byte[] byteArray = super.toByteArray(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + byteArray = super.toByteArray(); + } finally { + rwLock.unlockRead(stamp); + } + } + return byteArray; + } + + @Override + public long[] toLongArray() { + long stamp = rwLock.tryOptimisticRead(); + long[] longArray = super.toLongArray(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + longArray = super.toLongArray(); + } finally { + rwLock.unlockRead(stamp); + } + } + return longArray; + } + + @Override + public void flip(int bitIndex) { + long stamp = rwLock.writeLock(); + try { + super.flip(bitIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void flip(int fromIndex, int toIndex) { + long stamp = rwLock.writeLock(); + try { + super.flip(fromIndex, toIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void set(int bitIndex, boolean value) { + long stamp = rwLock.writeLock(); + try { + super.set(bitIndex, value); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void set(int fromIndex, int toIndex, boolean value) { + long stamp = rwLock.writeLock(); + try { + super.set(fromIndex, toIndex, value); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public BitSet get(int fromIndex, int toIndex) { + long stamp = rwLock.tryOptimisticRead(); + BitSet bitSet = super.get(fromIndex, toIndex); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + bitSet = super.get(fromIndex, toIndex); + } finally { + rwLock.unlockRead(stamp); + } + } + return bitSet; + } + + /** + * Thread-safe version of {@code length()}. + * StampedLock is not reentrant and that's why the length() method is not overridden. Overriding length() method + * would require to use a reentrant lock which would be less performant. + * + * @return length of the bit set + */ + public int safeLength() { + long stamp = rwLock.tryOptimisticRead(); + int length = super.length(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + length = super.length(); + } finally { + rwLock.unlockRead(stamp); + } + } + return length; + } + + @Override + public boolean intersects(BitSet set) { + long stamp = rwLock.writeLock(); + try { + return super.intersects(set); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void and(BitSet set) { + long stamp = rwLock.writeLock(); + try { + super.and(set); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void or(BitSet set) { + long stamp = rwLock.writeLock(); + try { + super.or(set); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void xor(BitSet set) { + long stamp = rwLock.writeLock(); + try { + super.xor(set); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void andNot(BitSet set) { + long stamp = rwLock.writeLock(); + try { + super.andNot(set); + } finally { + rwLock.unlockWrite(stamp); + } + } + + /** + * Returns the clone of the internal wrapped {@code BitSet}. + * This won't be a clone of the {@code ConcurrentBitSet} object. + * + * @return a clone of the internal wrapped {@code BitSet} + */ + @Override + public Object clone() { + long stamp = rwLock.tryOptimisticRead(); + BitSet clonedBitSet = (BitSet) super.clone(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + clonedBitSet = (BitSet) super.clone(); + } finally { + rwLock.unlockRead(stamp); + } + } + return clonedBitSet; + } + + @Override + public String toString() { + long stamp = rwLock.tryOptimisticRead(); + String str = super.toString(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + str = super.toString(); + } finally { + rwLock.unlockRead(stamp); + } + } + return str; + } + + /** + * This operation is not supported on {@code ConcurrentBitSet}. + */ + @Override + public IntStream stream() { + throw new UnsupportedOperationException("stream is not supported"); + } + + public boolean equals(final Object o) { + if (o == this) { + return true; + } + if (!(o instanceof ConcurrentBitSet)) { + return false; + } + long stamp = rwLock.tryOptimisticRead(); + boolean isEqual = super.equals(o); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + isEqual = super.equals(o); + } finally { + rwLock.unlockRead(stamp); + } + } + return isEqual; + } + + public int hashCode() { + long stamp = rwLock.tryOptimisticRead(); + int hashCode = super.hashCode(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + hashCode = super.hashCode(); + } finally { + rwLock.unlockRead(stamp); + } + } + return hashCode; + } +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java deleted file mode 100644 index 3b8f033e95d52..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java +++ /dev/null @@ -1,773 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.collections; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.nio.ByteBuffer; -import java.util.concurrent.locks.StampedLock; -import org.roaringbitmap.Container; -import org.roaringbitmap.IntConsumer; -import org.roaringbitmap.RelativeRangeConsumer; -import org.roaringbitmap.RoaringBitmap; - -public class ConcurrentRoaringBitmap extends RoaringBitmap { - - private final StampedLock lock = new StampedLock(); - - @Override - public void add(int x) { - long stamp = lock.writeLock(); - try { - super.add(x); - } finally { - lock.unlockWrite(stamp); - } - } - - - @Override - public void addN(int[] dat, int offset, int n) { - long stamp = lock.writeLock(); - try { - super.addN(dat, offset, n); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void add(long rangeStart, long rangeEnd) { - long stamp = lock.writeLock(); - try { - super.add(rangeStart, rangeEnd); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void add(int rangeStart, int rangeEnd) { - long stamp = lock.writeLock(); - try { - super.add(rangeStart, rangeEnd); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public boolean intersects(long minimum, long supremum) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - boolean intersects = super.intersects(minimum, supremum); - if (lock.validate(stamp)) { - return intersects; - } - } - stamp = lock.readLock(); - try { - return super.intersects(minimum, supremum); - } finally { - lock.unlockRead(stamp); - } - } - - - public void and(final RoaringBitmap x2) { - long stamp = lock.writeLock(); - try { - super.and(x2); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void andNot(RoaringBitmap x2) { - long stamp = lock.writeLock(); - try { - super.andNot(x2); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void orNot(RoaringBitmap other, long rangeEnd) { - long stamp = lock.writeLock(); - try { - super.orNot(other, rangeEnd); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public boolean checkedAdd(int x) { - long stamp = lock.writeLock(); - try { - return super.checkedAdd(x); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public boolean checkedRemove(int x) { - long stamp = lock.writeLock(); - try { - return super.checkedRemove(x); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void clear() { - long stamp = lock.writeLock(); - try { - super.clear(); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public RoaringBitmap clone() { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - RoaringBitmap clone = super.clone(); - if (lock.validate(stamp)) { - return clone; - } - } - stamp = lock.readLock(); - try { - return super.clone(); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public boolean contains(int x) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - boolean contains = super.contains(x); - if (lock.validate(stamp)) { - return contains; - } - } - stamp = lock.readLock(); - try { - return super.contains(x); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public boolean contains(long minimum, long supremum) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - boolean contains = super.contains(minimum, supremum); - if (lock.validate(stamp)) { - return contains; - } - } - stamp = lock.readLock(); - try { - return super.contains(minimum, supremum); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public void deserialize(DataInput in, byte[] buffer) throws IOException { - long stamp = lock.writeLock(); - try { - super.deserialize(in, buffer); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void deserialize(DataInput in) throws IOException { - long stamp = lock.writeLock(); - try { - super.deserialize(in); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void deserialize(ByteBuffer bbf) throws IOException { - long stamp = lock.writeLock(); - try { - super.deserialize(bbf); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public boolean equals(Object o) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - boolean equals = super.equals(o); - if (lock.validate(stamp)) { - return equals; - } - } - stamp = lock.readLock(); - try { - return super.equals(o); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public boolean isHammingSimilar(RoaringBitmap other, int tolerance) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - boolean isHammingSimilar = super.isHammingSimilar(other, tolerance); - if (lock.validate(stamp)) { - return isHammingSimilar; - } - } - stamp = lock.readLock(); - try { - return super.isHammingSimilar(other, tolerance); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public void flip(int x) { - long stamp = lock.writeLock(); - try { - super.flip(x); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void flip(long rangeStart, long rangeEnd) { - long stamp = lock.writeLock(); - try { - super.flip(rangeStart, rangeEnd); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void flip(int rangeStart, int rangeEnd) { - long stamp = lock.writeLock(); - try { - super.flip(rangeStart, rangeEnd); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public long getLongCardinality() { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - long longCardinality = super.getLongCardinality(); - if (lock.validate(stamp)) { - return longCardinality; - } - } - stamp = lock.readLock(); - try { - return super.getLongCardinality(); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public boolean cardinalityExceeds(long threshold) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - boolean cardinalityExceeds = super.cardinalityExceeds(threshold); - if (lock.validate(stamp)) { - return cardinalityExceeds; - } - } - stamp = lock.readLock(); - try { - return super.cardinalityExceeds(threshold); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public void forEach(IntConsumer ic) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - super.forEach(ic); - if (lock.validate(stamp)) { - return; - } - } - stamp = lock.readLock(); - try { - super.forEach(ic); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public void forAllInRange(int uStart, int length, RelativeRangeConsumer rrc) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - super.forAllInRange(uStart, length, rrc); - if (lock.validate(stamp)) { - return; - } - } - stamp = lock.readLock(); - try { - super.forAllInRange(uStart, length, rrc); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public long getLongSizeInBytes() { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - long longSizeInBytes = super.getLongSizeInBytes(); - if (lock.validate(stamp)) { - return longSizeInBytes; - } - } - stamp = lock.readLock(); - try { - return super.getLongSizeInBytes(); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public int hashCode() { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - int hashCode = super.hashCode(); - if (lock.validate(stamp)) { - return hashCode; - } - } - stamp = lock.readLock(); - try { - return super.hashCode(); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public boolean hasRunCompression() { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - boolean hasRunCompression = super.hasRunCompression(); - if (lock.validate(stamp)) { - return hasRunCompression; - } - } - stamp = lock.readLock(); - try { - return super.hasRunCompression(); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public boolean isEmpty() { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - boolean isEmpty = super.isEmpty(); - if (lock.validate(stamp)) { - return isEmpty; - } - } - stamp = lock.readLock(); - try { - return super.isEmpty(); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public RoaringBitmap limit(int maxcardinality) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - RoaringBitmap limit = super.limit(maxcardinality); - if (lock.validate(stamp)) { - return limit; - } - } - stamp = lock.readLock(); - try { - return super.limit(maxcardinality); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public void or(RoaringBitmap x2) { - long stamp = lock.writeLock(); - try { - super.or(x2); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public long rankLong(int x) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - long rankLong = super.rankLong(x); - if (lock.validate(stamp)) { - return rankLong; - } - } - stamp = lock.readLock(); - try { - return super.rankLong(x); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public long rangeCardinality(long start, long end) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - long rangeCardinality = super.rangeCardinality(start, end); - if (lock.validate(stamp)) { - return rangeCardinality; - } - } - stamp = lock.readLock(); - try { - return super.rangeCardinality(start, end); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public void readExternal(ObjectInput in) throws IOException { - long stamp = lock.writeLock(); - try { - super.readExternal(in); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void remove(int x) { - long stamp = lock.writeLock(); - try { - super.remove(x); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void remove(long rangeStart, long rangeEnd) { - long stamp = lock.writeLock(); - try { - super.remove(rangeStart, rangeEnd); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public boolean removeRunCompression() { - long stamp = lock.writeLock(); - try { - return super.removeRunCompression(); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public boolean runOptimize() { - long stamp = lock.writeLock(); - try { - return super.runOptimize(); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public boolean contains(RoaringBitmap subset) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - boolean contains = super.contains(subset); - if (lock.validate(stamp)) { - return contains; - } - } - stamp = lock.readLock(); - try { - return super.contains(subset); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public int select(int j) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - int select = super.select(j); - if (lock.validate(stamp)) { - return select; - } - } - stamp = lock.readLock(); - try { - return super.select(j); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public long nextValue(int fromValue) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - long nextValue = super.nextValue(fromValue); - if (lock.validate(stamp)) { - return nextValue; - } - } - stamp = lock.readLock(); - try { - return super.nextValue(fromValue); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public long previousValue(int fromValue) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - long previousValue = super.previousValue(fromValue); - if (lock.validate(stamp)) { - return previousValue; - } - } - stamp = lock.readLock(); - try { - return super.previousValue(fromValue); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public long nextAbsentValue(int fromValue) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - long nextAbsentValue = super.nextAbsentValue(fromValue); - if (lock.validate(stamp)) { - return nextAbsentValue; - } - } - stamp = lock.readLock(); - try { - return super.nextAbsentValue(fromValue); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public long previousAbsentValue(int fromValue) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - long previousAbsentValue = super.previousAbsentValue(fromValue); - if (lock.validate(stamp)) { - return previousAbsentValue; - } - } - stamp = lock.readLock(); - try { - return super.previousAbsentValue(fromValue); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public int first() { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - int first = super.first(); - if (lock.validate(stamp)) { - return first; - } - } - stamp = lock.readLock(); - try { - return super.first(); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public int last() { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - int last = super.last(); - if (lock.validate(stamp)) { - return last; - } - } - stamp = lock.readLock(); - try { - return super.last(); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public void serialize(DataOutput out) throws IOException { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - super.serialize(out); - if (lock.validate(stamp)) { - return; - } - } - stamp = lock.readLock(); - try { - super.serialize(out); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public void serialize(ByteBuffer buffer) { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - super.serialize(buffer); - if (lock.validate(stamp)) { - return; - } - } - stamp = lock.readLock(); - try { - super.serialize(buffer); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public int[] toArray() { - long stamp = lock.tryOptimisticRead(); - if (stamp != 0) { - int[] toArray = super.toArray(); - if (lock.validate(stamp)) { - return toArray; - } - } - stamp = lock.readLock(); - try { - return super.toArray(); - } finally { - lock.unlockRead(stamp); - } - } - - @Override - public void append(char key, Container container) { - long stamp = lock.writeLock(); - try { - super.append(key, container); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void trim() { - long stamp = lock.writeLock(); - try { - super.trim(); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void writeExternal(ObjectOutput out) throws IOException { - long stamp = lock.writeLock(); - try { - super.writeExternal(out); - } finally { - lock.unlockWrite(stamp); - } - } - - @Override - public void xor(RoaringBitmap x2) { - long stamp = lock.writeLock(); - try { - super.xor(x2); - } finally { - lock.unlockWrite(stamp); - } - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java new file mode 100644 index 0000000000000..e1b1d02b3c7be --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import java.nio.ByteBuffer; +import java.util.BitSet; +import java.util.stream.IntStream; + +import org.roaringbitmap.RoaringBitmap; + +@SuppressWarnings("all") +public class RoaringBitSet extends BitSet { + private static final long serialVersionUID = 1L; + + private RoaringBitmap roaringBitmap; + + public RoaringBitSet() { + super(0); + roaringBitmap = new RoaringBitmap(); + } + + private RoaringBitSet(RoaringBitmap roaringBitmap) { + super(0); + this.roaringBitmap = roaringBitmap; + } + + public RoaringBitmap getRoaringBitmap() { + return roaringBitmap; + } + + @Override + public void set(int bitIndex) { + roaringBitmap.add(bitIndex); + } + + @Override + public void set(int bitIndex, boolean value) { + if (value) { + roaringBitmap.add(bitIndex); + } else { + roaringBitmap.remove(bitIndex); + } + } + + @Override + public void set(int fromIndex, int toIndex) { + roaringBitmap.add(fromIndex, toIndex); + } + + @Override + public void set(int fromIndex, int toIndex, boolean value) { + if (value) { + roaringBitmap.add(fromIndex, toIndex); + } else { + roaringBitmap.remove(fromIndex, toIndex); + } + } + + @Override + public void clear(int bitIndex) { + roaringBitmap.remove(bitIndex); + } + + @Override + public void clear(int fromIndex, int toIndex) { + roaringBitmap.remove(fromIndex, toIndex); + } + + @Override + public void clear() { + roaringBitmap.clear(); + } + + @Override + public boolean get(int bitIndex) { + return roaringBitmap.contains(bitIndex); + } + + @Override + public BitSet get(int fromIndex, int toIndex) { + RoaringBitmap rb = roaringBitmap.clone(); + rb.flip(0, fromIndex); + rb.flip(toIndex, rb.last()); + return new RoaringBitSet(rb); + } + + @Override + public int nextSetBit(int fromIndex) { + return (int) roaringBitmap.nextValue(fromIndex); + } + + @Override + public int nextClearBit(int fromIndex) { + return (int) roaringBitmap.nextAbsentValue(fromIndex); + } + + @Override + public int previousSetBit(int fromIndex) { + return (int) roaringBitmap.previousValue(fromIndex); + } + + @Override + public int previousClearBit(int fromIndex) { + return (int) roaringBitmap.previousAbsentValue(fromIndex); + } + + @Override + public int length() { + if (roaringBitmap.isEmpty()) { + return 0; + } + return roaringBitmap.last() + 1; + } + + @Override + public boolean isEmpty() { + return roaringBitmap.isEmpty(); + } + + @Override + public boolean intersects(BitSet set) { + RoaringBitmap bitmap = set instanceof RoaringBitSet ? ((RoaringBitSet) set).roaringBitmap : fromBitSet(set); + if (bitmap.isEmpty() || roaringBitmap.isEmpty()) { + return false; + } + long rangeStart = bitmap.first(); + long rangeEnd = bitmap.last(); + return roaringBitmap.intersects(rangeStart, rangeEnd); + } + + @Override + public int cardinality() { + return roaringBitmap.getCardinality(); + } + + @Override + public void and(BitSet set) { + if (set instanceof RoaringBitSet) { + roaringBitmap.and(((RoaringBitSet) set).roaringBitmap); + } else { + roaringBitmap.and(fromBitSet(set)); + } + } + + @Override + public void or(BitSet set) { + if (set instanceof RoaringBitSet) { + roaringBitmap.or(((RoaringBitSet) set).roaringBitmap); + } else { + roaringBitmap.or(fromBitSet(set)); + } + } + + @Override + public void xor(BitSet set) { + if (set instanceof RoaringBitSet) { + roaringBitmap.xor(((RoaringBitSet) set).roaringBitmap); + } else { + roaringBitmap.xor(fromBitSet(set)); + } + } + + @Override + public void andNot(BitSet set) { + if (set instanceof RoaringBitSet) { + roaringBitmap.andNot(((RoaringBitSet) set).roaringBitmap); + } else { + roaringBitmap.andNot(fromBitSet(set)); + } + } + + @Override + public int hashCode() { + return roaringBitmap.hashCode(); + } + + @Override + public int size() { + return roaringBitmap.getCardinality(); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof RoaringBitSet) { + return roaringBitmap.equals(((RoaringBitSet) obj).roaringBitmap); + } else { + return super.equals(obj); + } + } + + @Override + public Object clone() { + return new RoaringBitSet(roaringBitmap.clone()); + } + + + @Override + public IntStream stream() { + return roaringBitmap.stream(); + } + + @Override + public String toString() { + return roaringBitmap.toString(); + } + + @Override + public void flip(int bitIndex) { + roaringBitmap.flip(bitIndex, bitIndex + 1); + } + + @Override + public void flip(int fromIndex, int toIndex) { + roaringBitmap.flip(fromIndex, toIndex); + } + + @Override + public long[] toLongArray() { + int[] arr = roaringBitmap.toArray(); + ByteBuffer buffer = ByteBuffer.allocate(arr.length * 4); + buffer.asIntBuffer().put(arr); + return buffer.asLongBuffer().array(); + } + + @Override + public byte[] toByteArray() { + int[] arr = roaringBitmap.toArray(); + ByteBuffer buffer = ByteBuffer.allocate(arr.length * 4); + buffer.asIntBuffer().put(arr); + return buffer.array(); + } + + private static RoaringBitmap fromBitSet(BitSet bitSet) { + long[] arr = bitSet.toLongArray(); + ByteBuffer buffer = ByteBuffer.allocate(arr.length * 8); + buffer.asLongBuffer().put(arr); + return RoaringBitmap.bitmapOf(buffer.asIntBuffer().array()); + } +} From f5f8924ae9139f81757b507ee93048f20e4c3460 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Sun, 16 Jun 2024 22:21:44 +0800 Subject: [PATCH 05/22] improve code --- .../collections/ConcurrentRoaringBitSet.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java index ec7003d180669..04989a6820290 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.common.util.collections; import java.util.BitSet; From ac10e67f2d8aed5aaadee766a748b447b1f1a0d1 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Sun, 16 Jun 2024 22:26:25 +0800 Subject: [PATCH 06/22] fix checkstyle --- .../org/apache/pulsar/common/util/collections/RoaringBitSet.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java index e1b1d02b3c7be..9e916916a7d8c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java @@ -21,7 +21,6 @@ import java.nio.ByteBuffer; import java.util.BitSet; import java.util.stream.IntStream; - import org.roaringbitmap.RoaringBitmap; @SuppressWarnings("all") From e273c8ebcc76b32a5a45c1af72b21ffb0cdc26a5 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Sun, 16 Jun 2024 22:39:11 +0800 Subject: [PATCH 07/22] fix checkstyle --- .../common/util/collections/ConcurrentRoaringBitSet.java | 6 ------ .../pulsar/common/util/collections/RoaringBitSet.java | 7 +++++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java index 04989a6820290..4945e1c24c57c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java @@ -413,12 +413,6 @@ public IntStream stream() { } public boolean equals(final Object o) { - if (o == this) { - return true; - } - if (!(o instanceof ConcurrentBitSet)) { - return false; - } long stamp = rwLock.tryOptimisticRead(); boolean isEqual = super.equals(o); if (!rwLock.validate(stamp)) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java index 9e916916a7d8c..05f830a53f7d0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java @@ -196,15 +196,18 @@ public int size() { @Override public boolean equals(Object obj) { + if (obj == this) { + return true; + } if (obj instanceof RoaringBitSet) { return roaringBitmap.equals(((RoaringBitSet) obj).roaringBitmap); - } else { - return super.equals(obj); } + return false; } @Override public Object clone() { + Object ignore = super.clone(); return new RoaringBitSet(roaringBitmap.clone()); } From cc4ce2b91a59e04d5296b9a1ee08e0edfe11fc99 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Sun, 16 Jun 2024 22:56:06 +0800 Subject: [PATCH 08/22] fix checkstyle --- .../common/util/collections/ConcurrentRoaringBitSet.java | 4 ++-- .../apache/pulsar/common/util/collections/RoaringBitSet.java | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java index 4945e1c24c57c..67788ba1ef841 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java @@ -375,12 +375,12 @@ public void andNot(BitSet set) { @Override public Object clone() { long stamp = rwLock.tryOptimisticRead(); - BitSet clonedBitSet = (BitSet) super.clone(); + Object clonedBitSet = super.clone(); if (!rwLock.validate(stamp)) { // Fallback to read lock stamp = rwLock.readLock(); try { - clonedBitSet = (BitSet) super.clone(); + clonedBitSet = super.clone(); } finally { rwLock.unlockRead(stamp); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java index 05f830a53f7d0..e79b2207a964e 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java @@ -202,12 +202,15 @@ public boolean equals(Object obj) { if (obj instanceof RoaringBitSet) { return roaringBitmap.equals(((RoaringBitSet) obj).roaringBitmap); } + if (obj instanceof BitSet) { + return roaringBitmap.equals(fromBitSet((BitSet) obj)); + } return false; } @Override public Object clone() { - Object ignore = super.clone(); + super.clone(); return new RoaringBitSet(roaringBitmap.clone()); } From 8c767c7da2f44d4bf5f40d2811c332eb97df752c Mon Sep 17 00:00:00 2001 From: dao-jun Date: Sun, 16 Jun 2024 23:31:49 +0800 Subject: [PATCH 09/22] fix code --- .../ConcurrentOpenLongPairRangeSet.java | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java index da15b78a60fc9..c383300cfdf77 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java @@ -22,7 +22,6 @@ import com.google.common.collect.BoundType; import com.google.common.collect.Range; import java.util.ArrayList; -import java.util.BitSet; import java.util.List; import java.util.Map.Entry; import java.util.NavigableMap; @@ -37,13 +36,13 @@ * 
  * Usage:
  * a. This can be used if one doesn't want to create object for every new inserted {@code range}
- * b. It creates {@link BitSet} for every unique first-key of the range.
+ * b. It creates {@link RoaringBitSet} for every unique first-key of the range.
  * So, this rangeSet is not suitable for large number of unique keys.
  *
*/ public class ConcurrentOpenLongPairRangeSet> implements LongPairRangeSet { - protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); + protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); private boolean threadSafe = true; private final int bitSetSize; private final LongPairConsumer consumer; @@ -82,7 +81,7 @@ public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, lon if (lowerKey != upperKey) { // (1) set lower to last in lowerRange.getKey() if (isValid(lowerKey, lowerValue)) { - BitSet rangeBitSet = rangeBitSetMap.get(lowerKey); + RoaringBitSet rangeBitSet = rangeBitSetMap.get(lowerKey); // if lower and upper has different key/ledger then set ranges for lower-key only if // a. bitSet already exist and given value is not the last value in the bitset. // it will prevent setting up values which are not actually expected to set @@ -94,16 +93,14 @@ public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, lon } // (2) set 0th-index to upper-index in upperRange.getKey() if (isValid(upperKey, upperValue)) { - BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); - if (rangeBitSet != null) { - rangeBitSet.set(0, (int) upperValue + 1); - } + RoaringBitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); + rangeBitSet.set(0, (int) upperValue + 1); } // No-op if values are not valid eg: if lower == LongPair.earliest or upper == LongPair.latest then nothing // to set } else { long key = lowerKey; - BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(key, (k) -> createNewBitSet()); + RoaringBitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(key, (k) -> createNewBitSet()); rangeBitSet.set((int) lowerValue, (int) upperValue + 1); } updatedAfterCachedForSize = true; @@ -118,7 +115,7 @@ private boolean isValid(long key, long value) { @Override public boolean contains(long key, long value) { - BitSet rangeBitSet = rangeBitSetMap.get(key); + RoaringBitSet rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { return rangeBitSet.get(getSafeEntry(value)); } @@ -127,7 +124,7 @@ public boolean contains(long key, long value) { @Override public Range rangeContaining(long key, long value) { - BitSet rangeBitSet = rangeBitSetMap.get(key); + RoaringBitSet rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { if (!rangeBitSet.get(getSafeEntry(value))) { // if position is not part of any range then return null @@ -152,7 +149,7 @@ public boolean isEmpty() { if (rangeBitSetMap.isEmpty()) { return true; } - for (BitSet rangeBitSet : rangeBitSetMap.values()) { + for (RoaringBitSet rangeBitSet : rangeBitSetMap.values()) { if (!rangeBitSet.isEmpty()) { return false; } @@ -172,8 +169,8 @@ public Range span() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); - Entry lastSet = rangeBitSetMap.lastEntry(); + Entry firstSet = rangeBitSetMap.firstEntry(); + Entry lastSet = rangeBitSetMap.lastEntry(); int first = firstSet.getValue().nextSetBit(0); int last = lastSet.getValue().previousSetBit(lastSet.getValue().size()); return Range.openClosed(consumer.apply(firstSet.getKey(), first - 1), consumer.apply(lastSet.getKey(), last)); @@ -236,7 +233,7 @@ public Range firstRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); + Entry firstSet = rangeBitSetMap.firstEntry(); int lower = firstSet.getValue().nextSetBit(0); int upper = Math.max(lower, firstSet.getValue().nextClearBit(lower) - 1); return Range.openClosed(consumer.apply(firstSet.getKey(), lower - 1), consumer.apply(firstSet.getKey(), upper)); @@ -247,7 +244,7 @@ public Range lastRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry lastSet = rangeBitSetMap.lastEntry(); + Entry lastSet = rangeBitSetMap.lastEntry(); int upper = lastSet.getValue().previousSetBit(lastSet.getValue().size()); int lower = Math.min(lastSet.getValue().previousClearBit(upper), upper); return Range.openClosed(consumer.apply(lastSet.getKey(), lower), consumer.apply(lastSet.getKey(), upper)); @@ -255,11 +252,11 @@ public Range lastRange() { @Override public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { - NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); + NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); MutableInt v = new MutableInt(0); subMap.forEach((key, bitset) -> { if (key == lowerKey || key == upperKey) { - BitSet temp = (BitSet) bitset.clone(); + RoaringBitSet temp = (RoaringBitSet) bitset.clone(); // Trim the bitset index which < lowerValue if (key == lowerKey) { temp.clear(0, (int) Math.max(0, lowerValue)); @@ -413,7 +410,7 @@ private int getSafeEntry(long value) { return (int) Math.max(value, -1); } - private BitSet createNewBitSet() { + private RoaringBitSet createNewBitSet() { return this.threadSafe ? new ConcurrentRoaringBitSet() : new RoaringBitSet(); } From e9e6d2a632c31070cf62894efcf1d9e481074ada Mon Sep 17 00:00:00 2001 From: dao-jun Date: Sun, 16 Jun 2024 23:41:40 +0800 Subject: [PATCH 10/22] fix code --- .../apache/pulsar/common/util/collections/RoaringBitSet.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java index e79b2207a964e..8ae6d344fa71b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java @@ -191,7 +191,10 @@ public int hashCode() { @Override public int size() { - return roaringBitmap.getCardinality(); + if (roaringBitmap.isEmpty()) { + return 0; + } + return roaringBitmap.last() + 1; } @Override From 0c31da218a31a998715e5d0eca0cb4d55d5e6286 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Mon, 17 Jun 2024 14:19:28 +0800 Subject: [PATCH 11/22] fix code --- .../ConcurrentOpenLongPairRangeSet.java | 36 ++++++++++--------- .../util/collections/RoaringBitSet.java | 21 ++++------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java index c383300cfdf77..0fcc10833c836 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java @@ -22,6 +22,7 @@ import com.google.common.collect.BoundType; import com.google.common.collect.Range; import java.util.ArrayList; +import java.util.BitSet; import java.util.List; import java.util.Map.Entry; import java.util.NavigableMap; @@ -36,13 +37,13 @@ * 
  * Usage:
  * a. This can be used if one doesn't want to create object for every new inserted {@code range}
- * b. It creates {@link RoaringBitSet} for every unique first-key of the range.
+ * b. It creates {@link BitSet} for every unique first-key of the range.
  * So, this rangeSet is not suitable for large number of unique keys.
  *
*/ public class ConcurrentOpenLongPairRangeSet> implements LongPairRangeSet { - protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); + protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); private boolean threadSafe = true; private final int bitSetSize; private final LongPairConsumer consumer; @@ -81,7 +82,7 @@ public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, lon if (lowerKey != upperKey) { // (1) set lower to last in lowerRange.getKey() if (isValid(lowerKey, lowerValue)) { - RoaringBitSet rangeBitSet = rangeBitSetMap.get(lowerKey); + BitSet rangeBitSet = rangeBitSetMap.get(lowerKey); // if lower and upper has different key/ledger then set ranges for lower-key only if // a. bitSet already exist and given value is not the last value in the bitset. // it will prevent setting up values which are not actually expected to set @@ -93,14 +94,16 @@ public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, lon } // (2) set 0th-index to upper-index in upperRange.getKey() if (isValid(upperKey, upperValue)) { - RoaringBitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); - rangeBitSet.set(0, (int) upperValue + 1); + BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); + if (rangeBitSet != null) { + rangeBitSet.set(0, (int) upperValue + 1); + } } // No-op if values are not valid eg: if lower == LongPair.earliest or upper == LongPair.latest then nothing // to set } else { long key = lowerKey; - RoaringBitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(key, (k) -> createNewBitSet()); + BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(key, (k) -> createNewBitSet()); rangeBitSet.set((int) lowerValue, (int) upperValue + 1); } updatedAfterCachedForSize = true; @@ -115,7 +118,7 @@ private boolean isValid(long key, long value) { @Override public boolean contains(long key, long value) { - RoaringBitSet rangeBitSet = rangeBitSetMap.get(key); + BitSet rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { return rangeBitSet.get(getSafeEntry(value)); } @@ -124,7 +127,7 @@ public boolean contains(long key, long value) { @Override public Range rangeContaining(long key, long value) { - RoaringBitSet rangeBitSet = rangeBitSetMap.get(key); + BitSet rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { if (!rangeBitSet.get(getSafeEntry(value))) { // if position is not part of any range then return null @@ -149,7 +152,7 @@ public boolean isEmpty() { if (rangeBitSetMap.isEmpty()) { return true; } - for (RoaringBitSet rangeBitSet : rangeBitSetMap.values()) { + for (BitSet rangeBitSet : rangeBitSetMap.values()) { if (!rangeBitSet.isEmpty()) { return false; } @@ -169,8 +172,8 @@ public Range span() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); - Entry lastSet = rangeBitSetMap.lastEntry(); + Entry firstSet = rangeBitSetMap.firstEntry(); + Entry lastSet = rangeBitSetMap.lastEntry(); int first = firstSet.getValue().nextSetBit(0); int last = lastSet.getValue().previousSetBit(lastSet.getValue().size()); return Range.openClosed(consumer.apply(firstSet.getKey(), first - 1), consumer.apply(lastSet.getKey(), last)); @@ -233,7 +236,7 @@ public Range firstRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); + Entry firstSet = rangeBitSetMap.firstEntry(); int lower = firstSet.getValue().nextSetBit(0); int upper = Math.max(lower, firstSet.getValue().nextClearBit(lower) - 1); return Range.openClosed(consumer.apply(firstSet.getKey(), lower - 1), consumer.apply(firstSet.getKey(), upper)); @@ -244,7 +247,7 @@ public Range lastRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry lastSet = rangeBitSetMap.lastEntry(); + Entry lastSet = rangeBitSetMap.lastEntry(); int upper = lastSet.getValue().previousSetBit(lastSet.getValue().size()); int lower = Math.min(lastSet.getValue().previousClearBit(upper), upper); return Range.openClosed(consumer.apply(lastSet.getKey(), lower), consumer.apply(lastSet.getKey(), upper)); @@ -252,11 +255,11 @@ public Range lastRange() { @Override public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { - NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); + NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); MutableInt v = new MutableInt(0); subMap.forEach((key, bitset) -> { if (key == lowerKey || key == upperKey) { - RoaringBitSet temp = (RoaringBitSet) bitset.clone(); + BitSet temp = (BitSet) bitset.clone(); // Trim the bitset index which < lowerValue if (key == lowerKey) { temp.clear(0, (int) Math.max(0, lowerValue)); @@ -410,8 +413,7 @@ private int getSafeEntry(long value) { return (int) Math.max(value, -1); } - private RoaringBitSet createNewBitSet() { + private BitSet createNewBitSet() { return this.threadSafe ? new ConcurrentRoaringBitSet() : new RoaringBitSet(); } - } \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java index 8ae6d344fa71b..737277788483b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java @@ -18,9 +18,9 @@ */ package org.apache.pulsar.common.util.collections; -import java.nio.ByteBuffer; import java.util.BitSet; import java.util.stream.IntStream; +import org.roaringbitmap.BitSetUtil; import org.roaringbitmap.RoaringBitmap; @SuppressWarnings("all") @@ -194,7 +194,9 @@ public int size() { if (roaringBitmap.isEmpty()) { return 0; } - return roaringBitmap.last() + 1; + int lastBit = Math.max(roaringBitmap.last(), 64); + int remainder = lastBit % 64; + return remainder == 0 ? lastBit : lastBit + 64 - remainder; } @Override @@ -240,24 +242,15 @@ public void flip(int fromIndex, int toIndex) { @Override public long[] toLongArray() { - int[] arr = roaringBitmap.toArray(); - ByteBuffer buffer = ByteBuffer.allocate(arr.length * 4); - buffer.asIntBuffer().put(arr); - return buffer.asLongBuffer().array(); + return BitSetUtil.toLongArray(roaringBitmap); } @Override public byte[] toByteArray() { - int[] arr = roaringBitmap.toArray(); - ByteBuffer buffer = ByteBuffer.allocate(arr.length * 4); - buffer.asIntBuffer().put(arr); - return buffer.array(); + return BitSetUtil.toByteArray(roaringBitmap); } private static RoaringBitmap fromBitSet(BitSet bitSet) { - long[] arr = bitSet.toLongArray(); - ByteBuffer buffer = ByteBuffer.allocate(arr.length * 8); - buffer.asLongBuffer().put(arr); - return RoaringBitmap.bitmapOf(buffer.asIntBuffer().array()); + return BitSetUtil.bitmapOf(bitSet); } } From 745656744d3739907a0f0c2d4f6959a87a321a2f Mon Sep 17 00:00:00 2001 From: dao-jun Date: Mon, 17 Jun 2024 16:41:19 +0800 Subject: [PATCH 12/22] Add tests --- .../util/collections/RoaringBitSet.java | 18 +- .../util/collections/RoaringBitSetTest.java | 930 ++++++++++++++++++ 2 files changed, 937 insertions(+), 11 deletions(-) create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/RoaringBitSetTest.java diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java index 737277788483b..c70dae50e18d0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java @@ -93,10 +93,9 @@ public boolean get(int bitIndex) { @Override public BitSet get(int fromIndex, int toIndex) { - RoaringBitmap rb = roaringBitmap.clone(); - rb.flip(0, fromIndex); - rb.flip(toIndex, rb.last()); - return new RoaringBitSet(rb); + BitSet bitSet = BitSetUtil.bitsetOf(roaringBitmap); + bitSet = bitSet.get(fromIndex, toIndex); + return new RoaringBitSet(fromBitSet(bitSet)); } @Override @@ -134,13 +133,10 @@ public boolean isEmpty() { @Override public boolean intersects(BitSet set) { - RoaringBitmap bitmap = set instanceof RoaringBitSet ? ((RoaringBitSet) set).roaringBitmap : fromBitSet(set); - if (bitmap.isEmpty() || roaringBitmap.isEmpty()) { - return false; + if (set instanceof RoaringBitSet) { + return RoaringBitmap.intersects(roaringBitmap, ((RoaringBitSet) set).roaringBitmap); } - long rangeStart = bitmap.first(); - long rangeEnd = bitmap.last(); - return roaringBitmap.intersects(rangeStart, rangeEnd); + return RoaringBitmap.intersects(roaringBitmap, fromBitSet(set)); } @Override @@ -194,7 +190,7 @@ public int size() { if (roaringBitmap.isEmpty()) { return 0; } - int lastBit = Math.max(roaringBitmap.last(), 64); + int lastBit = Math.max(length(), 64); int remainder = lastBit % 64; return remainder == 0 ? lastBit : lastBit + 64 - remainder; } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/RoaringBitSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/RoaringBitSetTest.java new file mode 100644 index 0000000000000..fefc0e2839a1a --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/RoaringBitSetTest.java @@ -0,0 +1,930 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Random; +import org.testng.Assert; +import org.testng.annotations.Test; + +// From https://github.com/openjdk/jdk/blob/master/test/jdk/java/util/BitSet/BSMethods.java +@SuppressWarnings("all") +public class RoaringBitSetTest { + private static Random generator = new Random(); + + @Test + public void testSetGetClearFlip() { + int failCount = 0; + + for (int i = 0; i < 100; i++) { + RoaringBitSet testSet = new RoaringBitSet(); + HashSet history = new HashSet<>(); + + // Set a random number of bits in random places + // up to a random maximum + int nextBitToSet = 0; + int numberOfSetBits = generator.nextInt(100) + 1; + int highestPossibleSetBit = generator.nextInt(1000) + 1; + for (int x = 0; x < numberOfSetBits; x++) { + nextBitToSet = generator.nextInt(highestPossibleSetBit); + history.add(Integer.valueOf(nextBitToSet)); + testSet.set(nextBitToSet); + } + + // Make sure each bit is set appropriately + for (int x = 0; x < highestPossibleSetBit; x++) { + if (testSet.get(x) != history.contains(Integer.valueOf(x))) + failCount++; + } + + // Clear the bits + Iterator setBitIterator = history.iterator(); + while (setBitIterator.hasNext()) { + Integer setBit = setBitIterator.next(); + testSet.clear(setBit.intValue()); + } + + // Verify they were cleared + for (int x = 0; x < highestPossibleSetBit; x++) + if (testSet.get(x)) + failCount++; + if (testSet.length() != 0) + failCount++; + + // Set them with set(int, boolean) + setBitIterator = history.iterator(); + while (setBitIterator.hasNext()) { + Integer setBit = setBitIterator.next(); + testSet.set(setBit.intValue(), true); + } + + // Make sure each bit is set appropriately + for (int x = 0; x < highestPossibleSetBit; x++) { + if (testSet.get(x) != history.contains(Integer.valueOf(x))) + failCount++; + } + + // Clear them with set(int, boolean) + setBitIterator = history.iterator(); + while (setBitIterator.hasNext()) { + Integer setBit = setBitIterator.next(); + testSet.set(setBit.intValue(), false); + } + + // Verify they were cleared + for (int x = 0; x < highestPossibleSetBit; x++) + if (testSet.get(x)) + failCount++; + if (testSet.length() != 0) + failCount++; + + // Flip them on + setBitIterator = history.iterator(); + while (setBitIterator.hasNext()) { + Integer setBit = setBitIterator.next(); + testSet.flip(setBit.intValue()); + } + + // Verify they were flipped + for (int x = 0; x < highestPossibleSetBit; x++) { + if (testSet.get(x) != history.contains(Integer.valueOf(x))) + failCount++; + } + + // Flip them off + setBitIterator = history.iterator(); + while (setBitIterator.hasNext()) { + Integer setBit = setBitIterator.next(); + testSet.flip(setBit.intValue()); + } + + // Verify they were flipped + for (int x = 0; x < highestPossibleSetBit; x++) + if (testSet.get(x)) + failCount++; + if (testSet.length() != 0) + failCount++; + + checkSanity(testSet); + } + + Assert.assertEquals(failCount, 0); + } + + @Test + public void testClear() { + int failCount = 0; + + for (int i = 0; i < 1000; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + + // Make a fairly random bitset + int numberOfSetBits = generator.nextInt(100) + 1; + int highestPossibleSetBit = generator.nextInt(1000) + 1; + + for (int x = 0; x < numberOfSetBits; x++) + b1.set(generator.nextInt(highestPossibleSetBit)); + + RoaringBitSet b2 = (RoaringBitSet) b1.clone(); + + // Clear out a random range + int rangeStart = generator.nextInt(100); + int rangeEnd = rangeStart + generator.nextInt(100); + + // Use the clear(int, int) call on b1 + b1.clear(rangeStart, rangeEnd); + + // Use a loop on b2 + for (int x = rangeStart; x < rangeEnd; x++) + b2.clear(x); + + // Verify their equality + if (!b1.equals(b2)) { + failCount++; + } + checkEquality(b1, b2); + } + + Assert.assertEquals(failCount, 0); + } + + @Test + public void testFlip() { + int failCount = 0; + + for (int i = 0; i < 1000; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + + // Make a fairly random bitset + int numberOfSetBits = generator.nextInt(100) + 1; + int highestPossibleSetBit = generator.nextInt(1000) + 1; + + for (int x = 0; x < numberOfSetBits; x++) + b1.set(generator.nextInt(highestPossibleSetBit)); + + RoaringBitSet b2 = (RoaringBitSet) b1.clone(); + + // Flip a random range + int rangeStart = generator.nextInt(100); + int rangeEnd = rangeStart + generator.nextInt(100); + + // Use the flip(int, int) call on b1 + b1.flip(rangeStart, rangeEnd); + + // Use a loop on b2 + for (int x = rangeStart; x < rangeEnd; x++) + b2.flip(x); + + // Verify their equality + if (!b1.equals(b2)) + failCount++; + checkEquality(b1, b2); + } + + Assert.assertEquals(failCount, 0); + } + + @Test + public void testSet() { + int failCount = 0; + + // Test set(int, int) + for (int i = 0; i < 1000; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + + // Make a fairly random bitset + int numberOfSetBits = generator.nextInt(100) + 1; + int highestPossibleSetBit = generator.nextInt(1000) + 1; + + for (int x = 0; x < numberOfSetBits; x++) + b1.set(generator.nextInt(highestPossibleSetBit)); + + RoaringBitSet b2 = (RoaringBitSet) b1.clone(); + + // Set a random range + int rangeStart = generator.nextInt(100); + int rangeEnd = rangeStart + generator.nextInt(100); + + // Use the set(int, int) call on b1 + b1.set(rangeStart, rangeEnd); + + // Use a loop on b2 + for (int x = rangeStart; x < rangeEnd; x++) + b2.set(x); + + // Verify their equality + if (!b1.equals(b2)) { + failCount++; + } + checkEquality(b1, b2); + } + + // Test set(int, int, boolean) + for (int i = 0; i < 100; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + + // Make a fairly random bitset + int numberOfSetBits = generator.nextInt(100) + 1; + int highestPossibleSetBit = generator.nextInt(1000) + 1; + + for (int x = 0; x < numberOfSetBits; x++) + b1.set(generator.nextInt(highestPossibleSetBit)); + + RoaringBitSet b2 = (RoaringBitSet) b1.clone(); + boolean setOrClear = generator.nextBoolean(); + + // Set a random range + int rangeStart = generator.nextInt(100); + int rangeEnd = rangeStart + generator.nextInt(100); + + // Use the set(int, int, boolean) call on b1 + b1.set(rangeStart, rangeEnd, setOrClear); + + // Use a loop on b2 + for (int x = rangeStart; x < rangeEnd; x++) + b2.set(x, setOrClear); + + // Verify their equality + if (!b1.equals(b2)) { + failCount++; + } + checkEquality(b1, b2); + } + + Assert.assertEquals(failCount, 0); + } + + @Test + public void testGet() { + int failCount = 0; + + for (int i = 0; i < 1000; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + + // Make a fairly random bitset + int numberOfSetBits = generator.nextInt(100) + 1; + int highestPossibleSetBit = generator.nextInt(1000) + 1; + + for (int x = 0; x < numberOfSetBits; x++) + b1.set(generator.nextInt(highestPossibleSetBit)); + + // Get a new set from a random range + int rangeStart = generator.nextInt(100); + int rangeEnd = rangeStart + generator.nextInt(100); + + RoaringBitSet b2 = (RoaringBitSet) b1.get(rangeStart, rangeEnd); + + RoaringBitSet b3 = new RoaringBitSet(); + for (int x = rangeStart; x < rangeEnd; x++) + b3.set(x - rangeStart, b1.get(x)); + + // Verify their equality + if (!b2.equals(b3)) { + failCount++; + } + checkEquality(b2, b3); + } + + Assert.assertEquals(failCount, 0); + } + + @Test + public void testAndNot() { + int failCount = 0; + + for (int i = 0; i < 100; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + RoaringBitSet b2 = new RoaringBitSet(); + + // Set some random bits in first set and remember them + int nextBitToSet = 0; + for (int x = 0; x < 10; x++) + b1.set(generator.nextInt(255)); + + // Set some random bits in second set and remember them + for (int x = 10; x < 20; x++) + b2.set(generator.nextInt(255)); + + // andNot the sets together + RoaringBitSet b3 = (RoaringBitSet) b1.clone(); + b3.andNot(b2); + + // Examine each bit of b3 for errors + for (int x = 0; x < 256; x++) { + boolean bit1 = b1.get(x); + boolean bit2 = b2.get(x); + boolean bit3 = b3.get(x); + if (!(bit3 == (bit1 & (!bit2)))) + failCount++; + } + checkSanity(b1, b2, b3); + } + Assert.assertEquals(failCount, 0); + } + + @Test + public void testAnd() { + int failCount = 0; + + for (int i = 0; i < 100; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + RoaringBitSet b2 = new RoaringBitSet(); + + // Set some random bits in first set and remember them + int nextBitToSet = 0; + for (int x = 0; x < 10; x++) + b1.set(generator.nextInt(255)); + + // Set more random bits in second set and remember them + for (int x = 10; x < 20; x++) + b2.set(generator.nextInt(255)); + + // And the sets together + RoaringBitSet b3 = (RoaringBitSet) b1.clone(); + b3.and(b2); + + // Examine each bit of b3 for errors + for (int x = 0; x < 256; x++) { + boolean bit1 = b1.get(x); + boolean bit2 = b2.get(x); + boolean bit3 = b3.get(x); + if (!(bit3 == (bit1 & bit2))) + failCount++; + } + checkSanity(b1, b2, b3); + } + + // `and' that happens to clear the last word + RoaringBitSet b4 = makeSet(2, 127); + b4.and(makeSet(2, 64)); + checkSanity(b4); + if (!(b4.equals(makeSet(2)))) + failCount++; + + Assert.assertEquals(failCount, 0); + } + + @Test + public void testOr() { + int failCount = 0; + + for (int i = 0; i < 100; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + RoaringBitSet b2 = new RoaringBitSet(); + int[] history = new int[20]; + + // Set some random bits in first set and remember them + int nextBitToSet = 0; + for (int x = 0; x < 10; x++) { + nextBitToSet = generator.nextInt(255); + history[x] = nextBitToSet; + b1.set(nextBitToSet); + } + + // Set more random bits in second set and remember them + for (int x = 10; x < 20; x++) { + nextBitToSet = generator.nextInt(255); + history[x] = nextBitToSet; + b2.set(nextBitToSet); + } + + // Or the sets together + RoaringBitSet b3 = (RoaringBitSet) b1.clone(); + b3.or(b2); + + // Verify the set bits of b3 from the history + int historyIndex = 0; + for (int x = 0; x < 20; x++) { + if (!b3.get(history[x])) + failCount++; + } + + // Examine each bit of b3 for errors + for (int x = 0; x < 256; x++) { + boolean bit1 = b1.get(x); + boolean bit2 = b2.get(x); + boolean bit3 = b3.get(x); + if (!(bit3 == (bit1 | bit2))) + failCount++; + } + checkSanity(b1, b2, b3); + } + Assert.assertEquals(failCount, 0); + } + + @Test + public void testXor() { + int failCount = 0; + + for (int i = 0; i < 100; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + RoaringBitSet b2 = new RoaringBitSet(); + + // Set some random bits in first set and remember them + int nextBitToSet = 0; + for (int x = 0; x < 10; x++) + b1.set(generator.nextInt(255)); + + // Set more random bits in second set and remember them + for (int x = 10; x < 20; x++) + b2.set(generator.nextInt(255)); + + // Xor the sets together + RoaringBitSet b3 = (RoaringBitSet) b1.clone(); + b3.xor(b2); + + // Examine each bit of b3 for errors + for (int x = 0; x < 256; x++) { + boolean bit1 = b1.get(x); + boolean bit2 = b2.get(x); + boolean bit3 = b3.get(x); + if (!(bit3 == (bit1 ^ bit2))) + failCount++; + } + checkSanity(b1, b2, b3); + b3.xor(b3); + checkEmpty(b3); + } + + // xor that happens to clear the last word + RoaringBitSet b4 = makeSet(2, 64, 127); + b4.xor(makeSet(64, 127)); + checkSanity(b4); + if (!(b4.equals(makeSet(2)))) + failCount++; + + Assert.assertEquals(failCount, 0); + } + + @Test + public void testLength() { + int failCount = 0; + + // Test length after set + for (int i = 0; i < 100; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + int highestSetBit = 0; + + for (int x = 0; x < 100; x++) { + int nextBitToSet = generator.nextInt(255); + if (nextBitToSet > highestSetBit) + highestSetBit = nextBitToSet; + b1.set(nextBitToSet); + if (b1.length() != highestSetBit + 1) + failCount++; + } + checkSanity(b1); + } + + // Test length after flip + for (int i = 0; i < 100; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + for (int x = 0; x < 100; x++) { + // Flip a random range twice + int rangeStart = generator.nextInt(100); + int rangeEnd = rangeStart + generator.nextInt(100); + b1.flip(rangeStart); + b1.flip(rangeStart); + if (b1.length() != 0) + failCount++; + b1.flip(rangeStart, rangeEnd); + b1.flip(rangeStart, rangeEnd); + if (b1.length() != 0) + failCount++; + } + checkSanity(b1); + } + + // Test length after or + for (int i = 0; i < 100; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + RoaringBitSet b2 = new RoaringBitSet(); + int bit1 = generator.nextInt(100); + int bit2 = generator.nextInt(100); + int highestSetBit = (bit1 > bit2) ? bit1 : bit2; + b1.set(bit1); + b2.set(bit2); + b1.or(b2); + if (b1.length() != highestSetBit + 1) + failCount++; + checkSanity(b1, b2); + } + Assert.assertEquals(failCount, 0); + } + + @Test + public void testEquals() { + int failCount = 0; + + for (int i = 0; i < 100; i++) { + // Create BitSets of different sizes + RoaringBitSet b1 = new RoaringBitSet(); + RoaringBitSet b2 = new RoaringBitSet(); + + // Set some random bits + int nextBitToSet = 0; + for (int x = 0; x < 10; x++) { + nextBitToSet += generator.nextInt(50) + 1; + b1.set(nextBitToSet); + b2.set(nextBitToSet); + } + + // Verify their equality despite different storage sizes + if (!b1.equals(b2)) + failCount++; + checkEquality(b1, b2); + } + Assert.assertEquals(failCount, 0); + } + + @Test + public void testNextSetBit() { + int failCount = 0; + + for (int i = 0; i < 100; i++) { + int numberOfSetBits = generator.nextInt(100) + 1; + RoaringBitSet testSet = new RoaringBitSet(); + int[] history = new int[numberOfSetBits]; + + // Set some random bits and remember them + int nextBitToSet = 0; + for (int x = 0; x < numberOfSetBits; x++) { + nextBitToSet += generator.nextInt(30) + 1; + history[x] = nextBitToSet; + testSet.set(nextBitToSet); + } + + // Verify their retrieval using nextSetBit() + int historyIndex = 0; + for (int x = testSet.nextSetBit(0); x >= 0; x = testSet.nextSetBit(x + 1)) { + if (x != history[historyIndex++]) + failCount++; + } + + checkSanity(testSet); + } + + Assert.assertEquals(failCount, 0); + } + + @Test + public void testNextClearBit() { + int failCount = 0; + + for (int i = 0; i < 1000; i++) { + RoaringBitSet b = new RoaringBitSet(); + int[] history = new int[10]; + + // Set all the bits + for (int x = 0; x < 256; x++) + b.set(x); + + // Clear some random bits and remember them + int nextBitToClear = 0; + for (int x = 0; x < 10; x++) { + nextBitToClear += generator.nextInt(24) + 1; + history[x] = nextBitToClear; + b.clear(nextBitToClear); + } + + // Verify their retrieval using nextClearBit() + int historyIndex = 0; + for (int x = b.nextClearBit(0); x < 256; x = b.nextClearBit(x + 1)) { + if (x != history[historyIndex++]) + failCount++; + } + + checkSanity(b); + } + + // regression test for 4350178 + RoaringBitSet bs = new RoaringBitSet(); + if (bs.nextClearBit(0) != 0) + failCount++; + for (int i = 0; i < 64; i++) { + bs.set(i); + if (bs.nextClearBit(0) != i + 1) + failCount++; + } + + checkSanity(bs); + Assert.assertEquals(failCount, 0); + } + + @Test + public void testIntersects() { + int failCount = 0; + + for (int i = 0; i < 100; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + RoaringBitSet b2 = new RoaringBitSet(); + + // Set some random bits in first set + int nextBitToSet = 0; + for (int x = 0; x < 30; x++) { + nextBitToSet = generator.nextInt(255); + b1.set(nextBitToSet); + } + + // Set more random bits in second set + for (int x = 0; x < 30; x++) { + nextBitToSet = generator.nextInt(255); + b2.set(nextBitToSet); + } + + // Make sure they intersect + nextBitToSet = generator.nextInt(255); + b1.set(nextBitToSet); + b2.set(nextBitToSet); + + if (!b1.intersects(b2)) + failCount++; + + // Remove the common set bits + b1.andNot(b2); + + // Make sure they don't intersect + if (b1.intersects(b2)) + failCount++; + + checkSanity(b1, b2); + } + + Assert.assertEquals(failCount, 0); + } + + @Test + public void testCardinality() { + int failCount = 0; + + for (int i = 0; i < 100; i++) { + RoaringBitSet b1 = new RoaringBitSet(); + + // Set a random number of increasing bits + int nextBitToSet = 0; + int iterations = generator.nextInt(20) + 1; + for (int x = 0; x < iterations; x++) { + nextBitToSet += generator.nextInt(20) + 1; + b1.set(nextBitToSet); + } + + if (b1.cardinality() != iterations) { + failCount++; + } + + checkSanity(b1); + } + + Assert.assertEquals(failCount, 0); + } + + @Test + public void testEmpty() { + int failCount = 0; + + RoaringBitSet b1 = new RoaringBitSet(); + if (!b1.isEmpty()) + failCount++; + + int nextBitToSet = 0; + int numberOfSetBits = generator.nextInt(100) + 1; + int highestPossibleSetBit = generator.nextInt(1000) + 1; + for (int x = 0; x < numberOfSetBits; x++) { + nextBitToSet = generator.nextInt(highestPossibleSetBit); + b1.set(nextBitToSet); + if (b1.isEmpty()) + failCount++; + b1.clear(nextBitToSet); + if (!b1.isEmpty()) + failCount++; + } + + Assert.assertEquals(failCount, 0); + } + + @Test + public void testEmpty2() { + { + RoaringBitSet t = new RoaringBitSet(); + t.set(100); + t.clear(3, 600); + checkEmpty(t); + } + checkEmpty(new RoaringBitSet()); + + RoaringBitSet s = new RoaringBitSet(); + checkEmpty(s); + s.clear(92); + checkEmpty(s); + s.clear(127, 127); + checkEmpty(s); + s.set(127, 127); + checkEmpty(s); + s.set(128, 128); + checkEmpty(s); + RoaringBitSet empty = new RoaringBitSet(); + { + RoaringBitSet t = new RoaringBitSet(); + t.and(empty); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + t.or(empty); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + t.xor(empty); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + t.andNot(empty); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + t.and(t); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + t.or(t); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + t.xor(t); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + t.andNot(t); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + t.and(makeSet(1)); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + t.and(makeSet(127)); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + t.and(makeSet(128)); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + t.flip(7); + t.flip(7); + checkEmpty(t); + } + { + RoaringBitSet t = new RoaringBitSet(); + checkEmpty((RoaringBitSet) t.get(200, 300)); + } + { + RoaringBitSet t = makeSet(2, 5); + Assert.assertEquals(makeSet(0, 3), t.get(2, 6)); + } + } + + @Test + public void testLogicalIdentities() { + int failCount = 0; + + // Verify that (!b1)|(!b2) == !(b1&b2) + for (int i = 0; i < 50; i++) { + // Construct two fairly random bitsets + RoaringBitSet b1 = new RoaringBitSet(); + RoaringBitSet b2 = new RoaringBitSet(); + + int numberOfSetBits = generator.nextInt(100) + 1; + int highestPossibleSetBit = generator.nextInt(1000) + 1; + + for (int x = 0; x < numberOfSetBits; x++) { + b1.set(generator.nextInt(highestPossibleSetBit)); + b2.set(generator.nextInt(highestPossibleSetBit)); + } + + RoaringBitSet b3 = (RoaringBitSet) b1.clone(); + RoaringBitSet b4 = (RoaringBitSet) b2.clone(); + + for (int x = 0; x < highestPossibleSetBit; x++) { + b1.flip(x); + b2.flip(x); + } + b1.or(b2); + b3.and(b4); + for (int x = 0; x < highestPossibleSetBit; x++) + b3.flip(x); + if (!b1.equals(b3)) + failCount++; + checkSanity(b1, b2, b3, b4); + } + + // Verify that (b1&(!b2)|(b2&(!b1) == b1^b2 + for (int i = 0; i < 50; i++) { + // Construct two fairly random bitsets + RoaringBitSet b1 = new RoaringBitSet(); + RoaringBitSet b2 = new RoaringBitSet(); + + int numberOfSetBits = generator.nextInt(100) + 1; + int highestPossibleSetBit = generator.nextInt(1000) + 1; + + for (int x = 0; x < numberOfSetBits; x++) { + b1.set(generator.nextInt(highestPossibleSetBit)); + b2.set(generator.nextInt(highestPossibleSetBit)); + } + + RoaringBitSet b3 = (RoaringBitSet) b1.clone(); + RoaringBitSet b4 = (RoaringBitSet) b2.clone(); + RoaringBitSet b5 = (RoaringBitSet) b1.clone(); + RoaringBitSet b6 = (RoaringBitSet) b2.clone(); + + for (int x = 0; x < highestPossibleSetBit; x++) + b2.flip(x); + b1.and(b2); + for (int x = 0; x < highestPossibleSetBit; x++) + b3.flip(x); + b3.and(b4); + b1.or(b3); + b5.xor(b6); + if (!b1.equals(b5)) + failCount++; + checkSanity(b1, b2, b3, b4, b5, b6); + } + Assert.assertEquals(failCount, 0); + } + + private static void checkSanity(RoaringBitSet... sets) { + for (RoaringBitSet s : sets) { + int len = s.length(); + int cardinality1 = s.cardinality(); + int cardinality2 = 0; + for (int i = s.nextSetBit(0); i >= 0; i = s.nextSetBit(i + 1)) { + Assert.assertTrue(s.get(i)); + cardinality2++; + } + Assert.assertEquals(s.nextSetBit(len), -1); + Assert.assertEquals(len, s.nextClearBit(len)); + Assert.assertEquals((len == 0), s.isEmpty()); + Assert.assertEquals(cardinality2, cardinality1); + Assert.assertTrue(len <= s.size()); + Assert.assertTrue(len >= 0); + Assert.assertTrue(cardinality1 >= 0); + } + } + + private static void checkEquality(RoaringBitSet s, RoaringBitSet t) { + checkSanity(s, t); + Assert.assertEquals(t, s); + Assert.assertEquals(t.toString(), s.toString()); + Assert.assertEquals(s.length(), t.length()); + Assert.assertEquals(s.cardinality(), t.cardinality()); + } + + private static RoaringBitSet makeSet(int... elts) { + RoaringBitSet s = new RoaringBitSet(); + for (int elt : elts) + s.set(elt); + return s; + } + + private static void checkEmpty(RoaringBitSet s) { + Assert.assertTrue(s.isEmpty()); + Assert.assertEquals(s.length(), 0); + Assert.assertEquals(s.cardinality(), 0); + Assert.assertEquals(s, new RoaringBitSet()); + Assert.assertEquals(s.nextSetBit(0), -1); + Assert.assertEquals(s.nextSetBit(127), -1); + Assert.assertEquals(s.nextSetBit(128), -1); + Assert.assertEquals(s.nextClearBit(0), 0); + Assert.assertEquals(s.nextClearBit(127), 127); + Assert.assertEquals(s.nextClearBit(128), 128); + Assert.assertEquals(s.toString(), "{}"); + Assert.assertFalse(s.get(0)); + } +} From 228e1619a35f9bf4892346b5ab518771c61126e6 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Mon, 17 Jun 2024 18:26:24 +0800 Subject: [PATCH 13/22] fix code --- .../apache/pulsar/common/util/collections/RoaringBitSet.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java index c70dae50e18d0..f90f408fd87ae 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java @@ -203,9 +203,6 @@ public boolean equals(Object obj) { if (obj instanceof RoaringBitSet) { return roaringBitmap.equals(((RoaringBitSet) obj).roaringBitmap); } - if (obj instanceof BitSet) { - return roaringBitmap.equals(fromBitSet((BitSet) obj)); - } return false; } From 1de4042f95e6be375c03b28d1f9bfe9010728c30 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Mon, 17 Jun 2024 18:46:00 +0800 Subject: [PATCH 14/22] Address review comment. --- .../ConcurrentOpenLongPairRangeSet.java | 31 ++++---- .../collections/ConcurrentRoaringBitSet.java | 14 ++-- .../util/collections/RoaringBitSet.java | 77 ++++--------------- 3 files changed, 37 insertions(+), 85 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java index 0fcc10833c836..7a6518aab07cf 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java @@ -22,7 +22,6 @@ import com.google.common.collect.BoundType; import com.google.common.collect.Range; import java.util.ArrayList; -import java.util.BitSet; import java.util.List; import java.util.Map.Entry; import java.util.NavigableMap; @@ -37,13 +36,13 @@ * 
  * Usage:
  * a. This can be used if one doesn't want to create object for every new inserted {@code range}
- * b. It creates {@link BitSet} for every unique first-key of the range.
+ * b. It creates {@link RoaringBitSet} for every unique first-key of the range.
  * So, this rangeSet is not suitable for large number of unique keys.
  *
*/ public class ConcurrentOpenLongPairRangeSet> implements LongPairRangeSet { - protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); + protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); private boolean threadSafe = true; private final int bitSetSize; private final LongPairConsumer consumer; @@ -82,7 +81,7 @@ public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, lon if (lowerKey != upperKey) { // (1) set lower to last in lowerRange.getKey() if (isValid(lowerKey, lowerValue)) { - BitSet rangeBitSet = rangeBitSetMap.get(lowerKey); + RoaringBitSet rangeBitSet = rangeBitSetMap.get(lowerKey); // if lower and upper has different key/ledger then set ranges for lower-key only if // a. bitSet already exist and given value is not the last value in the bitset. // it will prevent setting up values which are not actually expected to set @@ -94,7 +93,7 @@ public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, lon } // (2) set 0th-index to upper-index in upperRange.getKey() if (isValid(upperKey, upperValue)) { - BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); + RoaringBitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); if (rangeBitSet != null) { rangeBitSet.set(0, (int) upperValue + 1); } @@ -103,7 +102,7 @@ public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, lon // to set } else { long key = lowerKey; - BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(key, (k) -> createNewBitSet()); + RoaringBitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(key, (k) -> createNewBitSet()); rangeBitSet.set((int) lowerValue, (int) upperValue + 1); } updatedAfterCachedForSize = true; @@ -118,7 +117,7 @@ private boolean isValid(long key, long value) { @Override public boolean contains(long key, long value) { - BitSet rangeBitSet = rangeBitSetMap.get(key); + RoaringBitSet rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { return rangeBitSet.get(getSafeEntry(value)); } @@ -127,7 +126,7 @@ public boolean contains(long key, long value) { @Override public Range rangeContaining(long key, long value) { - BitSet rangeBitSet = rangeBitSetMap.get(key); + RoaringBitSet rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { if (!rangeBitSet.get(getSafeEntry(value))) { // if position is not part of any range then return null @@ -152,7 +151,7 @@ public boolean isEmpty() { if (rangeBitSetMap.isEmpty()) { return true; } - for (BitSet rangeBitSet : rangeBitSetMap.values()) { + for (RoaringBitSet rangeBitSet : rangeBitSetMap.values()) { if (!rangeBitSet.isEmpty()) { return false; } @@ -172,8 +171,8 @@ public Range span() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); - Entry lastSet = rangeBitSetMap.lastEntry(); + Entry firstSet = rangeBitSetMap.firstEntry(); + Entry lastSet = rangeBitSetMap.lastEntry(); int first = firstSet.getValue().nextSetBit(0); int last = lastSet.getValue().previousSetBit(lastSet.getValue().size()); return Range.openClosed(consumer.apply(firstSet.getKey(), first - 1), consumer.apply(lastSet.getKey(), last)); @@ -236,7 +235,7 @@ public Range firstRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); + Entry firstSet = rangeBitSetMap.firstEntry(); int lower = firstSet.getValue().nextSetBit(0); int upper = Math.max(lower, firstSet.getValue().nextClearBit(lower) - 1); return Range.openClosed(consumer.apply(firstSet.getKey(), lower - 1), consumer.apply(firstSet.getKey(), upper)); @@ -247,7 +246,7 @@ public Range lastRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry lastSet = rangeBitSetMap.lastEntry(); + Entry lastSet = rangeBitSetMap.lastEntry(); int upper = lastSet.getValue().previousSetBit(lastSet.getValue().size()); int lower = Math.min(lastSet.getValue().previousClearBit(upper), upper); return Range.openClosed(consumer.apply(lastSet.getKey(), lower), consumer.apply(lastSet.getKey(), upper)); @@ -255,11 +254,11 @@ public Range lastRange() { @Override public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { - NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); + NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); MutableInt v = new MutableInt(0); subMap.forEach((key, bitset) -> { if (key == lowerKey || key == upperKey) { - BitSet temp = (BitSet) bitset.clone(); + RoaringBitSet temp = (RoaringBitSet) bitset.clone(); // Trim the bitset index which < lowerValue if (key == lowerKey) { temp.clear(0, (int) Math.max(0, lowerValue)); @@ -413,7 +412,7 @@ private int getSafeEntry(long value) { return (int) Math.max(value, -1); } - private BitSet createNewBitSet() { + private RoaringBitSet createNewBitSet() { return this.threadSafe ? new ConcurrentRoaringBitSet() : new RoaringBitSet(); } } \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java index 67788ba1ef841..8ebd482ec3022 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java @@ -279,9 +279,9 @@ public void set(int fromIndex, int toIndex, boolean value) { } @Override - public BitSet get(int fromIndex, int toIndex) { + public RoaringBitSet get(int fromIndex, int toIndex) { long stamp = rwLock.tryOptimisticRead(); - BitSet bitSet = super.get(fromIndex, toIndex); + RoaringBitSet bitSet = super.get(fromIndex, toIndex); if (!rwLock.validate(stamp)) { // Fallback to read lock stamp = rwLock.readLock(); @@ -317,7 +317,7 @@ public int safeLength() { } @Override - public boolean intersects(BitSet set) { + public boolean intersects(RoaringBitSet set) { long stamp = rwLock.writeLock(); try { return super.intersects(set); @@ -327,7 +327,7 @@ public boolean intersects(BitSet set) { } @Override - public void and(BitSet set) { + public void and(RoaringBitSet set) { long stamp = rwLock.writeLock(); try { super.and(set); @@ -337,7 +337,7 @@ public void and(BitSet set) { } @Override - public void or(BitSet set) { + public void or(RoaringBitSet set) { long stamp = rwLock.writeLock(); try { super.or(set); @@ -347,7 +347,7 @@ public void or(BitSet set) { } @Override - public void xor(BitSet set) { + public void xor(RoaringBitSet set) { long stamp = rwLock.writeLock(); try { super.xor(set); @@ -357,7 +357,7 @@ public void xor(BitSet set) { } @Override - public void andNot(BitSet set) { + public void andNot(RoaringBitSet set) { long stamp = rwLock.writeLock(); try { super.andNot(set); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java index f90f408fd87ae..adc4085d4a279 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java @@ -23,19 +23,20 @@ import org.roaringbitmap.BitSetUtil; import org.roaringbitmap.RoaringBitmap; +/** + * A RoaringBitmap-backed BitSet implementation. + */ @SuppressWarnings("all") -public class RoaringBitSet extends BitSet { +public class RoaringBitSet { private static final long serialVersionUID = 1L; private RoaringBitmap roaringBitmap; public RoaringBitSet() { - super(0); roaringBitmap = new RoaringBitmap(); } private RoaringBitSet(RoaringBitmap roaringBitmap) { - super(0); this.roaringBitmap = roaringBitmap; } @@ -43,12 +44,10 @@ public RoaringBitmap getRoaringBitmap() { return roaringBitmap; } - @Override public void set(int bitIndex) { roaringBitmap.add(bitIndex); } - @Override public void set(int bitIndex, boolean value) { if (value) { roaringBitmap.add(bitIndex); @@ -57,12 +56,10 @@ public void set(int bitIndex, boolean value) { } } - @Override public void set(int fromIndex, int toIndex) { roaringBitmap.add(fromIndex, toIndex); } - @Override public void set(int fromIndex, int toIndex, boolean value) { if (value) { roaringBitmap.add(fromIndex, toIndex); @@ -71,54 +68,44 @@ public void set(int fromIndex, int toIndex, boolean value) { } } - @Override public void clear(int bitIndex) { roaringBitmap.remove(bitIndex); } - @Override public void clear(int fromIndex, int toIndex) { roaringBitmap.remove(fromIndex, toIndex); } - @Override public void clear() { roaringBitmap.clear(); } - @Override public boolean get(int bitIndex) { return roaringBitmap.contains(bitIndex); } - @Override - public BitSet get(int fromIndex, int toIndex) { + public RoaringBitSet get(int fromIndex, int toIndex) { BitSet bitSet = BitSetUtil.bitsetOf(roaringBitmap); bitSet = bitSet.get(fromIndex, toIndex); return new RoaringBitSet(fromBitSet(bitSet)); } - @Override public int nextSetBit(int fromIndex) { return (int) roaringBitmap.nextValue(fromIndex); } - @Override public int nextClearBit(int fromIndex) { return (int) roaringBitmap.nextAbsentValue(fromIndex); } - @Override public int previousSetBit(int fromIndex) { return (int) roaringBitmap.previousValue(fromIndex); } - @Override public int previousClearBit(int fromIndex) { return (int) roaringBitmap.previousAbsentValue(fromIndex); } - @Override public int length() { if (roaringBitmap.isEmpty()) { return 0; @@ -126,58 +113,32 @@ public int length() { return roaringBitmap.last() + 1; } - @Override public boolean isEmpty() { return roaringBitmap.isEmpty(); } - @Override - public boolean intersects(BitSet set) { - if (set instanceof RoaringBitSet) { - return RoaringBitmap.intersects(roaringBitmap, ((RoaringBitSet) set).roaringBitmap); - } - return RoaringBitmap.intersects(roaringBitmap, fromBitSet(set)); + public boolean intersects(RoaringBitSet set) { + return RoaringBitmap.intersects(roaringBitmap, ((RoaringBitSet) set).roaringBitmap); } - @Override public int cardinality() { return roaringBitmap.getCardinality(); } - @Override - public void and(BitSet set) { - if (set instanceof RoaringBitSet) { - roaringBitmap.and(((RoaringBitSet) set).roaringBitmap); - } else { - roaringBitmap.and(fromBitSet(set)); - } + public void and(RoaringBitSet set) { + roaringBitmap.and(((RoaringBitSet) set).roaringBitmap); } - @Override - public void or(BitSet set) { - if (set instanceof RoaringBitSet) { - roaringBitmap.or(((RoaringBitSet) set).roaringBitmap); - } else { - roaringBitmap.or(fromBitSet(set)); - } + public void or(RoaringBitSet set) { + roaringBitmap.or(((RoaringBitSet) set).roaringBitmap); } - @Override - public void xor(BitSet set) { - if (set instanceof RoaringBitSet) { - roaringBitmap.xor(((RoaringBitSet) set).roaringBitmap); - } else { - roaringBitmap.xor(fromBitSet(set)); - } + public void xor(RoaringBitSet set) { + roaringBitmap.xor(((RoaringBitSet) set).roaringBitmap); } - @Override - public void andNot(BitSet set) { - if (set instanceof RoaringBitSet) { - roaringBitmap.andNot(((RoaringBitSet) set).roaringBitmap); - } else { - roaringBitmap.andNot(fromBitSet(set)); - } + public void andNot(RoaringBitSet set) { + roaringBitmap.andNot(((RoaringBitSet) set).roaringBitmap); } @Override @@ -185,7 +146,6 @@ public int hashCode() { return roaringBitmap.hashCode(); } - @Override public int size() { if (roaringBitmap.isEmpty()) { return 0; @@ -208,12 +168,9 @@ public boolean equals(Object obj) { @Override public Object clone() { - super.clone(); return new RoaringBitSet(roaringBitmap.clone()); } - - @Override public IntStream stream() { return roaringBitmap.stream(); } @@ -223,22 +180,18 @@ public String toString() { return roaringBitmap.toString(); } - @Override public void flip(int bitIndex) { roaringBitmap.flip(bitIndex, bitIndex + 1); } - @Override public void flip(int fromIndex, int toIndex) { roaringBitmap.flip(fromIndex, toIndex); } - @Override public long[] toLongArray() { return BitSetUtil.toLongArray(roaringBitmap); } - @Override public byte[] toByteArray() { return BitSetUtil.toByteArray(roaringBitmap); } From e555587746065687b617afa71cb612701c10f3fa Mon Sep 17 00:00:00 2001 From: dao-jun Date: Mon, 17 Jun 2024 18:53:36 +0800 Subject: [PATCH 15/22] fix checkstyle --- .../pulsar/common/util/collections/ConcurrentRoaringBitSet.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java index 8ebd482ec3022..4848e78fcc905 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.common.util.collections; -import java.util.BitSet; import java.util.concurrent.locks.StampedLock; import java.util.stream.IntStream; From deafe570685eaf18e83b3cc284b4834debb2e7f7 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Mon, 17 Jun 2024 19:32:27 +0800 Subject: [PATCH 16/22] fix checkstyle --- .../collections/ConcurrentRoaringBitSet.java | 22 ++++++++++++++++--- .../util/collections/RoaringBitSet.java | 11 +++++++--- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java index 4848e78fcc905..7ab61a0173154 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java @@ -157,6 +157,22 @@ public int previousClearBit(int fromIndex) { return previousClearBit; } + @Override + public int length() { + long stamp = rwLock.tryOptimisticRead(); + int length = super.length(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + length = super.length(); + } finally { + rwLock.unlockRead(stamp); + } + } + return length; + } + @Override public boolean isEmpty() { long stamp = rwLock.tryOptimisticRead(); @@ -374,17 +390,17 @@ public void andNot(RoaringBitSet set) { @Override public Object clone() { long stamp = rwLock.tryOptimisticRead(); - Object clonedBitSet = super.clone(); + RoaringBitSet clone = (RoaringBitSet) super.clone(); if (!rwLock.validate(stamp)) { // Fallback to read lock stamp = rwLock.readLock(); try { - clonedBitSet = super.clone(); + clone = (RoaringBitSet) super.clone(); } finally { rwLock.unlockRead(stamp); } } - return clonedBitSet; + return clone; } @Override diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java index adc4085d4a279..960454f7fd83d 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java @@ -27,7 +27,7 @@ * A RoaringBitmap-backed BitSet implementation. */ @SuppressWarnings("all") -public class RoaringBitSet { +public class RoaringBitSet implements Cloneable { private static final long serialVersionUID = 1L; private RoaringBitmap roaringBitmap; @@ -166,9 +166,14 @@ public boolean equals(Object obj) { return false; } - @Override public Object clone() { - return new RoaringBitSet(roaringBitmap.clone()); + try { + RoaringBitSet result = (RoaringBitSet) super.clone(); + result.roaringBitmap = roaringBitmap.clone(); + return result; + } catch (CloneNotSupportedException e) { + throw new InternalError(e); + } } public IntStream stream() { From 327e2ab3d347fe43118749aea9279974f9043d7c Mon Sep 17 00:00:00 2001 From: dao-jun Date: Mon, 17 Jun 2024 20:13:54 +0800 Subject: [PATCH 17/22] address comment --- .../collections/ConcurrentRoaringBitSet.java | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java index 7ab61a0173154..3cab19853bbef 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java @@ -309,28 +309,6 @@ public RoaringBitSet get(int fromIndex, int toIndex) { return bitSet; } - /** - * Thread-safe version of {@code length()}. - * StampedLock is not reentrant and that's why the length() method is not overridden. Overriding length() method - * would require to use a reentrant lock which would be less performant. - * - * @return length of the bit set - */ - public int safeLength() { - long stamp = rwLock.tryOptimisticRead(); - int length = super.length(); - if (!rwLock.validate(stamp)) { - // Fallback to read lock - stamp = rwLock.readLock(); - try { - length = super.length(); - } finally { - rwLock.unlockRead(stamp); - } - } - return length; - } - @Override public boolean intersects(RoaringBitSet set) { long stamp = rwLock.writeLock(); From fad3370e61bada03088d158dfe74e109f92bc4c5 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Mon, 17 Jun 2024 21:54:28 +0800 Subject: [PATCH 18/22] fix shell/LICENSE.bin.txt --- distribution/shell/src/assemble/LICENSE.bin.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index ff590023ff3a5..ce4ae3443ad26 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -388,6 +388,8 @@ The Apache Software License, Version 2.0 - log4j-core-2.23.1.jar - log4j-slf4j2-impl-2.23.1.jar - log4j-web-2.23.1.jar + * RoaringBitmap + - org.roaringbitmap-RoaringBitmap-1.0.6.jar * OpenTelemetry - opentelemetry-api-1.38.0.jar - opentelemetry-api-incubator-1.38.0-alpha.jar From 2663ab2c77841a9b28e7901d4fa39dc283d87e2a Mon Sep 17 00:00:00 2001 From: dao-jun Date: Mon, 17 Jun 2024 22:58:59 +0800 Subject: [PATCH 19/22] fix shell/LICENSE.bin.txt --- distribution/shell/src/assemble/LICENSE.bin.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index ce4ae3443ad26..ff590023ff3a5 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -388,8 +388,6 @@ The Apache Software License, Version 2.0 - log4j-core-2.23.1.jar - log4j-slf4j2-impl-2.23.1.jar - log4j-web-2.23.1.jar - * RoaringBitmap - - org.roaringbitmap-RoaringBitmap-1.0.6.jar * OpenTelemetry - opentelemetry-api-1.38.0.jar - opentelemetry-api-incubator-1.38.0-alpha.jar From aa2f78b83386af78c12b9861f2fb5745c42f81a5 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Mon, 17 Jun 2024 23:41:50 +0800 Subject: [PATCH 20/22] fix shell/LICENSE.bin.txt --- distribution/shell/src/assemble/LICENSE.bin.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index ff590023ff3a5..6b8214c370bb2 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -383,6 +383,8 @@ The Apache Software License, Version 2.0 - simpleclient_tracer_common-0.16.0.jar - simpleclient_tracer_otel-0.16.0.jar - simpleclient_tracer_otel_agent-0.16.0.jar + * RoaringBitmap + - RoaringBitmap-1.0.6.jar * Log4J - log4j-api-2.23.1.jar - log4j-core-2.23.1.jar From 785e2e6c5bbce1ffbe5e2c0194639cd83667f355 Mon Sep 17 00:00:00 2001 From: dao-jun Date: Tue, 18 Jun 2024 09:51:02 +0800 Subject: [PATCH 21/22] fix code --- .../common/util/collections/RoaringBitSet.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java index 960454f7fd83d..eb3238b2392db 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.common.util.collections; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.BitSet; import java.util.stream.IntStream; import org.roaringbitmap.BitSetUtil; @@ -30,7 +31,7 @@ public class RoaringBitSet implements Cloneable { private static final long serialVersionUID = 1L; - private RoaringBitmap roaringBitmap; + private final RoaringBitmap roaringBitmap; public RoaringBitSet() { roaringBitmap = new RoaringBitmap(); @@ -166,14 +167,9 @@ public boolean equals(Object obj) { return false; } + @SuppressFBWarnings("CN_IDIOM_NO_SUPER_CALL") public Object clone() { - try { - RoaringBitSet result = (RoaringBitSet) super.clone(); - result.roaringBitmap = roaringBitmap.clone(); - return result; - } catch (CloneNotSupportedException e) { - throw new InternalError(e); - } + return new RoaringBitSet(roaringBitmap.clone()); } public IntStream stream() { From ca06e9eb8f07e3821719fbb71fe60f35bca6f1bc Mon Sep 17 00:00:00 2001 From: dao-jun Date: Thu, 20 Jun 2024 00:47:23 +0800 Subject: [PATCH 22/22] code fix --- .../server/src/assemble/LICENSE.bin.txt | 2 +- .../shell/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- .../ConcurrentOpenLongPairRangeSet.java | 38 +- .../collections/ConcurrentRoaringBitSet.java | 16 +- .../util/collections/RoaringBitSet.java | 203 ---- .../util/collections/RoaringBitSetTest.java | 930 ------------------ 7 files changed, 31 insertions(+), 1162 deletions(-) delete mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java delete mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/RoaringBitSetTest.java diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 9f5209bc7fd8d..3b30a40ff83e9 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -514,7 +514,7 @@ The Apache Software License, Version 2.0 * RxJava - io.reactivex.rxjava3-rxjava-3.0.1.jar * RoaringBitmap - - org.roaringbitmap-RoaringBitmap-1.0.6.jar + - org.roaringbitmap-RoaringBitmap-1.1.0.jar * OpenTelemetry - io.opentelemetry-opentelemetry-api-1.38.0.jar - io.opentelemetry-opentelemetry-api-incubator-1.38.0-alpha.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 6b8214c370bb2..6cefa42bed85a 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -384,7 +384,7 @@ The Apache Software License, Version 2.0 - simpleclient_tracer_otel-0.16.0.jar - simpleclient_tracer_otel_agent-0.16.0.jar * RoaringBitmap - - RoaringBitmap-1.0.6.jar + - RoaringBitmap-1.1.0.jar * Log4J - log4j-api-2.23.1.jar - log4j-core-2.23.1.jar diff --git a/pom.xml b/pom.xml index 62644d38d167c..8325336aa9684 100644 --- a/pom.xml +++ b/pom.xml @@ -317,7 +317,7 @@ flexible messaging model and an intuitive client API. 1.3 0.4 9.1.0 - 1.0.6 + 1.1.0 1.6.1 6.4.0 3.33.0 diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java index 7a6518aab07cf..b5ad89d1695d4 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java @@ -22,12 +22,14 @@ import com.google.common.collect.BoundType; import com.google.common.collect.Range; import java.util.ArrayList; +import java.util.BitSet; import java.util.List; import java.util.Map.Entry; import java.util.NavigableMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang.mutable.MutableInt; +import org.roaringbitmap.RoaringBitSet; /** * A Concurrent set comprising zero or more ranges of type {@link LongPair}. This can be alternative of @@ -36,14 +38,14 @@ * 
  * Usage:
  * a. This can be used if one doesn't want to create object for every new inserted {@code range}
- * b. It creates {@link RoaringBitSet} for every unique first-key of the range.
+ * b. It creates {@link BitSet} for every unique first-key of the range.
  * So, this rangeSet is not suitable for large number of unique keys.
  *
*/ public class ConcurrentOpenLongPairRangeSet> implements LongPairRangeSet { - protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); - private boolean threadSafe = true; + protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); + private final boolean threadSafe; private final int bitSetSize; private final LongPairConsumer consumer; @@ -81,7 +83,7 @@ public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, lon if (lowerKey != upperKey) { // (1) set lower to last in lowerRange.getKey() if (isValid(lowerKey, lowerValue)) { - RoaringBitSet rangeBitSet = rangeBitSetMap.get(lowerKey); + BitSet rangeBitSet = rangeBitSetMap.get(lowerKey); // if lower and upper has different key/ledger then set ranges for lower-key only if // a. bitSet already exist and given value is not the last value in the bitset. // it will prevent setting up values which are not actually expected to set @@ -93,16 +95,14 @@ public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, lon } // (2) set 0th-index to upper-index in upperRange.getKey() if (isValid(upperKey, upperValue)) { - RoaringBitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); - if (rangeBitSet != null) { - rangeBitSet.set(0, (int) upperValue + 1); - } + BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); + rangeBitSet.set(0, (int) upperValue + 1); } // No-op if values are not valid eg: if lower == LongPair.earliest or upper == LongPair.latest then nothing // to set } else { long key = lowerKey; - RoaringBitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(key, (k) -> createNewBitSet()); + BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(key, (k) -> createNewBitSet()); rangeBitSet.set((int) lowerValue, (int) upperValue + 1); } updatedAfterCachedForSize = true; @@ -117,7 +117,7 @@ private boolean isValid(long key, long value) { @Override public boolean contains(long key, long value) { - RoaringBitSet rangeBitSet = rangeBitSetMap.get(key); + BitSet rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { return rangeBitSet.get(getSafeEntry(value)); } @@ -126,7 +126,7 @@ public boolean contains(long key, long value) { @Override public Range rangeContaining(long key, long value) { - RoaringBitSet rangeBitSet = rangeBitSetMap.get(key); + BitSet rangeBitSet = rangeBitSetMap.get(key); if (rangeBitSet != null) { if (!rangeBitSet.get(getSafeEntry(value))) { // if position is not part of any range then return null @@ -151,7 +151,7 @@ public boolean isEmpty() { if (rangeBitSetMap.isEmpty()) { return true; } - for (RoaringBitSet rangeBitSet : rangeBitSetMap.values()) { + for (BitSet rangeBitSet : rangeBitSetMap.values()) { if (!rangeBitSet.isEmpty()) { return false; } @@ -171,8 +171,8 @@ public Range span() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); - Entry lastSet = rangeBitSetMap.lastEntry(); + Entry firstSet = rangeBitSetMap.firstEntry(); + Entry lastSet = rangeBitSetMap.lastEntry(); int first = firstSet.getValue().nextSetBit(0); int last = lastSet.getValue().previousSetBit(lastSet.getValue().size()); return Range.openClosed(consumer.apply(firstSet.getKey(), first - 1), consumer.apply(lastSet.getKey(), last)); @@ -235,7 +235,7 @@ public Range firstRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry firstSet = rangeBitSetMap.firstEntry(); + Entry firstSet = rangeBitSetMap.firstEntry(); int lower = firstSet.getValue().nextSetBit(0); int upper = Math.max(lower, firstSet.getValue().nextClearBit(lower) - 1); return Range.openClosed(consumer.apply(firstSet.getKey(), lower - 1), consumer.apply(firstSet.getKey(), upper)); @@ -246,7 +246,7 @@ public Range lastRange() { if (rangeBitSetMap.isEmpty()) { return null; } - Entry lastSet = rangeBitSetMap.lastEntry(); + Entry lastSet = rangeBitSetMap.lastEntry(); int upper = lastSet.getValue().previousSetBit(lastSet.getValue().size()); int lower = Math.min(lastSet.getValue().previousClearBit(upper), upper); return Range.openClosed(consumer.apply(lastSet.getKey(), lower), consumer.apply(lastSet.getKey(), upper)); @@ -254,11 +254,11 @@ public Range lastRange() { @Override public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { - NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); + NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); MutableInt v = new MutableInt(0); subMap.forEach((key, bitset) -> { if (key == lowerKey || key == upperKey) { - RoaringBitSet temp = (RoaringBitSet) bitset.clone(); + BitSet temp = (BitSet) bitset.clone(); // Trim the bitset index which < lowerValue if (key == lowerKey) { temp.clear(0, (int) Math.max(0, lowerValue)); @@ -412,7 +412,7 @@ private int getSafeEntry(long value) { return (int) Math.max(value, -1); } - private RoaringBitSet createNewBitSet() { + private BitSet createNewBitSet() { return this.threadSafe ? new ConcurrentRoaringBitSet() : new RoaringBitSet(); } } \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java index 3cab19853bbef..814e58400993b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitSet.java @@ -18,8 +18,10 @@ */ package org.apache.pulsar.common.util.collections; +import java.util.BitSet; import java.util.concurrent.locks.StampedLock; import java.util.stream.IntStream; +import org.roaringbitmap.RoaringBitSet; public class ConcurrentRoaringBitSet extends RoaringBitSet { private final StampedLock rwLock = new StampedLock(); @@ -294,9 +296,9 @@ public void set(int fromIndex, int toIndex, boolean value) { } @Override - public RoaringBitSet get(int fromIndex, int toIndex) { + public BitSet get(int fromIndex, int toIndex) { long stamp = rwLock.tryOptimisticRead(); - RoaringBitSet bitSet = super.get(fromIndex, toIndex); + BitSet bitSet = super.get(fromIndex, toIndex); if (!rwLock.validate(stamp)) { // Fallback to read lock stamp = rwLock.readLock(); @@ -310,7 +312,7 @@ public RoaringBitSet get(int fromIndex, int toIndex) { } @Override - public boolean intersects(RoaringBitSet set) { + public boolean intersects(BitSet set) { long stamp = rwLock.writeLock(); try { return super.intersects(set); @@ -320,7 +322,7 @@ public boolean intersects(RoaringBitSet set) { } @Override - public void and(RoaringBitSet set) { + public void and(BitSet set) { long stamp = rwLock.writeLock(); try { super.and(set); @@ -330,7 +332,7 @@ public void and(RoaringBitSet set) { } @Override - public void or(RoaringBitSet set) { + public void or(BitSet set) { long stamp = rwLock.writeLock(); try { super.or(set); @@ -340,7 +342,7 @@ public void or(RoaringBitSet set) { } @Override - public void xor(RoaringBitSet set) { + public void xor(BitSet set) { long stamp = rwLock.writeLock(); try { super.xor(set); @@ -350,7 +352,7 @@ public void xor(RoaringBitSet set) { } @Override - public void andNot(RoaringBitSet set) { + public void andNot(BitSet set) { long stamp = rwLock.writeLock(); try { super.andNot(set); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java deleted file mode 100644 index eb3238b2392db..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/RoaringBitSet.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.collections; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.BitSet; -import java.util.stream.IntStream; -import org.roaringbitmap.BitSetUtil; -import org.roaringbitmap.RoaringBitmap; - -/** - * A RoaringBitmap-backed BitSet implementation. - */ -@SuppressWarnings("all") -public class RoaringBitSet implements Cloneable { - private static final long serialVersionUID = 1L; - - private final RoaringBitmap roaringBitmap; - - public RoaringBitSet() { - roaringBitmap = new RoaringBitmap(); - } - - private RoaringBitSet(RoaringBitmap roaringBitmap) { - this.roaringBitmap = roaringBitmap; - } - - public RoaringBitmap getRoaringBitmap() { - return roaringBitmap; - } - - public void set(int bitIndex) { - roaringBitmap.add(bitIndex); - } - - public void set(int bitIndex, boolean value) { - if (value) { - roaringBitmap.add(bitIndex); - } else { - roaringBitmap.remove(bitIndex); - } - } - - public void set(int fromIndex, int toIndex) { - roaringBitmap.add(fromIndex, toIndex); - } - - public void set(int fromIndex, int toIndex, boolean value) { - if (value) { - roaringBitmap.add(fromIndex, toIndex); - } else { - roaringBitmap.remove(fromIndex, toIndex); - } - } - - public void clear(int bitIndex) { - roaringBitmap.remove(bitIndex); - } - - public void clear(int fromIndex, int toIndex) { - roaringBitmap.remove(fromIndex, toIndex); - } - - public void clear() { - roaringBitmap.clear(); - } - - public boolean get(int bitIndex) { - return roaringBitmap.contains(bitIndex); - } - - public RoaringBitSet get(int fromIndex, int toIndex) { - BitSet bitSet = BitSetUtil.bitsetOf(roaringBitmap); - bitSet = bitSet.get(fromIndex, toIndex); - return new RoaringBitSet(fromBitSet(bitSet)); - } - - public int nextSetBit(int fromIndex) { - return (int) roaringBitmap.nextValue(fromIndex); - } - - public int nextClearBit(int fromIndex) { - return (int) roaringBitmap.nextAbsentValue(fromIndex); - } - - public int previousSetBit(int fromIndex) { - return (int) roaringBitmap.previousValue(fromIndex); - } - - public int previousClearBit(int fromIndex) { - return (int) roaringBitmap.previousAbsentValue(fromIndex); - } - - public int length() { - if (roaringBitmap.isEmpty()) { - return 0; - } - return roaringBitmap.last() + 1; - } - - public boolean isEmpty() { - return roaringBitmap.isEmpty(); - } - - public boolean intersects(RoaringBitSet set) { - return RoaringBitmap.intersects(roaringBitmap, ((RoaringBitSet) set).roaringBitmap); - } - - public int cardinality() { - return roaringBitmap.getCardinality(); - } - - public void and(RoaringBitSet set) { - roaringBitmap.and(((RoaringBitSet) set).roaringBitmap); - } - - public void or(RoaringBitSet set) { - roaringBitmap.or(((RoaringBitSet) set).roaringBitmap); - } - - public void xor(RoaringBitSet set) { - roaringBitmap.xor(((RoaringBitSet) set).roaringBitmap); - } - - public void andNot(RoaringBitSet set) { - roaringBitmap.andNot(((RoaringBitSet) set).roaringBitmap); - } - - @Override - public int hashCode() { - return roaringBitmap.hashCode(); - } - - public int size() { - if (roaringBitmap.isEmpty()) { - return 0; - } - int lastBit = Math.max(length(), 64); - int remainder = lastBit % 64; - return remainder == 0 ? lastBit : lastBit + 64 - remainder; - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (obj instanceof RoaringBitSet) { - return roaringBitmap.equals(((RoaringBitSet) obj).roaringBitmap); - } - return false; - } - - @SuppressFBWarnings("CN_IDIOM_NO_SUPER_CALL") - public Object clone() { - return new RoaringBitSet(roaringBitmap.clone()); - } - - public IntStream stream() { - return roaringBitmap.stream(); - } - - @Override - public String toString() { - return roaringBitmap.toString(); - } - - public void flip(int bitIndex) { - roaringBitmap.flip(bitIndex, bitIndex + 1); - } - - public void flip(int fromIndex, int toIndex) { - roaringBitmap.flip(fromIndex, toIndex); - } - - public long[] toLongArray() { - return BitSetUtil.toLongArray(roaringBitmap); - } - - public byte[] toByteArray() { - return BitSetUtil.toByteArray(roaringBitmap); - } - - private static RoaringBitmap fromBitSet(BitSet bitSet) { - return BitSetUtil.bitmapOf(bitSet); - } -} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/RoaringBitSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/RoaringBitSetTest.java deleted file mode 100644 index fefc0e2839a1a..0000000000000 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/RoaringBitSetTest.java +++ /dev/null @@ -1,930 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.collections; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Random; -import org.testng.Assert; -import org.testng.annotations.Test; - -// From https://github.com/openjdk/jdk/blob/master/test/jdk/java/util/BitSet/BSMethods.java -@SuppressWarnings("all") -public class RoaringBitSetTest { - private static Random generator = new Random(); - - @Test - public void testSetGetClearFlip() { - int failCount = 0; - - for (int i = 0; i < 100; i++) { - RoaringBitSet testSet = new RoaringBitSet(); - HashSet history = new HashSet<>(); - - // Set a random number of bits in random places - // up to a random maximum - int nextBitToSet = 0; - int numberOfSetBits = generator.nextInt(100) + 1; - int highestPossibleSetBit = generator.nextInt(1000) + 1; - for (int x = 0; x < numberOfSetBits; x++) { - nextBitToSet = generator.nextInt(highestPossibleSetBit); - history.add(Integer.valueOf(nextBitToSet)); - testSet.set(nextBitToSet); - } - - // Make sure each bit is set appropriately - for (int x = 0; x < highestPossibleSetBit; x++) { - if (testSet.get(x) != history.contains(Integer.valueOf(x))) - failCount++; - } - - // Clear the bits - Iterator setBitIterator = history.iterator(); - while (setBitIterator.hasNext()) { - Integer setBit = setBitIterator.next(); - testSet.clear(setBit.intValue()); - } - - // Verify they were cleared - for (int x = 0; x < highestPossibleSetBit; x++) - if (testSet.get(x)) - failCount++; - if (testSet.length() != 0) - failCount++; - - // Set them with set(int, boolean) - setBitIterator = history.iterator(); - while (setBitIterator.hasNext()) { - Integer setBit = setBitIterator.next(); - testSet.set(setBit.intValue(), true); - } - - // Make sure each bit is set appropriately - for (int x = 0; x < highestPossibleSetBit; x++) { - if (testSet.get(x) != history.contains(Integer.valueOf(x))) - failCount++; - } - - // Clear them with set(int, boolean) - setBitIterator = history.iterator(); - while (setBitIterator.hasNext()) { - Integer setBit = setBitIterator.next(); - testSet.set(setBit.intValue(), false); - } - - // Verify they were cleared - for (int x = 0; x < highestPossibleSetBit; x++) - if (testSet.get(x)) - failCount++; - if (testSet.length() != 0) - failCount++; - - // Flip them on - setBitIterator = history.iterator(); - while (setBitIterator.hasNext()) { - Integer setBit = setBitIterator.next(); - testSet.flip(setBit.intValue()); - } - - // Verify they were flipped - for (int x = 0; x < highestPossibleSetBit; x++) { - if (testSet.get(x) != history.contains(Integer.valueOf(x))) - failCount++; - } - - // Flip them off - setBitIterator = history.iterator(); - while (setBitIterator.hasNext()) { - Integer setBit = setBitIterator.next(); - testSet.flip(setBit.intValue()); - } - - // Verify they were flipped - for (int x = 0; x < highestPossibleSetBit; x++) - if (testSet.get(x)) - failCount++; - if (testSet.length() != 0) - failCount++; - - checkSanity(testSet); - } - - Assert.assertEquals(failCount, 0); - } - - @Test - public void testClear() { - int failCount = 0; - - for (int i = 0; i < 1000; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - - // Make a fairly random bitset - int numberOfSetBits = generator.nextInt(100) + 1; - int highestPossibleSetBit = generator.nextInt(1000) + 1; - - for (int x = 0; x < numberOfSetBits; x++) - b1.set(generator.nextInt(highestPossibleSetBit)); - - RoaringBitSet b2 = (RoaringBitSet) b1.clone(); - - // Clear out a random range - int rangeStart = generator.nextInt(100); - int rangeEnd = rangeStart + generator.nextInt(100); - - // Use the clear(int, int) call on b1 - b1.clear(rangeStart, rangeEnd); - - // Use a loop on b2 - for (int x = rangeStart; x < rangeEnd; x++) - b2.clear(x); - - // Verify their equality - if (!b1.equals(b2)) { - failCount++; - } - checkEquality(b1, b2); - } - - Assert.assertEquals(failCount, 0); - } - - @Test - public void testFlip() { - int failCount = 0; - - for (int i = 0; i < 1000; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - - // Make a fairly random bitset - int numberOfSetBits = generator.nextInt(100) + 1; - int highestPossibleSetBit = generator.nextInt(1000) + 1; - - for (int x = 0; x < numberOfSetBits; x++) - b1.set(generator.nextInt(highestPossibleSetBit)); - - RoaringBitSet b2 = (RoaringBitSet) b1.clone(); - - // Flip a random range - int rangeStart = generator.nextInt(100); - int rangeEnd = rangeStart + generator.nextInt(100); - - // Use the flip(int, int) call on b1 - b1.flip(rangeStart, rangeEnd); - - // Use a loop on b2 - for (int x = rangeStart; x < rangeEnd; x++) - b2.flip(x); - - // Verify their equality - if (!b1.equals(b2)) - failCount++; - checkEquality(b1, b2); - } - - Assert.assertEquals(failCount, 0); - } - - @Test - public void testSet() { - int failCount = 0; - - // Test set(int, int) - for (int i = 0; i < 1000; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - - // Make a fairly random bitset - int numberOfSetBits = generator.nextInt(100) + 1; - int highestPossibleSetBit = generator.nextInt(1000) + 1; - - for (int x = 0; x < numberOfSetBits; x++) - b1.set(generator.nextInt(highestPossibleSetBit)); - - RoaringBitSet b2 = (RoaringBitSet) b1.clone(); - - // Set a random range - int rangeStart = generator.nextInt(100); - int rangeEnd = rangeStart + generator.nextInt(100); - - // Use the set(int, int) call on b1 - b1.set(rangeStart, rangeEnd); - - // Use a loop on b2 - for (int x = rangeStart; x < rangeEnd; x++) - b2.set(x); - - // Verify their equality - if (!b1.equals(b2)) { - failCount++; - } - checkEquality(b1, b2); - } - - // Test set(int, int, boolean) - for (int i = 0; i < 100; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - - // Make a fairly random bitset - int numberOfSetBits = generator.nextInt(100) + 1; - int highestPossibleSetBit = generator.nextInt(1000) + 1; - - for (int x = 0; x < numberOfSetBits; x++) - b1.set(generator.nextInt(highestPossibleSetBit)); - - RoaringBitSet b2 = (RoaringBitSet) b1.clone(); - boolean setOrClear = generator.nextBoolean(); - - // Set a random range - int rangeStart = generator.nextInt(100); - int rangeEnd = rangeStart + generator.nextInt(100); - - // Use the set(int, int, boolean) call on b1 - b1.set(rangeStart, rangeEnd, setOrClear); - - // Use a loop on b2 - for (int x = rangeStart; x < rangeEnd; x++) - b2.set(x, setOrClear); - - // Verify their equality - if (!b1.equals(b2)) { - failCount++; - } - checkEquality(b1, b2); - } - - Assert.assertEquals(failCount, 0); - } - - @Test - public void testGet() { - int failCount = 0; - - for (int i = 0; i < 1000; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - - // Make a fairly random bitset - int numberOfSetBits = generator.nextInt(100) + 1; - int highestPossibleSetBit = generator.nextInt(1000) + 1; - - for (int x = 0; x < numberOfSetBits; x++) - b1.set(generator.nextInt(highestPossibleSetBit)); - - // Get a new set from a random range - int rangeStart = generator.nextInt(100); - int rangeEnd = rangeStart + generator.nextInt(100); - - RoaringBitSet b2 = (RoaringBitSet) b1.get(rangeStart, rangeEnd); - - RoaringBitSet b3 = new RoaringBitSet(); - for (int x = rangeStart; x < rangeEnd; x++) - b3.set(x - rangeStart, b1.get(x)); - - // Verify their equality - if (!b2.equals(b3)) { - failCount++; - } - checkEquality(b2, b3); - } - - Assert.assertEquals(failCount, 0); - } - - @Test - public void testAndNot() { - int failCount = 0; - - for (int i = 0; i < 100; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - RoaringBitSet b2 = new RoaringBitSet(); - - // Set some random bits in first set and remember them - int nextBitToSet = 0; - for (int x = 0; x < 10; x++) - b1.set(generator.nextInt(255)); - - // Set some random bits in second set and remember them - for (int x = 10; x < 20; x++) - b2.set(generator.nextInt(255)); - - // andNot the sets together - RoaringBitSet b3 = (RoaringBitSet) b1.clone(); - b3.andNot(b2); - - // Examine each bit of b3 for errors - for (int x = 0; x < 256; x++) { - boolean bit1 = b1.get(x); - boolean bit2 = b2.get(x); - boolean bit3 = b3.get(x); - if (!(bit3 == (bit1 & (!bit2)))) - failCount++; - } - checkSanity(b1, b2, b3); - } - Assert.assertEquals(failCount, 0); - } - - @Test - public void testAnd() { - int failCount = 0; - - for (int i = 0; i < 100; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - RoaringBitSet b2 = new RoaringBitSet(); - - // Set some random bits in first set and remember them - int nextBitToSet = 0; - for (int x = 0; x < 10; x++) - b1.set(generator.nextInt(255)); - - // Set more random bits in second set and remember them - for (int x = 10; x < 20; x++) - b2.set(generator.nextInt(255)); - - // And the sets together - RoaringBitSet b3 = (RoaringBitSet) b1.clone(); - b3.and(b2); - - // Examine each bit of b3 for errors - for (int x = 0; x < 256; x++) { - boolean bit1 = b1.get(x); - boolean bit2 = b2.get(x); - boolean bit3 = b3.get(x); - if (!(bit3 == (bit1 & bit2))) - failCount++; - } - checkSanity(b1, b2, b3); - } - - // `and' that happens to clear the last word - RoaringBitSet b4 = makeSet(2, 127); - b4.and(makeSet(2, 64)); - checkSanity(b4); - if (!(b4.equals(makeSet(2)))) - failCount++; - - Assert.assertEquals(failCount, 0); - } - - @Test - public void testOr() { - int failCount = 0; - - for (int i = 0; i < 100; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - RoaringBitSet b2 = new RoaringBitSet(); - int[] history = new int[20]; - - // Set some random bits in first set and remember them - int nextBitToSet = 0; - for (int x = 0; x < 10; x++) { - nextBitToSet = generator.nextInt(255); - history[x] = nextBitToSet; - b1.set(nextBitToSet); - } - - // Set more random bits in second set and remember them - for (int x = 10; x < 20; x++) { - nextBitToSet = generator.nextInt(255); - history[x] = nextBitToSet; - b2.set(nextBitToSet); - } - - // Or the sets together - RoaringBitSet b3 = (RoaringBitSet) b1.clone(); - b3.or(b2); - - // Verify the set bits of b3 from the history - int historyIndex = 0; - for (int x = 0; x < 20; x++) { - if (!b3.get(history[x])) - failCount++; - } - - // Examine each bit of b3 for errors - for (int x = 0; x < 256; x++) { - boolean bit1 = b1.get(x); - boolean bit2 = b2.get(x); - boolean bit3 = b3.get(x); - if (!(bit3 == (bit1 | bit2))) - failCount++; - } - checkSanity(b1, b2, b3); - } - Assert.assertEquals(failCount, 0); - } - - @Test - public void testXor() { - int failCount = 0; - - for (int i = 0; i < 100; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - RoaringBitSet b2 = new RoaringBitSet(); - - // Set some random bits in first set and remember them - int nextBitToSet = 0; - for (int x = 0; x < 10; x++) - b1.set(generator.nextInt(255)); - - // Set more random bits in second set and remember them - for (int x = 10; x < 20; x++) - b2.set(generator.nextInt(255)); - - // Xor the sets together - RoaringBitSet b3 = (RoaringBitSet) b1.clone(); - b3.xor(b2); - - // Examine each bit of b3 for errors - for (int x = 0; x < 256; x++) { - boolean bit1 = b1.get(x); - boolean bit2 = b2.get(x); - boolean bit3 = b3.get(x); - if (!(bit3 == (bit1 ^ bit2))) - failCount++; - } - checkSanity(b1, b2, b3); - b3.xor(b3); - checkEmpty(b3); - } - - // xor that happens to clear the last word - RoaringBitSet b4 = makeSet(2, 64, 127); - b4.xor(makeSet(64, 127)); - checkSanity(b4); - if (!(b4.equals(makeSet(2)))) - failCount++; - - Assert.assertEquals(failCount, 0); - } - - @Test - public void testLength() { - int failCount = 0; - - // Test length after set - for (int i = 0; i < 100; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - int highestSetBit = 0; - - for (int x = 0; x < 100; x++) { - int nextBitToSet = generator.nextInt(255); - if (nextBitToSet > highestSetBit) - highestSetBit = nextBitToSet; - b1.set(nextBitToSet); - if (b1.length() != highestSetBit + 1) - failCount++; - } - checkSanity(b1); - } - - // Test length after flip - for (int i = 0; i < 100; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - for (int x = 0; x < 100; x++) { - // Flip a random range twice - int rangeStart = generator.nextInt(100); - int rangeEnd = rangeStart + generator.nextInt(100); - b1.flip(rangeStart); - b1.flip(rangeStart); - if (b1.length() != 0) - failCount++; - b1.flip(rangeStart, rangeEnd); - b1.flip(rangeStart, rangeEnd); - if (b1.length() != 0) - failCount++; - } - checkSanity(b1); - } - - // Test length after or - for (int i = 0; i < 100; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - RoaringBitSet b2 = new RoaringBitSet(); - int bit1 = generator.nextInt(100); - int bit2 = generator.nextInt(100); - int highestSetBit = (bit1 > bit2) ? bit1 : bit2; - b1.set(bit1); - b2.set(bit2); - b1.or(b2); - if (b1.length() != highestSetBit + 1) - failCount++; - checkSanity(b1, b2); - } - Assert.assertEquals(failCount, 0); - } - - @Test - public void testEquals() { - int failCount = 0; - - for (int i = 0; i < 100; i++) { - // Create BitSets of different sizes - RoaringBitSet b1 = new RoaringBitSet(); - RoaringBitSet b2 = new RoaringBitSet(); - - // Set some random bits - int nextBitToSet = 0; - for (int x = 0; x < 10; x++) { - nextBitToSet += generator.nextInt(50) + 1; - b1.set(nextBitToSet); - b2.set(nextBitToSet); - } - - // Verify their equality despite different storage sizes - if (!b1.equals(b2)) - failCount++; - checkEquality(b1, b2); - } - Assert.assertEquals(failCount, 0); - } - - @Test - public void testNextSetBit() { - int failCount = 0; - - for (int i = 0; i < 100; i++) { - int numberOfSetBits = generator.nextInt(100) + 1; - RoaringBitSet testSet = new RoaringBitSet(); - int[] history = new int[numberOfSetBits]; - - // Set some random bits and remember them - int nextBitToSet = 0; - for (int x = 0; x < numberOfSetBits; x++) { - nextBitToSet += generator.nextInt(30) + 1; - history[x] = nextBitToSet; - testSet.set(nextBitToSet); - } - - // Verify their retrieval using nextSetBit() - int historyIndex = 0; - for (int x = testSet.nextSetBit(0); x >= 0; x = testSet.nextSetBit(x + 1)) { - if (x != history[historyIndex++]) - failCount++; - } - - checkSanity(testSet); - } - - Assert.assertEquals(failCount, 0); - } - - @Test - public void testNextClearBit() { - int failCount = 0; - - for (int i = 0; i < 1000; i++) { - RoaringBitSet b = new RoaringBitSet(); - int[] history = new int[10]; - - // Set all the bits - for (int x = 0; x < 256; x++) - b.set(x); - - // Clear some random bits and remember them - int nextBitToClear = 0; - for (int x = 0; x < 10; x++) { - nextBitToClear += generator.nextInt(24) + 1; - history[x] = nextBitToClear; - b.clear(nextBitToClear); - } - - // Verify their retrieval using nextClearBit() - int historyIndex = 0; - for (int x = b.nextClearBit(0); x < 256; x = b.nextClearBit(x + 1)) { - if (x != history[historyIndex++]) - failCount++; - } - - checkSanity(b); - } - - // regression test for 4350178 - RoaringBitSet bs = new RoaringBitSet(); - if (bs.nextClearBit(0) != 0) - failCount++; - for (int i = 0; i < 64; i++) { - bs.set(i); - if (bs.nextClearBit(0) != i + 1) - failCount++; - } - - checkSanity(bs); - Assert.assertEquals(failCount, 0); - } - - @Test - public void testIntersects() { - int failCount = 0; - - for (int i = 0; i < 100; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - RoaringBitSet b2 = new RoaringBitSet(); - - // Set some random bits in first set - int nextBitToSet = 0; - for (int x = 0; x < 30; x++) { - nextBitToSet = generator.nextInt(255); - b1.set(nextBitToSet); - } - - // Set more random bits in second set - for (int x = 0; x < 30; x++) { - nextBitToSet = generator.nextInt(255); - b2.set(nextBitToSet); - } - - // Make sure they intersect - nextBitToSet = generator.nextInt(255); - b1.set(nextBitToSet); - b2.set(nextBitToSet); - - if (!b1.intersects(b2)) - failCount++; - - // Remove the common set bits - b1.andNot(b2); - - // Make sure they don't intersect - if (b1.intersects(b2)) - failCount++; - - checkSanity(b1, b2); - } - - Assert.assertEquals(failCount, 0); - } - - @Test - public void testCardinality() { - int failCount = 0; - - for (int i = 0; i < 100; i++) { - RoaringBitSet b1 = new RoaringBitSet(); - - // Set a random number of increasing bits - int nextBitToSet = 0; - int iterations = generator.nextInt(20) + 1; - for (int x = 0; x < iterations; x++) { - nextBitToSet += generator.nextInt(20) + 1; - b1.set(nextBitToSet); - } - - if (b1.cardinality() != iterations) { - failCount++; - } - - checkSanity(b1); - } - - Assert.assertEquals(failCount, 0); - } - - @Test - public void testEmpty() { - int failCount = 0; - - RoaringBitSet b1 = new RoaringBitSet(); - if (!b1.isEmpty()) - failCount++; - - int nextBitToSet = 0; - int numberOfSetBits = generator.nextInt(100) + 1; - int highestPossibleSetBit = generator.nextInt(1000) + 1; - for (int x = 0; x < numberOfSetBits; x++) { - nextBitToSet = generator.nextInt(highestPossibleSetBit); - b1.set(nextBitToSet); - if (b1.isEmpty()) - failCount++; - b1.clear(nextBitToSet); - if (!b1.isEmpty()) - failCount++; - } - - Assert.assertEquals(failCount, 0); - } - - @Test - public void testEmpty2() { - { - RoaringBitSet t = new RoaringBitSet(); - t.set(100); - t.clear(3, 600); - checkEmpty(t); - } - checkEmpty(new RoaringBitSet()); - - RoaringBitSet s = new RoaringBitSet(); - checkEmpty(s); - s.clear(92); - checkEmpty(s); - s.clear(127, 127); - checkEmpty(s); - s.set(127, 127); - checkEmpty(s); - s.set(128, 128); - checkEmpty(s); - RoaringBitSet empty = new RoaringBitSet(); - { - RoaringBitSet t = new RoaringBitSet(); - t.and(empty); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - t.or(empty); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - t.xor(empty); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - t.andNot(empty); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - t.and(t); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - t.or(t); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - t.xor(t); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - t.andNot(t); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - t.and(makeSet(1)); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - t.and(makeSet(127)); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - t.and(makeSet(128)); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - t.flip(7); - t.flip(7); - checkEmpty(t); - } - { - RoaringBitSet t = new RoaringBitSet(); - checkEmpty((RoaringBitSet) t.get(200, 300)); - } - { - RoaringBitSet t = makeSet(2, 5); - Assert.assertEquals(makeSet(0, 3), t.get(2, 6)); - } - } - - @Test - public void testLogicalIdentities() { - int failCount = 0; - - // Verify that (!b1)|(!b2) == !(b1&b2) - for (int i = 0; i < 50; i++) { - // Construct two fairly random bitsets - RoaringBitSet b1 = new RoaringBitSet(); - RoaringBitSet b2 = new RoaringBitSet(); - - int numberOfSetBits = generator.nextInt(100) + 1; - int highestPossibleSetBit = generator.nextInt(1000) + 1; - - for (int x = 0; x < numberOfSetBits; x++) { - b1.set(generator.nextInt(highestPossibleSetBit)); - b2.set(generator.nextInt(highestPossibleSetBit)); - } - - RoaringBitSet b3 = (RoaringBitSet) b1.clone(); - RoaringBitSet b4 = (RoaringBitSet) b2.clone(); - - for (int x = 0; x < highestPossibleSetBit; x++) { - b1.flip(x); - b2.flip(x); - } - b1.or(b2); - b3.and(b4); - for (int x = 0; x < highestPossibleSetBit; x++) - b3.flip(x); - if (!b1.equals(b3)) - failCount++; - checkSanity(b1, b2, b3, b4); - } - - // Verify that (b1&(!b2)|(b2&(!b1) == b1^b2 - for (int i = 0; i < 50; i++) { - // Construct two fairly random bitsets - RoaringBitSet b1 = new RoaringBitSet(); - RoaringBitSet b2 = new RoaringBitSet(); - - int numberOfSetBits = generator.nextInt(100) + 1; - int highestPossibleSetBit = generator.nextInt(1000) + 1; - - for (int x = 0; x < numberOfSetBits; x++) { - b1.set(generator.nextInt(highestPossibleSetBit)); - b2.set(generator.nextInt(highestPossibleSetBit)); - } - - RoaringBitSet b3 = (RoaringBitSet) b1.clone(); - RoaringBitSet b4 = (RoaringBitSet) b2.clone(); - RoaringBitSet b5 = (RoaringBitSet) b1.clone(); - RoaringBitSet b6 = (RoaringBitSet) b2.clone(); - - for (int x = 0; x < highestPossibleSetBit; x++) - b2.flip(x); - b1.and(b2); - for (int x = 0; x < highestPossibleSetBit; x++) - b3.flip(x); - b3.and(b4); - b1.or(b3); - b5.xor(b6); - if (!b1.equals(b5)) - failCount++; - checkSanity(b1, b2, b3, b4, b5, b6); - } - Assert.assertEquals(failCount, 0); - } - - private static void checkSanity(RoaringBitSet... sets) { - for (RoaringBitSet s : sets) { - int len = s.length(); - int cardinality1 = s.cardinality(); - int cardinality2 = 0; - for (int i = s.nextSetBit(0); i >= 0; i = s.nextSetBit(i + 1)) { - Assert.assertTrue(s.get(i)); - cardinality2++; - } - Assert.assertEquals(s.nextSetBit(len), -1); - Assert.assertEquals(len, s.nextClearBit(len)); - Assert.assertEquals((len == 0), s.isEmpty()); - Assert.assertEquals(cardinality2, cardinality1); - Assert.assertTrue(len <= s.size()); - Assert.assertTrue(len >= 0); - Assert.assertTrue(cardinality1 >= 0); - } - } - - private static void checkEquality(RoaringBitSet s, RoaringBitSet t) { - checkSanity(s, t); - Assert.assertEquals(t, s); - Assert.assertEquals(t.toString(), s.toString()); - Assert.assertEquals(s.length(), t.length()); - Assert.assertEquals(s.cardinality(), t.cardinality()); - } - - private static RoaringBitSet makeSet(int... elts) { - RoaringBitSet s = new RoaringBitSet(); - for (int elt : elts) - s.set(elt); - return s; - } - - private static void checkEmpty(RoaringBitSet s) { - Assert.assertTrue(s.isEmpty()); - Assert.assertEquals(s.length(), 0); - Assert.assertEquals(s.cardinality(), 0); - Assert.assertEquals(s, new RoaringBitSet()); - Assert.assertEquals(s.nextSetBit(0), -1); - Assert.assertEquals(s.nextSetBit(127), -1); - Assert.assertEquals(s.nextSetBit(128), -1); - Assert.assertEquals(s.nextClearBit(0), 0); - Assert.assertEquals(s.nextClearBit(127), 127); - Assert.assertEquals(s.nextClearBit(128), 128); - Assert.assertEquals(s.toString(), "{}"); - Assert.assertFalse(s.get(0)); - } -}