IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-11-29 03:54:01 -06:00
Code Issues Packages Projects Releases Wiki Activity

Detect duplicates after a gap

Browse Source 
- rename earliest -> lowest to clarify the real intention
- polish code
- add unit test
- increase duplicate counter if pending messages already contains the sequence number
This commit is contained in:
Lari Hotari 2021-10-21 00:35:11 +03:00
parent 956c3b698e
commit 6b50060a9a
2 changed files with 55 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/ReceivedMessageSequenceTracker.java
View File

@ -16,13 +16,12 @@ class ReceivedMessageSequenceTracker implements AutoCloseable{
public static final int MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS = 20;
// message out-of-sequence error counter
private final Counter msgErrOutOfSeqCounter;
// message out-of-sequence error counter
// duplicate message error counter
private final Counter msgErrDuplicateCounter;
// message loss error counter
private final Counter msgErrLossCounter;
long expectedNumber = -1;
SortedSet<Long> pendingOutOfSeqNumbers = new TreeSet<>();
private final SortedSet<Long> pendingOutOfSeqNumbers = new TreeSet<>();
private long expectedNumber = -1;
ReceivedMessageSequenceTracker(Counter msgErrOutOfSeqCounter, Counter msgErrDuplicateCounter, Counter msgErrLossCounter) {
@ -50,9 +49,11 @@ class ReceivedMessageSequenceTracker implements AutoCloseable{
boolean messagesSkipped = false;
if (sequenceNumber > expectedNumber) {
if (pendingOutOfSeqNumbers.size() == MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS) {
messagesSkipped = processEarliestPendingOutOfSequenceNumber();
messagesSkipped = processLowestPendingOutOfSequenceNumber();
}
if(!pendingOutOfSeqNumbers.add(sequenceNumber)) {
msgErrDuplicateCounter.inc();
}
pendingOutOfSeqNumbers.add(sequenceNumber);
} else {
// sequenceNumber == expectedNumber
expectedNumber++;
@ -61,15 +62,15 @@ class ReceivedMessageSequenceTracker implements AutoCloseable{
cleanUpTooFarBehindOutOfSequenceNumbers();
}
private boolean processEarliestPendingOutOfSequenceNumber() {
// remove the earliest pending out of sequence number
Long earliestOutOfSeqNumber = pendingOutOfSeqNumbers.first();
pendingOutOfSeqNumbers.remove(earliestOutOfSeqNumber);
if (earliestOutOfSeqNumber > expectedNumber) {
// skip the expected number ahead to the number after the earliest sequence number
private boolean processLowestPendingOutOfSequenceNumber() {
// remove the lowest pending out of sequence number
Long lowestOutOfSeqNumber = pendingOutOfSeqNumbers.first();
pendingOutOfSeqNumbers.remove(lowestOutOfSeqNumber);
if (lowestOutOfSeqNumber > expectedNumber) {
// skip the expected number ahead to the number after the lowest sequence number
// increment the counter with the amount of sequence numbers that got skipped
msgErrLossCounter.inc(earliestOutOfSeqNumber - expectedNumber);
expectedNumber = earliestOutOfSeqNumber + 1;
msgErrLossCounter.inc(lowestOutOfSeqNumber - expectedNumber);
expectedNumber = lowestOutOfSeqNumber + 1;
return true;
} else {
msgErrLossCounter.inc();
@ -107,7 +108,7 @@ class ReceivedMessageSequenceTracker implements AutoCloseable{
@Override
public void close() {
while (!pendingOutOfSeqNumbers.isEmpty()) {
processPendingOutOfSequenceNumbers(processEarliestPendingOutOfSequenceNumber());
processPendingOutOfSequenceNumbers(processLowestPendingOutOfSequenceNumber());
}
}
}

39
driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/ops/ReceivedMessageSequenceTrackerTest.java
View File

@ -149,4 +149,43 @@ class ReceivedMessageSequenceTrackerTest {
assertEquals(1, msgErrLossCounter.getCount());
}
@Test
void shouldDetectGapAndMessageDuplication() {
// when
for (long l = 0; l < 100L; l++) {
if (l != 11L) {
messageSequenceTracker.sequenceNumberReceived(l);
}
if (l == 12L) {
messageSequenceTracker.sequenceNumberReceived(l);
}
}
messageSequenceTracker.close();
// then
assertEquals(0, msgErrOutOfSeqCounter.getCount());
assertEquals(1, msgErrDuplicateCounter.getCount());
assertEquals(1, msgErrLossCounter.getCount());
}
@Test
void shouldDetectGapAndMessageDuplicationTimes2() {
// when
for (long l = 0; l < 100L; l++) {
if (l != 11L) {
messageSequenceTracker.sequenceNumberReceived(l);
}
if (l == 12L) {
messageSequenceTracker.sequenceNumberReceived(l);
messageSequenceTracker.sequenceNumberReceived(l);
}
}
messageSequenceTracker.close();
// then
assertEquals(0, msgErrOutOfSeqCounter.getCount());
assertEquals(2, msgErrDuplicateCounter.getCount());
assertEquals(1, msgErrLossCounter.getCount());
}
}