Skip to content

Commit

Permalink
Merge pull request #576 from HASMAC-AS/GH-573-fix-calc-hasmac
Browse files Browse the repository at this point in the history
  • Loading branch information
@ate47
ate47 authored Jan 15, 2025
2 parents 3327de3 + 751053b commit 6c8fa6b
Show file tree
Hide file tree
Showing 4 changed files with 155 additions and 23 deletions.
2 changes: 1 addition & 1 deletion ...int-core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/Bitmap375Big.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public class Bitmap375Big extends Bitmap64Big {

private static final Logger logger = LoggerFactory.getLogger(Bitmap375Big.class);

private static final boolean oldBinarySearch;
public static final boolean oldBinarySearch;

static {
// check if the system property "useOldBinarySeearch" is set to true
Expand Down
35 changes: 23 additions & 12 deletions ...int-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/AbstractLongArray.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package com.the_qa_company.qendpoint.core.util.disk;

import com.the_qa_company.qendpoint.core.compact.bitmap.Bitmap375Big;
import com.the_qa_company.qendpoint.core.util.BitUtil;

public abstract class AbstractLongArray implements LongArray {

private static final int ESTIMATED_LOCATION_ARRAY_SIZE;
static final int ESTIMATED_LOCATION_ARRAY_SIZE;

static {
// get total amount of memory that this java program is allowed to use
Expand All @@ -26,7 +27,7 @@ public abstract class AbstractLongArray implements LongArray {
private final long[] estimatedLocationMin = new long[ESTIMATED_LOCATION_ARRAY_SIZE];
private final long[] estimatedLocation = new long[ESTIMATED_LOCATION_ARRAY_SIZE];

private long estimatedLocationBucketSize;
private long estimatedLocationBucketSize = 1;

long maxValue = 1;

Expand All @@ -35,15 +36,10 @@ public long getEstimatedLocationArrayBucketSize() {
return estimatedLocationBucketSize;
}

private void updateEstimatedLocationArrayBucketSize() {
//int minBucketSize = (int) (maxValue / ESTIMATED_LOCATION_ARRAY_SIZE);
//// we want to have the next power of 2
//int next = 1;
//while (next < minBucketSize) {
// next <<= 1;
//}
// this.estimatedLocationBucketSize = next;
this.estimatedLocationBucketSize = ((1L << BitUtil.log2(maxValue)) - 1) / ESTIMATED_LOCATION_ARRAY_SIZE + 1;
void updateEstimatedLocationArrayBucketSize() {
this.estimatedLocationBucketSize = maxValue / ESTIMATED_LOCATION_ARRAY_SIZE + 1;
// this.estimatedLocationBucketSize = ((1L << BitUtil.log2(maxValue)) - 1) / ESTIMATED_LOCATION_ARRAY_SIZE + 1;
assert this.estimatedLocationBucketSize > 0;
}

@Override
Expand All @@ -63,17 +59,32 @@ public long[] getEstimatedLocationArrayMax() {

@Override
public void recalculateEstimatedValueLocation() {
if (Bitmap375Big.oldBinarySearch) {
return;
}

updateEstimatedLocationArrayBucketSize();
long estimatedLocationBucketSize = getEstimatedLocationArrayBucketSize();
long len = length();

for (long i = 0; i < len; i++) {
long val = get(i);
if (val == 0) {
// val shouldn't be zero, since this represents a value that
// does not exist
continue;
}

int index = (int) (val / estimatedLocationBucketSize + 1);
int index = getEstimatedLocationIndex(val);
if (index >= estimatedLocation.length || index >= estimatedLocationMax.length
|| index >= estimatedLocationMin.length) {
logger.warn("Index out of bounds for " + getClass().getSimpleName()
+ " when recalculateEstimatedValueLocation for value " + val + " and estimatedBucketSize "
+ estimatedLocationBucketSize + " and index " + index + " and estimatedLocation.length "
+ estimatedLocation.length + " and estimatedLocationMax.length " + estimatedLocationMax.length
+ " and estimatedLocationMin.length " + estimatedLocationMin.length);
continue;
}
estimatedLocationMax[index] = Math.max(estimatedLocationMax[index], i);
if (estimatedLocationMin[index] == 0) {
estimatedLocationMin[index] = i;
Expand Down
33 changes: 23 additions & 10 deletions qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LongArray.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,9 +243,12 @@ default long getEstimatedLocationArrayBucketSize() {
}

default long getEstimatedLocationLowerBound(long val) {
int index = (int) (val / getEstimatedLocationArrayBucketSize() + 1);
if (index - 1 >= 0) {
long t = getEstimatedLocationArrayMax()[index - 1];
int index = getEstimatedLocationIndex(val);
// move to the index below to get the estimated lower bound
index -= 1;
long[] estimatedLocationArrayMax = getEstimatedLocationArrayMax();
if (index >= 0 && index < estimatedLocationArrayMax.length) {
long t = estimatedLocationArrayMax[index];
if (t > 0) {
return t;
}
Expand All @@ -254,10 +257,12 @@ default long getEstimatedLocationLowerBound(long val) {
}

default long getEstimatedLocationUpperBound(long val) {
int index = (int) (val / getEstimatedLocationArrayBucketSize() + 1);
int index = getEstimatedLocationIndex(val);
// move to the index above to get the upper bound
index += 1;
long[] estimatedLocationMin = getEstimatedLocationArrayMin();
if (index + 1 < estimatedLocationMin.length) {
long t = estimatedLocationMin[index + 1];
if (index < estimatedLocationMin.length && index >= 0) {
long t = estimatedLocationMin[index];
if (t > 0) {
return Math.min(length(), t);
}
Expand All @@ -267,10 +272,10 @@ default long getEstimatedLocationUpperBound(long val) {
}

default long getEstimatedLocation(long val, long min, long max) {
int index = (int) (val / getEstimatedLocationArrayBucketSize() + 1);
int index = getEstimatedLocationIndex(val);
var estimatedLocation = getEstimatedLocationArray();

if (index >= estimatedLocation.length) {
if (index >= estimatedLocation.length || index < 0) {
return (min + max) / 2;
}
long t = estimatedLocation[index];
Expand All @@ -281,15 +286,23 @@ default long getEstimatedLocation(long val, long min, long max) {
}
}

default int getEstimatedLocationIndex(long val) {
long estimatedLocationArrayBucketSize = getEstimatedLocationArrayBucketSize();
if (estimatedLocationArrayBucketSize <= 0) {
return Integer.MIN_VALUE;
}
return (int) (val / getEstimatedLocationArrayBucketSize());
}

default void recalculateEstimatedValueLocation() {
logger.info("Class {} does not support recalculateEstimatedValueLocation()",
this.getClass().getCanonicalName());
}

default void updateEstimatedValueLocation(long val, long min) {
int index = (int) (val / getEstimatedLocationArrayBucketSize() + 1);
int index = getEstimatedLocationIndex(val);
long[] estimatedLocation = getEstimatedLocationArray();
if (index >= estimatedLocation.length) {
if (index >= estimatedLocation.length || index < 0) {
return;
}
estimatedLocation[index] = min;
Expand Down
108 changes: 108 additions & 0 deletions ...core/src/test/java/com/the_qa_company/qendpoint/core/util/disk/AbstractLongArrayTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package com.the_qa_company.qendpoint.core.util.disk;

import org.junit.Assert;
import org.junit.Test;

import java.io.IOException;
import java.util.Random;

public class AbstractLongArrayTest {

@Test
public void updateEstimatedLocationArrayBucketSize() {

Random random = new Random();
AbstractLongArray longArray = new AbstractLongArray() {
@Override
public long get(long index) {
return Math.max(0, maxValue - index);
}

@Override
public long length() {
return AbstractLongArray.ESTIMATED_LOCATION_ARRAY_SIZE * 1024L;
}

@Override
public int sizeOf() {
return 0;
}

@Override
public void resize(long newSize) throws IOException {

}

@Override
public void clear() {

}

@Override
protected void innerSet(long index, long value) {

}
};

longArray.getEstimatedLocation(Long.MAX_VALUE, -1, Long.MAX_VALUE);
longArray.getEstimatedLocationLowerBound(Long.MAX_VALUE);
longArray.getEstimatedLocationUpperBound(Long.MAX_VALUE);

longArray.getEstimatedLocation(-1, -1, Long.MAX_VALUE);
longArray.getEstimatedLocationLowerBound(-1);
longArray.getEstimatedLocationUpperBound(-1);

longArray.getEstimatedLocation(0, -1, Long.MAX_VALUE);
longArray.getEstimatedLocationLowerBound(0);
longArray.getEstimatedLocationUpperBound(0);

longArray.recalculateEstimatedValueLocation();
longArray.getEstimatedLocation(Long.MAX_VALUE, -1, Long.MAX_VALUE);
longArray.getEstimatedLocationLowerBound(Long.MAX_VALUE);
longArray.getEstimatedLocationUpperBound(Long.MAX_VALUE);

longArray.getEstimatedLocation(-1, -1, Long.MAX_VALUE);
longArray.getEstimatedLocationLowerBound(-1);
longArray.getEstimatedLocationUpperBound(-1);

longArray.getEstimatedLocation(0, -1, Long.MAX_VALUE);
longArray.getEstimatedLocationLowerBound(0);
longArray.getEstimatedLocationUpperBound(0);

for (long i = 0; i < (AbstractLongArray.ESTIMATED_LOCATION_ARRAY_SIZE * 1024L) + 3; i++) {
testMaxValue(longArray, i);
}

longArray.recalculateEstimatedValueLocation();

for (long i = 1; i > 0 && i < (Long.MAX_VALUE - 1); i *= 2) {
testMaxValue(longArray, i);
}

longArray.recalculateEstimatedValueLocation();

testMaxValue(longArray, Long.MAX_VALUE);
System.out.println();

longArray.recalculateEstimatedValueLocation();

long estimatedLocation = longArray.getEstimatedLocation(Long.MAX_VALUE, -1, Long.MAX_VALUE);
long estimatedLocationLowerBound = longArray.getEstimatedLocationLowerBound(Long.MAX_VALUE);
long estimatedLocationUpperBound = longArray.getEstimatedLocationUpperBound(Long.MAX_VALUE);
}

private static void testMaxValue(AbstractLongArray longArray, long value) {

longArray.maxValue = value;

longArray.updateEstimatedLocationArrayBucketSize();

long estimatedLocationArrayBucketSize = longArray.getEstimatedLocationArrayBucketSize();

Assert.assertTrue(estimatedLocationArrayBucketSize + "", estimatedLocationArrayBucketSize > 0);

longArray.getEstimatedLocation(value, -1, Long.MAX_VALUE);
longArray.getEstimatedLocationLowerBound(value);
longArray.getEstimatedLocationUpperBound(value);
}
}

0 comments on commit 6c8fa6b

Please sign in to comment.