mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
Merge pull request #2032 from nosqlbench/jshook/nosqlbench-2031-blipblobs
jshook/nosqlbench-2031-blipblobs
This commit is contained in:
Jonathan Shook
GitHub
commit
416e18dbe6
@ -26,7 +26,7 @@
|
||||
|
||||
<properties>
|
||||
|
||||
<revision>5.21.3-SNAPSHOT</revision>
|
||||
<revision>5.21.4-SNAPSHOT</revision>
|
||||
<!-- Set this level to override the logging level for tests during build -->
|
||||
<project.testlevel>INFO</project.testlevel>
|
||||
<!-- Set this level to override the logging level for tests logging configuration during build -->
|
||||
|
||||
@ -16,6 +16,8 @@
|
||||
|
||||
package io.nosqlbench.adapters.api.scheduling;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Duration;
|
||||
@ -23,6 +25,7 @@ import java.time.Duration;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class TimeoutPredicateTest {
|
||||
private final static Logger logger = LogManager.getLogger(TimeoutPredicateTest.class);
|
||||
|
||||
@Test
|
||||
public void testNeverCompletablePreciate() {
|
||||
@ -90,13 +93,13 @@ public class TimeoutPredicateTest {
|
||||
);
|
||||
|
||||
TimeoutPredicate.Result<Long> result = canMakeIt.test();
|
||||
System.out.println(result);
|
||||
logger.info(result);
|
||||
|
||||
while (result.status() == TimeoutPredicate.Status.pending) {
|
||||
// canMakeIt.blockUntilNextInterval();
|
||||
result = canMakeIt.test();
|
||||
System.out.println(canMakeIt);
|
||||
System.out.println(result);
|
||||
logger.info(canMakeIt);
|
||||
logger.info(result);
|
||||
}
|
||||
|
||||
assertThat(result.status()).isEqualTo(TimeoutPredicate.Status.complete);
|
||||
|
||||
@ -26,6 +26,8 @@ import java.util.function.BiFunction;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static java.util.regex.Matcher.quoteReplacement;
|
||||
|
||||
/**
|
||||
* BindPointParser parses a user-provide string template into spans. It builds a simple list of
|
||||
* BindPoints, and provides both the parsed spans and the BindPoints in a result.
|
||||
@ -51,10 +53,10 @@ public class BindPointParser implements BiFunction<String, Map<String, String>,
|
||||
List<String> spans = new ArrayList<>();
|
||||
List<BindPoint> bindpoints = new ArrayList<>();
|
||||
|
||||
int genid=0;
|
||||
int genid = 0;
|
||||
while (m.find()) {
|
||||
String pre = template.substring(lastMatch, m.start());
|
||||
spans.add(pre);
|
||||
spans.add(unescape(pre));
|
||||
lastMatch = m.end();
|
||||
|
||||
String reference = m.group("reference");
|
||||
@ -64,21 +66,29 @@ public class BindPointParser implements BiFunction<String, Map<String, String>,
|
||||
bindpoints.add(BindPoint.of(reference, bindings.getOrDefault(reference, null), BindPoint.Type.reference));
|
||||
spans.add(reference);
|
||||
} else if (inline1 != null) {
|
||||
bindpoints.add(BindPoint.of(DEFINITION, inline1, BindPoint.Type.definition));
|
||||
bindpoints.add(BindPoint.of(DEFINITION, unescape(inline1), BindPoint.Type.definition));
|
||||
spans.add(inline1);
|
||||
} else if (inline2 != null) {
|
||||
bindpoints.add(BindPoint.of(DEFINITION, inline2, BindPoint.Type.definition));
|
||||
bindpoints.add(BindPoint.of(DEFINITION, unescape(inline2), BindPoint.Type.definition));
|
||||
spans.add(inline2);
|
||||
} else {
|
||||
throw new BasicError("Unable to parse: " + template);
|
||||
}
|
||||
|
||||
}
|
||||
spans.add(lastMatch >= 0 ? template.substring(lastMatch) : template);
|
||||
spans.add(lastMatch >= 0 ? unescape(template.substring(lastMatch)) : unescape(template));
|
||||
|
||||
for (String span : spans) {
|
||||
|
||||
}
|
||||
return new Result(spans, bindpoints);
|
||||
}
|
||||
|
||||
private static String unescape(String s) {
|
||||
String s1 = s.replaceAll("\\\\n", "\n");
|
||||
return s1;
|
||||
}
|
||||
|
||||
public final static class Result {
|
||||
|
||||
private final List<String> spans;
|
||||
|
||||
@ -88,4 +88,18 @@ public class BindPointParserTest {
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUnescapingNewlines() {
|
||||
BindPointParser bpp = new BindPointParser();
|
||||
assertThat(bpp.apply("a{{Template(\"-{}-\",Combinations(\"a-z\"))}}\\nb", Map.of())).isEqualTo(
|
||||
new BindPointParser.Result(
|
||||
List.of("a","Template(\"-{}-\",Combinations(\"a-z\"))","\nb"),
|
||||
List.of(
|
||||
BindPoint.of(BindPointParser.DEFINITION,"Template(\"-{}-\",Combinations(\"a-z\"))", BindPoint.Type.definition)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -38,9 +38,9 @@ public class AliasSamplerDoubleInt implements DoubleToIntFunction {
|
||||
private final ByteBuffer stats; // tuples of double,int,int (unfair coin, direct pointers to referents)
|
||||
private final double slotCount; // The number of fair die-roll slotCount that contain unfair coin probabilities
|
||||
private static final int _r0=0;
|
||||
private static final int _r1=_r0+Double.BYTES;
|
||||
private static final int _r2=_r1+Integer.BYTES;
|
||||
public static int RECORD_LEN = _r2 + Integer.BYTES; // Record size for the above.
|
||||
private static final int _r1=_r0+Double.BYTES; // unfair coin
|
||||
private static final int _r2=_r1+Integer.BYTES; // + referent 1
|
||||
public static int RECORD_LEN = _r2 + Integer.BYTES; // + referent 2 = Record size for the above.
|
||||
|
||||
// for testing
|
||||
AliasSamplerDoubleInt(ByteBuffer stats) {
|
||||
|
||||
@ -61,7 +61,9 @@ import java.util.stream.Collectors;
|
||||
* the values will appear monotonically as you scan through the unit interval of all long values.
|
||||
* Specifically, 0L represents 0.0d in the unit interval on input, and Long.MAX_VALUE represents
|
||||
* 1.0 on the unit interval.) This mode is only recommended for advanced scenarios and should otherwise be
|
||||
* avoided. You will know if you need this mode.
|
||||
* avoided. You will know if you need this mode. For alias sampling, the values may not always occur
|
||||
* in the order specified due to the alias table construction. However, the values will be clustered in the order
|
||||
* they appear in that table.
|
||||
*
|
||||
*/
|
||||
@Categories(Category.general)
|
||||
|
||||
@ -0,0 +1,95 @@
|
||||
package io.nosqlbench.virtdata.library.basics.shared.distributions;
|
||||
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
|
||||
import io.nosqlbench.nb.api.errors.BasicError;
|
||||
import io.nosqlbench.virtdata.api.annotations.Categories;
|
||||
import io.nosqlbench.virtdata.api.annotations.Category;
|
||||
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
|
||||
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
|
||||
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
|
||||
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.HashInterval;
|
||||
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.HashRange;
|
||||
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.ScaledDouble;
|
||||
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.function.LongToDoubleFunction;
|
||||
import java.util.function.LongToIntFunction;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@ThreadSafeMapper
|
||||
@Categories(Category.distributions)
|
||||
public class WeightedInts implements LongToIntFunction {
|
||||
private final AliasSamplerDoubleInt sampler;
|
||||
private final LongToIntFunction function;
|
||||
|
||||
public WeightedInts(String spec, String... modifiers) {
|
||||
sampler = new AliasSamplerDoubleInt(parseWeights(spec));
|
||||
this.function = applyModifiers(sampler, modifiers);
|
||||
}
|
||||
|
||||
private LongToIntFunction applyModifiers(AliasSamplerDoubleInt aliasSampler, String[] modifiers) {
|
||||
|
||||
String mode = "hash";
|
||||
|
||||
for (String modifier : modifiers) {
|
||||
switch (modifier) {
|
||||
case "map":
|
||||
mode = "map";
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException("Unrecognized modifier: " + modifier);
|
||||
}
|
||||
}
|
||||
|
||||
if (mode.equals("hash")) {
|
||||
HashInterval f2 = new HashInterval(0.0d, 1.0d);
|
||||
return (long l) -> aliasSampler.applyAsInt(f2.applyAsDouble(l));
|
||||
} else if (mode.equals("map")) {
|
||||
ScaledDouble f1 = new ScaledDouble();
|
||||
return (long l) -> aliasSampler.applyAsInt(f1.applyAsDouble(l));
|
||||
} else {
|
||||
throw new BasicError("Unable to determine mapping mode for weighted ints function");
|
||||
}
|
||||
}
|
||||
|
||||
private final static Pattern weight = Pattern.compile(
|
||||
"(?<value>\\d+)(:(?<weight>[0-9.]+))?([; ,]+)?"
|
||||
);
|
||||
|
||||
private List<EvProbD> parseWeights(String spec) {
|
||||
List<EvProbD> events = new ArrayList<>();
|
||||
Matcher matcher = weight.matcher(spec);
|
||||
while (matcher.find()) {
|
||||
int value = Integer.parseInt(matcher.group("value"));
|
||||
String weightSpec = matcher.group("weight");
|
||||
double weight = (weightSpec != null) ? Double.parseDouble(weightSpec) : 1.0d;
|
||||
events.add(new EvProbD(value, weight));
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int applyAsInt(long value) {
|
||||
return function.applyAsInt(value);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double;
|
||||
|
||||
import io.nosqlbench.virtdata.api.annotations.Categories;
|
||||
import io.nosqlbench.virtdata.api.annotations.Category;
|
||||
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
|
||||
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
|
||||
|
||||
import java.util.function.LongToDoubleFunction;
|
||||
|
||||
/**
|
||||
* Create a double value from a hashed long, over the valid range of long inputs.
|
||||
* This version provides a strict unit interval value, not a unit range value.
|
||||
* That is, it can yield any value between 0.0 and 1.0, EXCEPT 1.0.
|
||||
*/
|
||||
@ThreadSafeMapper
|
||||
@Categories({Category.general})
|
||||
public class HashInterval implements LongToDoubleFunction {
|
||||
|
||||
private final double min;
|
||||
private final double max;
|
||||
private final double interval;
|
||||
private final static double MAX_DOUBLE_VIA_LONG_PHI = ((double) Long.MAX_VALUE)+1026d;
|
||||
private final Hash hash = new Hash();
|
||||
|
||||
public HashInterval(double min, double max) {
|
||||
this.min = min;
|
||||
this.max = max;
|
||||
this.interval = max - min;
|
||||
if (min>max) {
|
||||
throw new RuntimeException("min must be less than or equal to max");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public double applyAsDouble(long value) {
|
||||
long hashed = hash.applyAsLong(value);
|
||||
double unitScale = ((double) hashed) / MAX_DOUBLE_VIA_LONG_PHI;
|
||||
double valueScaled =interval*unitScale + min;
|
||||
return valueScaled;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double;
|
||||
|
||||
import io.nosqlbench.virtdata.api.annotations.Categories;
|
||||
import io.nosqlbench.virtdata.api.annotations.Category;
|
||||
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
|
||||
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
|
||||
|
||||
import java.util.function.LongToDoubleFunction;
|
||||
|
||||
/*
|
||||
* <p>This function attempts to take a double
|
||||
* unit interval value from a long/long division over the whole
|
||||
* range of long values but via double value types, thus providing
|
||||
* a very linear sample. This means that the range of double
|
||||
* values to be accessed will not fall along all possible doubles,
|
||||
* but will still provide suitable values for ranges close to
|
||||
* high-precision points in the IEEE floating point number line.
|
||||
* This suffices for most reasonable ranges in practice outside
|
||||
* of scientific computing, where large exponents put adjacent
|
||||
* IEEE floating point values much further apart.</p>
|
||||
*
|
||||
* <p>This should be consider the default double range sampling
|
||||
* function for most uses, when the exponent is not needed for
|
||||
* readability.</p>
|
||||
*/
|
||||
|
||||
/**
|
||||
* Return the double value closest to the fraction (input) / (Long.MAX_VALUE).
|
||||
* This is essentially a scaling function from Long to Double over the range of
|
||||
* positive longs to the double unit interval, so [0.0d - 1.0d)
|
||||
*/
|
||||
@ThreadSafeMapper
|
||||
@Categories({Category.general})
|
||||
public class ScaledDouble implements LongToDoubleFunction {
|
||||
|
||||
public final static double MAX_DOUBLE_VIA_LONG_PHI = ((double) Long.MAX_VALUE)+1026d;
|
||||
|
||||
public ScaledDouble(){}
|
||||
|
||||
@Override
|
||||
public double applyAsDouble(long value) {
|
||||
double unitScaled = ((double) value) / MAX_DOUBLE_VIA_LONG_PHI;
|
||||
return unitScaled;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,101 @@
|
||||
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_int;
|
||||
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
|
||||
import io.nosqlbench.virtdata.api.annotations.Categories;
|
||||
import io.nosqlbench.virtdata.api.annotations.Category;
|
||||
import io.nosqlbench.virtdata.api.annotations.Example;
|
||||
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.function.LongToIntFunction;
|
||||
import java.util.function.LongUnaryOperator;
|
||||
|
||||
/**
|
||||
* SequenceOf bindings allow you to specify an order and count of a set of values which will then
|
||||
* be repeated in that order.
|
||||
*/
|
||||
@ThreadSafeMapper
|
||||
@Categories(Category.experimental)
|
||||
public class SequenceOf implements LongToIntFunction {
|
||||
private final static Logger logger = LogManager.getLogger(SequenceOf.class);
|
||||
|
||||
final int[] sequence;
|
||||
/**
|
||||
* <p>
|
||||
* This function produces values from a lookup table for direct control of numerical sequences.
|
||||
* The sequence spec is a string containing the sequence values and their occurences, defaulting to 1 each.
|
||||
* Example: "1:6 2 3 4 5", which means "1 at a relative frequency of 6 and 2, 3, 4, and 5 at a relative frequency
|
||||
* of 1 each. This will yield pattern "1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, ..."
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Each implementation of {@link SequenceOf} must include a type sigil as the first parameter to disambiguate
|
||||
* it from the others.
|
||||
* </p>
|
||||
*
|
||||
* @param ignored
|
||||
* any long value, discarded after signature matching. The exampleValue is thrown away, but is necessary for
|
||||
* matching the right version of SequenceOf.
|
||||
* @param spec
|
||||
* A string of numbers separated by spaces, semicolons, or commas. This is the sequence spec..
|
||||
*/
|
||||
@Example({"SequenceOf(1L,'3:3 2:2 1:1')","Generate sequence 3,3,3,2,2,1"})
|
||||
@Example({"SequenceOf(1L,'1000:99 1000000:1')","Generate sequence 1000 (99 times) and then 1000000 (1 time)"})
|
||||
public SequenceOf(int ignored, String spec) {
|
||||
this.sequence = parseSequence(spec);
|
||||
|
||||
}
|
||||
|
||||
public static int[] parseSequence(String input) {
|
||||
String[] entries = input.split("[;, ]");
|
||||
int[][] subarys = new int[entries.length][];
|
||||
int entry=0;
|
||||
int size=0;
|
||||
|
||||
String[] parts;
|
||||
for (int i = 0; i < entries.length; i++) {
|
||||
parts = entries[i].split(":");
|
||||
int value = Integer.parseInt(parts[0]);
|
||||
int count = (parts.length==1) ? 1 : Integer.parseInt(parts[1]);
|
||||
int[] segment = new int[count];
|
||||
Arrays.fill(segment,value);
|
||||
subarys[entry++]=segment;
|
||||
size+=segment.length;
|
||||
}
|
||||
if (size>1E6) {
|
||||
logger.warn("The sequence you have specified is very large, which may cause problems. You should consider" +
|
||||
" a different approach for this type of function.");
|
||||
}
|
||||
int[] sequence = new int[size];
|
||||
int offset=0;
|
||||
for (int[] subary : subarys) {
|
||||
System.arraycopy(subary,0,sequence,offset,subary.length);
|
||||
offset+=subary.length;
|
||||
}
|
||||
return sequence;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int applyAsInt(long value) {
|
||||
return sequence[(int) value % sequence.length];
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,100 @@
|
||||
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_long;
|
||||
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
|
||||
import io.nosqlbench.virtdata.api.annotations.Categories;
|
||||
import io.nosqlbench.virtdata.api.annotations.Category;
|
||||
import io.nosqlbench.virtdata.api.annotations.Example;
|
||||
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.function.LongUnaryOperator;
|
||||
|
||||
/**
|
||||
* SequenceOf bindings allow you to specify an order and count of a set of values which will then
|
||||
* be repeated in that order.
|
||||
*/
|
||||
@ThreadSafeMapper
|
||||
@Categories(Category.experimental)
|
||||
public class SequenceOf implements LongUnaryOperator {
|
||||
private final static Logger logger = LogManager.getLogger(SequenceOf.class);
|
||||
|
||||
final long[] sequence;
|
||||
/**
|
||||
* <p>
|
||||
* This function produces values from a lookup table for direct control of numerical sequences.
|
||||
* The sequence spec is a string containing the sequence values and their occurences, defaulting to 1 each.
|
||||
* Example: "1:6 2 3 4 5", which means "1 at a relative frequency of 6 and 2, 3, 4, and 5 at a relative frequency
|
||||
* of 1 each. This will yield pattern "1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, ..."
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Each implementation of {@link SequenceOf} must include a type sigil as the first parameter to disambiguate
|
||||
* it from the others.
|
||||
* </p>
|
||||
*
|
||||
* @param ignored
|
||||
* any long value, discarded after signature matching. The exampleValue is thrown away, but is necessary for
|
||||
* matching the right version of SequenceOf.
|
||||
* @param spec
|
||||
* A string of numbers separated by spaces, semicolons, or commas. This is the sequence spec..
|
||||
*/
|
||||
@Example({"SequenceOf(1L,'3:3 2:2 1:1')","Generate sequence 3,3,3,2,2,1"})
|
||||
@Example({"SequenceOf(1L,'1000:99 1000000:1')","Generate sequence 1000 (99 times) and then 1000000 (1 time)"})
|
||||
public SequenceOf(long ignored, String spec) {
|
||||
this.sequence = parseSequence(spec);
|
||||
|
||||
}
|
||||
|
||||
public static long[] parseSequence(String input) {
|
||||
String[] entries = input.split("[;, ]");
|
||||
long[][] subarys = new long[entries.length][];
|
||||
int entry=0;
|
||||
int size=0;
|
||||
|
||||
String[] parts;
|
||||
for (int i = 0; i < entries.length; i++) {
|
||||
parts = entries[i].split(":");
|
||||
long value = Long.parseLong(parts[0]);
|
||||
int count = (parts.length==1) ? 1 : Integer.parseInt(parts[1]);
|
||||
long[] segment = new long[count];
|
||||
Arrays.fill(segment,value);
|
||||
subarys[entry++]=segment;
|
||||
size+=segment.length;
|
||||
}
|
||||
if (size>1E6) {
|
||||
logger.warn("The sequence you have specified is very large, which may cause problems. You should consider" +
|
||||
" a different approach for this type of function.");
|
||||
}
|
||||
long[] sequence = new long[size];
|
||||
int offset=0;
|
||||
for (long[] subary : subarys) {
|
||||
System.arraycopy(subary,0,sequence,offset,subary.length);
|
||||
offset+=subary.length;
|
||||
}
|
||||
return sequence;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long applyAsLong(long operand) {
|
||||
return sequence[(int) operand % sequence.length];
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,74 @@
|
||||
package io.nosqlbench.virtdata.library.basics.shared.distributions;
|
||||
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
|
||||
import org.assertj.core.data.Offset;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
public class WeightedIntsTest {
|
||||
@Test
|
||||
public void testWeightedInts() {
|
||||
WeightedInts weightedInts = new WeightedInts("10:10 20:20: 30:30 40:40", "map");
|
||||
assertThat(weightedInts.applyAsInt(0L)).isEqualTo(10);
|
||||
assertThat(weightedInts.applyAsInt(1L)).isEqualTo(10);
|
||||
assertThat(weightedInts.applyAsInt(Long.MAX_VALUE)).isEqualTo(40);
|
||||
assertThat(weightedInts.applyAsInt(Long.MAX_VALUE-1L)).isEqualTo(40);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDistributionError() {
|
||||
WeightedInts weightedInts = new WeightedInts("10:10 20:20: 30:30 40:40");
|
||||
double[] weights =new double[100];
|
||||
|
||||
long count = 1000000;
|
||||
for (long i = 0; i < count; i++) {
|
||||
int value = weightedInts.applyAsInt(i);
|
||||
weights[value]++;
|
||||
}
|
||||
|
||||
// Verify that each label has been sampled at a frequency which is within
|
||||
// 0.1% of the expected value.
|
||||
Offset offset = Offset.offset(((double)count)/1000d);
|
||||
|
||||
assertThat(weights[10]).isCloseTo(((double)count)*(10.d/100.d), offset);
|
||||
assertThat(weights[20]).isCloseTo(((double)count)*(20.d/100.d), offset);
|
||||
assertThat(weights[30]).isCloseTo(((double)count)*(30.d/100.d), offset);
|
||||
assertThat(weights[40]).isCloseTo(((double)count)*(40.d/100.d), offset);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
@Disabled("leaving here to show boundary check logic for PHI")
|
||||
public void boundaryCheck() {
|
||||
for (long i = 0; i < 100000000; i++) {
|
||||
double pad = ((double) i)*1.0;
|
||||
double denominator = ((double) Long.MAX_VALUE) + pad;
|
||||
double scaled = ((double) Long.MAX_VALUE) / denominator;
|
||||
if (scaled < 1.0d) {
|
||||
System.out.println("phi:" + i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -17,6 +17,7 @@
|
||||
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_long;
|
||||
|
||||
import io.nosqlbench.nb.api.errors.BasicError;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@ -37,4 +38,15 @@ public class HashIntervalTest {
|
||||
assertThatExceptionOfType(BasicError.class)
|
||||
.isThrownBy(() -> new HashInterval(3L, 3L));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Disabled("exhaustive boundary check, expensive")
|
||||
public void testDoubleHashIntervalBounds() {
|
||||
io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.HashInterval hi = new io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.HashInterval(0.0, 1.0);
|
||||
for (long i = 0; i < 100000000; i++) {
|
||||
double v = hi.applyAsDouble(i);
|
||||
assertThat(v).isGreaterThanOrEqualTo(0.0d);
|
||||
assertThat(v).isLessThan(1.0d);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,97 @@
|
||||
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_long;
|
||||
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
public class SequenceOfTest {
|
||||
|
||||
@Test
|
||||
public void testSequenceSimpleToLong() {
|
||||
SequenceOf so = new SequenceOf(1L,"0 1 2 3 4 5 6 7 8 9");
|
||||
long[] results = new long[10];
|
||||
for (int i = 0; i < 10; i++) {
|
||||
results[i] = so.applyAsLong(i);
|
||||
}
|
||||
for (int i = 0; i < 10; i++) {
|
||||
assertEquals(i,results[i]);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSequenceSimpleToInt() {
|
||||
io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.SequenceOf so = new io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.SequenceOf(1,"0 1 2 3 4 5 6 7 8 9");
|
||||
long[] results = new long[10];
|
||||
for (int i = 0; i < 10; i++) {
|
||||
results[i] = so.applyAsInt(i);
|
||||
}
|
||||
for (int i = 0; i < 10; i++) {
|
||||
assertEquals(i,results[i]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testSequenceWeightedToLong() {
|
||||
SequenceOf so = new SequenceOf(1L,"0:6 1 2 3 4");
|
||||
int samples = 100;
|
||||
long[] results = new long[samples];
|
||||
for (int i = 0; i < samples; i++) {
|
||||
results[i]=so.applyAsLong(i);
|
||||
}
|
||||
assertThat(results[0]).isEqualTo(0);
|
||||
assertThat(results[1]).isEqualTo(0);
|
||||
assertThat(results[2]).isEqualTo(0);
|
||||
assertThat(results[3]).isEqualTo(0);
|
||||
assertThat(results[4]).isEqualTo(0);
|
||||
assertThat(results[5]).isEqualTo(0);
|
||||
assertThat(results[6]).isEqualTo(1);
|
||||
assertThat(results[7]).isEqualTo(2);
|
||||
assertThat(results[8]).isEqualTo(3);
|
||||
assertThat(results[9]).isEqualTo(4);
|
||||
assertThat(results[10]).isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSequenceWeightedToInt() {
|
||||
io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.SequenceOf so = new io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.SequenceOf(1,"0:6 1 2 3 4");
|
||||
int samples = 100;
|
||||
long[] results = new long[samples];
|
||||
for (int i = 0; i < samples; i++) {
|
||||
results[i]=so.applyAsInt(i);
|
||||
}
|
||||
assertThat(results[0]).isEqualTo(0);
|
||||
assertThat(results[1]).isEqualTo(0);
|
||||
assertThat(results[2]).isEqualTo(0);
|
||||
assertThat(results[3]).isEqualTo(0);
|
||||
assertThat(results[4]).isEqualTo(0);
|
||||
assertThat(results[5]).isEqualTo(0);
|
||||
assertThat(results[6]).isEqualTo(1);
|
||||
assertThat(results[7]).isEqualTo(2);
|
||||
assertThat(results[8]).isEqualTo(3);
|
||||
assertThat(results[9]).isEqualTo(4);
|
||||
assertThat(results[10]).isEqualTo(0);
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -0,0 +1,93 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.virtdata.library.curves4.discrete.int_int;
|
||||
|
||||
import io.nosqlbench.virtdata.api.annotations.Categories;
|
||||
import io.nosqlbench.virtdata.api.annotations.Category;
|
||||
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
|
||||
import org.apache.commons.math4.legacy.distribution.EnumeratedIntegerDistribution;
|
||||
|
||||
/**
|
||||
* Create a sampler based on enumeration of integer values an sample over them
|
||||
* using the EnumeratedInts distribution curve provided by Apache Commons Math.
|
||||
* This version will roughly produce the distribution, but since it also relies on
|
||||
* interpolation by default, non-step values may appear at low frequencies. If this
|
||||
* is a desired effect, then this function is suitable. For example: consider this
|
||||
* result:
|
||||
* <pre>{@code
|
||||
* nb5 run driver=stdout op="{{EnumeratedInts('10:10 20:20 30:30 40:40')}}\n" cycles=10000 | sort -n | uniq -c
|
||||
* 1 STDOUT0 (pending,current,complete)=(0,0,10000) 100.00% (last report)
|
||||
* 1 9
|
||||
* 1036 10
|
||||
* 2 11
|
||||
* 2 13
|
||||
* 1 14
|
||||
* 3 15
|
||||
* 2 16
|
||||
* 1 18
|
||||
* 1 19
|
||||
* 1937 20
|
||||
* 1 21
|
||||
* 1 23
|
||||
* 1 24
|
||||
* 1 25
|
||||
* 1 28
|
||||
* 1 29
|
||||
* 3077 30
|
||||
* 1 31
|
||||
* 1 33
|
||||
* 1 34
|
||||
* 2 35
|
||||
* 1 37
|
||||
* 1 39
|
||||
* 3924 40
|
||||
* }</pre>
|
||||
*
|
||||
* The values here which are not multiples of 10 are not specified, yet the appear. For some testing, this is
|
||||
* helpful as a fuzzer, but for more precise step-value sampling, see {@link AliasSampler}
|
||||
*/
|
||||
@ThreadSafeMapper
|
||||
@Categories({Category.distributions})
|
||||
public class EnumeratedInts extends IntToIntDiscreteCurve {
|
||||
public EnumeratedInts(String data, String... mods) {
|
||||
super(new EnumeratedIntegerDistribution(parseIntLabels(data), parseDoubleWeights(data)), mods);
|
||||
}
|
||||
|
||||
public static int[] parseIntLabels(String input) {
|
||||
String[] entries = input.split("[;, ]");
|
||||
int[] elements = new int[entries.length];
|
||||
String[] parts;
|
||||
for (int i = 0; i < entries.length; i++) {
|
||||
parts = entries[i].split(":");
|
||||
elements[i] = Integer.parseInt(parts[0]);
|
||||
}
|
||||
return elements;
|
||||
}
|
||||
|
||||
public static double[] parseDoubleWeights(String input) {
|
||||
String[] entries = input.split("[;, ]");
|
||||
double[] weights = new double[entries.length];
|
||||
String[] parts;
|
||||
for (int i = 0; i < entries.length; i++) {
|
||||
parts = entries[i].split(":");
|
||||
weights[i] = parts.length==2 ? Double.parseDouble(parts[1]) : 1.0d;
|
||||
}
|
||||
return weights;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.virtdata.library.curves4.discrete.long_int;
|
||||
|
||||
import io.nosqlbench.virtdata.api.annotations.Categories;
|
||||
import io.nosqlbench.virtdata.api.annotations.Category;
|
||||
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
|
||||
import io.nosqlbench.virtdata.library.curves4.discrete.int_int.IntToIntDiscreteCurve;
|
||||
import io.nosqlbench.virtdata.library.curves4.discrete.int_long.IntToLongDiscreteCurve;
|
||||
import org.apache.commons.math4.legacy.distribution.EnumeratedIntegerDistribution;
|
||||
|
||||
import static io.nosqlbench.virtdata.library.curves4.discrete.int_int.EnumeratedInts.parseDoubleWeights;
|
||||
import static io.nosqlbench.virtdata.library.curves4.discrete.int_int.EnumeratedInts.parseIntLabels;
|
||||
|
||||
@ThreadSafeMapper
|
||||
@Categories({Category.distributions})
|
||||
public class EnumeratedInts extends LongToIntDiscreteCurve {
|
||||
public EnumeratedInts(String data, String... mods) {
|
||||
super(new EnumeratedIntegerDistribution(parseIntLabels(data), parseDoubleWeights(data)), mods);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,89 @@
|
||||
package io.nosqlbench.virtdata.library.curves4.discrete.int_int;
|
||||
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.stream.StreamSupport;
|
||||
|
||||
import static java.util.Spliterators.spliterator;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
import org.assertj.core.data.Offset;
|
||||
import org.junit.jupiter.api.Test;
|
||||
public class EnumeratedIntsTest {
|
||||
|
||||
@Test
|
||||
public void EnumeratedIntsToInt() {
|
||||
EnumeratedInts ei = new EnumeratedInts("0 1 2 3 4");
|
||||
double counts[] = new double[5];
|
||||
Arrays.fill(counts,0.0d);
|
||||
int samples = 1000;
|
||||
for (int i = 0; i < samples; i++) {
|
||||
int v = ei.applyAsInt(i);
|
||||
assertThat(v).isGreaterThanOrEqualTo(0);
|
||||
assertThat(v).isLessThanOrEqualTo(4);
|
||||
counts[v] += 1.0d;
|
||||
}
|
||||
for (double count : counts) {
|
||||
assertThat(count/samples).isCloseTo((count/(double) samples), Offset.offset(0.01d));
|
||||
}
|
||||
StreamSupport.stream(spliterator(counts,0),false).forEach(System.out::println);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void EnumeratedIntsToLong() {
|
||||
io.nosqlbench.virtdata.library.curves4.discrete.long_int.EnumeratedInts ei =
|
||||
new io.nosqlbench.virtdata.library.curves4.discrete.long_int.EnumeratedInts ("0 1 2 3 4");
|
||||
double counts[] = new double[5];
|
||||
Arrays.fill(counts,0.0d);
|
||||
int samples = 1000;
|
||||
for (int i = 0; i < samples; i++) {
|
||||
long v = ei.applyAsInt(i);
|
||||
assertThat(v).isGreaterThanOrEqualTo(0);
|
||||
assertThat(v).isLessThanOrEqualTo(4);
|
||||
counts[(int) v] += 1.0d;
|
||||
}
|
||||
for (double count : counts) {
|
||||
assertThat(count/samples).isCloseTo((count/(double) samples), Offset.offset(0.01d));
|
||||
}
|
||||
StreamSupport.stream(spliterator(counts,0),false).forEach(System.out::println);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void EnumeratedLongToInt() {
|
||||
io.nosqlbench.virtdata.library.curves4.discrete.long_int.EnumeratedInts ei =
|
||||
new io.nosqlbench.virtdata.library.curves4.discrete.long_int.EnumeratedInts ("0 1 2 3 4");
|
||||
double counts[] = new double[5];
|
||||
Arrays.fill(counts,0.0d);
|
||||
int samples = 1000;
|
||||
for (int i = 0; i < samples; i++) {
|
||||
long v = ei.applyAsInt(i);
|
||||
assertThat(v).isGreaterThanOrEqualTo(0);
|
||||
assertThat(v).isLessThanOrEqualTo(4);
|
||||
counts[(int) v] += 1.0d;
|
||||
}
|
||||
for (double count : counts) {
|
||||
assertThat(count/samples).isCloseTo((count/(double) samples), Offset.offset(0.01d));
|
||||
}
|
||||
StreamSupport.stream(spliterator(counts,0),false).forEach(System.out::println);
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user