IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into listfuncs

Browse Source
This commit is contained in:
Jonathan Shook
2020-07-07 10:51:12 -05:00
parent 70c2866115 7461f0e8b8
commit 0de0cc7d69
784 changed files with 22555 additions and 4070 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
virtdata-lib-basics/pom.xml
View File

@@ -7,7 +7,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>3.12.83-SNAPSHOT</version>
<version>3.12.127-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -20,7 +20,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-api</artifactId>
<version>3.12.83-SNAPSHOT</version>
<version>3.12.127-SNAPSHOT</version>
</dependency>
<dependency>

18
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/CSVFrequencySampler.java
View File

@@ -49,12 +49,9 @@ import java.util.function.LongFunction;
@ThreadSafeMapper
public class CSVFrequencySampler implements LongFunction<String> {
private final String filename;
private final String columnName;
private final String[] lines;
private final AliasSamplerDoubleInt sampler;
private Hash hash;
private final Hash hash;
/**
* Create a sampler of strings from the given CSV file. The CSV file must have plain CSV headers
@@ -64,8 +61,7 @@ public class CSVFrequencySampler implements LongFunction<String> {
*/
@Example({"CSVFrequencySampler('values.csv','modelno')","Read values.csv, count the frequency of values in 'modelno' column, and sample from this column proportionally"})
public CSVFrequencySampler(String filename, String columnName) {
this.filename = filename;
this.columnName = columnName;
String filename1 = filename;
this.hash=new Hash();
@@ -78,13 +74,15 @@ public class CSVFrequencySampler implements LongFunction<String> {
CSVParser csvdata = NBIO.readFileCSV(filename);
Frequency freq = new Frequency();
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(columnName);
freq.addValue(value);
values.add(value);
if (csvdatum.get(columnName) != null) {
String value = csvdatum.get(columnName);
freq.addValue(value);
values.add(value);
}
}
int i = 0;
for (String value : values) {
frequencies.add(new EvProbD(i++,Double.valueOf(freq.getCount(value))));
frequencies.add(new EvProbD(i++, (double) freq.getCount(value)));
}
sampler = new AliasSamplerDoubleInt(frequencies);
lines = values.toArray(new String[0]);

8
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/DelimFrequencySampler.java
View File

@@ -81,9 +81,11 @@ public class DelimFrequencySampler implements LongFunction<String> {
CSVParser csvdata = NBIO.readFileDelimCSV(filename,delimiter);
Frequency freq = new Frequency();
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(columnName);
freq.addValue(value);
values.add(value);
if (csvdatum.get(columnName) != null) {
String value = csvdatum.get(columnName);
freq.addValue(value);
values.add(value);
}
}
int i = 0;
for (String value : values) {

10
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/WeightedStringsFromCSV.java
View File

@@ -76,10 +76,12 @@ public class WeightedStringsFromCSV implements LongFunction<String> {
}
CSVParser csvdata = NBIO.readFileCSV(filename);
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(valueColumn);
values.add(value);
String weight = csvdatum.get(weightColumn);
events.add(new EvProbD(values.size()-1,Double.valueOf(weight)));
if (csvdatum.get(valueColumn) != null && csvdatum.get(weightColumn) != null) {
String value = csvdatum.get(valueColumn);
values.add(value);
String weight = csvdatum.get(weightColumn);
events.add(new EvProbD(values.size() - 1, Double.valueOf(weight)));
}
}
}
sampler = new AliasSamplerDoubleInt(events);

49
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_double/to_bigdecimal/ToBigDecimal.java Normal file
View File

@@ -0,0 +1,49 @@
package io.nosqlbench.virtdata.library.basics.shared.from_double.to_bigdecimal;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.util.MathContextReader;
import java.math.BigDecimal;
import java.math.MathContext;
import java.math.RoundingMode;
import java.util.function.DoubleFunction;
import java.util.function.LongFunction;
@ThreadSafeMapper
@Categories(Category.conversion)
public class ToBigDecimal implements DoubleFunction<BigDecimal> {
private final MathContext mathContext;
@Example({"ToBigDecimal()", "Convert all double values to BigDecimal values with no limits (using MathContext" +
".UNLIMITED)"})
public ToBigDecimal() {
this.mathContext = MathContext.UNLIMITED;
}
/**
* Convert all input values to BigDecimal values with a specific MathContext.
* The value for context can be one of UNLIMITED,
* DECIMAL32, DECIMAL64, DECIMAL128, or any valid configuration supported by
* {@link MathContext#MathContext(String)}, such as {@code "precision=32 roundingMode=CEILING"}.
* In the latter form, roundingMode can be any valid value for {@link RoundingMode}, like
* UP, DOWN, CEILING, FLOOR, HALF_UP, HALF_DOWN, HALF_EVEN, or UNNECESSARY.
*/
@Example({"ToBigDecimal('DECIMAL32')", "IEEE 754R Decimal32 format, 7 digits, HALF_EVEN"})
@Example({"ToBigDecimal('DECIMAL64'),", "IEEE 754R Decimal64 format, 16 digits, HALF_EVEN"})
@Example({"ToBigDecimal('DECIMAL128')", "IEEE 754R Decimal128 format, 34 digits, HALF_EVEN"})
@Example({"ToBigDecimal('UNLIMITED')", "unlimited precision, HALF_UP"})
@Example({"ToBigDecimal('precision=17 roundingMode=UNNECESSARY')", "Custom precision with no rounding performed"})
public ToBigDecimal(String context) {
this.mathContext = MathContextReader.getMathContext(context);
}
@Override
public BigDecimal apply(double value) {
return new BigDecimal(value, mathContext);
}
}

48
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_int/to_bigdecimal/ToBigDecimal.java Normal file
View File

@@ -0,0 +1,48 @@
package io.nosqlbench.virtdata.library.basics.shared.from_int.to_bigdecimal;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.util.MathContextReader;
import java.math.BigDecimal;
import java.math.MathContext;
import java.math.RoundingMode;
import java.util.function.DoubleFunction;
import java.util.function.IntFunction;
@ThreadSafeMapper
@Categories(Category.conversion)
public class ToBigDecimal implements IntFunction<BigDecimal> {
private final MathContext mathContext;
@Example({"ToBigDecimal()", "Convert all int values to BigDecimal values with no limits (using MathContext" +
".UNLIMITED)"})
public ToBigDecimal() {
this.mathContext = MathContext.UNLIMITED;
}
/**
* Convert all input values to BigDecimal values with a specific MathContext.
* The value for context can be one of UNLIMITED,
* DECIMAL32, DECIMAL64, DECIMAL128, or any valid configuration supported by
* {@link MathContext#MathContext(String)}, such as {@code "precision=32 roundingMode=CEILING"}.
* In the latter form, roundingMode can be any valid value for {@link RoundingMode}, like
* UP, DOWN, CEILING, FLOOR, HALF_UP, HALF_DOWN, HALF_EVEN, or UNNECESSARY.
*/
@Example({"ToBigDecimal('DECIMAL32')", "IEEE 754R Decimal32 format, 7 digits, HALF_EVEN"})
@Example({"ToBigDecimal('DECIMAL64'),", "IEEE 754R Decimal64 format, 16 digits, HALF_EVEN"})
@Example({"ToBigDecimal('DECIMAL128')", "IEEE 754R Decimal128 format, 34 digits, HALF_EVEN"})
@Example({"ToBigDecimal('UNLIMITED')", "unlimited precision, HALF_UP"})
@Example({"ToBigDecimal('precision=17 roundingMode=UNNECESSARY')", "Custom precision with no rounding performed"})
public ToBigDecimal(String context) {
this.mathContext = MathContextReader.getMathContext(context);
}
@Override
public BigDecimal apply(int value) {
return new BigDecimal(value, mathContext);
}
}

68
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bigdecimal/ToBigDecimal.java Normal file
View File

@@ -0,0 +1,68 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_bigdecimal;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.util.MathContextReader;
import java.math.BigDecimal;
import java.math.MathContext;
import java.math.RoundingMode;
import java.util.function.LongFunction;
/**
* Convert values to BigDecimals at configurable scale or precision.
*
* <p>
* ToBigDecimal(...) functions which take whole-numbered inputs may have
* a scale parameter or a custom MathContext, but not both. The scale parameter
* is not supported for String or Double input forms.
* </p>
*/
@ThreadSafeMapper
@Categories(Category.conversion)
public class ToBigDecimal implements LongFunction<BigDecimal> {
private final MathContext mathContext;
private final int scale;
@Example({"ToBigDecimal()", "Convert all long values to whole-numbered BigDecimal values"})
public ToBigDecimal() {
this(0);
}
@Example({"ToBigDecimal(0)", "Convert all long values to whole-numbered BigDecimal values"})
@Example({"ToBigDecimal(2)", "Convert long 'pennies' BigDecimal with 2 digits after decimal point"})
public ToBigDecimal(int scale) {
this.scale = scale;
this.mathContext=null;
}
/**
* Convert all input values to BigDecimal values with a specific MathContext. This form is only
* supported for scale=0, meaning whole numbers. The value for context can be one of UNLIMITED,
* DECIMAL32, DECIMAL64, DECIMAL128, or any valid configuration supported by
* {@link MathContext#MathContext(String)}, such as {@code "precision=32 roundingMode=CEILING"}.
* In the latter form, roundingMode can be any valid value for {@link RoundingMode}, like
* UP, DOWN, CEILING, FLOOR, HALF_UP, HALF_DOWN, HALF_EVEN, or UNNECESSARY.
*/
@Example({"ToBigDecimal('DECIMAL32')","IEEE 754R Decimal32 format, 7 digits, HALF_EVEN"})
@Example({"ToBigDecimal('DECIMAL64'),","IEEE 754R Decimal64 format, 16 digits, HALF_EVEN"})
@Example({"ToBigDecimal('DECIMAL128')","IEEE 754R Decimal128 format, 34 digits, HALF_EVEN"})
@Example({"ToBigDecimal('UNLIMITED')","unlimited precision, HALF_UP"})
@Example({"ToBigDecimal('precision=17 roundingMode=UNNECESSARY')","Custom precision with no rounding performed"})
public ToBigDecimal(String context) {
this.scale=Integer.MIN_VALUE;
this.mathContext = MathContextReader.getMathContext(context);
}
@Override
public BigDecimal apply(long value) {
if (mathContext!=null) {
return new BigDecimal(value,mathContext);
} else {
return BigDecimal.valueOf(value,scale);
}
}
}

63
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/Set.java
View File

@@ -1,5 +1,6 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_collection;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
@@ -8,15 +9,14 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.HashSet;
import java.util.function.LongFunction;
import java.util.function.LongToIntFunction;
import java.util.function.LongUnaryOperator;
/**
* Create a {@code Set} from a long input based on two functions,
* the first to determine the set size, and the second to populate
* the set with object values. The input fed to the second function
* is incremented between elements.
*
* To create Sets of Strings from the String version of the same
* mapping functions, simply use {@link StringSet} instead.
* Create a {@code Set} from a long input based on two functions, the first to determine the set size, and the second to
* populate the set with object values. The input fed to the second function is incremented between elements.
* <p>
* To create Sets of Strings from the String version of the same mapping functions, simply use {@link StringSet}
* instead.
*/
@Categories({Category.collections})
@ThreadSafeMapper
@@ -31,13 +31,60 @@ public class Set implements LongFunction<java.util.Set<Object>> {
this.sizeFunc = sizeFunc;
this.valueFunc = valueFunc;
}
public Set(LongToIntFunction sizeFunc, LongUnaryOperator valueFunc) {
this.sizeFunc = sizeFunc;
this.valueFunc = valueFunc::applyAsLong;
}
public Set(LongToIntFunction sizeFunc, LongToIntFunction valueFunc) {
this.sizeFunc = sizeFunc;
this.valueFunc = valueFunc::applyAsInt;
}
public Set(LongFunction<Object> sizeFunc, LongFunction<Object> valueFunc) {
this.sizeFunc = checkSizeFunc(sizeFunc);
this.valueFunc = valueFunc;
}
public Set(LongFunction<Object> sizeFunc, LongUnaryOperator valueFunc) {
this.sizeFunc = checkSizeFunc(sizeFunc);
this.valueFunc = valueFunc::applyAsLong;
}
public Set(LongFunction<Object> sizeFunc, LongToIntFunction valueFunc) {
this.sizeFunc = checkSizeFunc(sizeFunc);
this.valueFunc = valueFunc::applyAsInt;
}
public Set(LongUnaryOperator sizeFunc, LongFunction<Object> valueFunc) {
this.sizeFunc = l -> (int) sizeFunc.applyAsLong(l);
this.valueFunc = valueFunc;
}
public Set(LongUnaryOperator sizeFunc, LongUnaryOperator valueFunc) {
this.sizeFunc = l -> (int) sizeFunc.applyAsLong(l);
this.valueFunc = valueFunc::applyAsLong;
}
public Set(LongUnaryOperator sizeFunc, LongToIntFunction valueFunc) {
this.sizeFunc = l -> (int) sizeFunc.applyAsLong(l);
this.valueFunc = valueFunc::applyAsInt;
}
private static LongToIntFunction checkSizeFunc(LongFunction<?> sizeFunc) {
Object sizeType = sizeFunc.apply(0);
if (int.class.isAssignableFrom(sizeType.getClass())) {
return value -> ((LongFunction<Integer>)sizeFunc).apply(value);
} else if (long.class.isAssignableFrom(sizeType.getClass())) {
return value -> ((LongFunction<Long>)sizeFunc).apply(value).intValue();
} else {
throw new BasicError("The size function produces " + sizeType.getClass().getCanonicalName() + ", which can't be used as an integer");
}
}
@Override
public java.util.Set<Object> apply(long value) {
int size = sizeFunc.applyAsInt(value);
java.util.Set<Object> set = new HashSet<>(size);
for (int i = 0; i < size; i++) {
set.add(valueFunc.apply(value+i));
set.add(valueFunc.apply(value + i));
}
return set;
}

50
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/StringSet.java
View File

@@ -1,5 +1,6 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_collection;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
@@ -8,6 +9,7 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.HashSet;
import java.util.function.LongFunction;
import java.util.function.LongToIntFunction;
import java.util.function.LongUnaryOperator;
/**
* Create a {@code Set<String>} from a long
@@ -29,11 +31,55 @@ public class StringSet implements LongFunction<java.util.Set<String>> {
private final LongFunction<Object> valueFunc;
@Example({"StringSet(HashRange(3,7),Add(15L))", "create a set between 3 and 7 elements of String representations of Long values"})
public StringSet(LongToIntFunction sizeFunc,
LongFunction<Object> valueFunc) {
public StringSet(LongToIntFunction sizeFunc, LongFunction<Object> valueFunc) {
this.sizeFunc = sizeFunc;
this.valueFunc = valueFunc;
}
public StringSet(LongToIntFunction sizeFunc, LongUnaryOperator valueFunc) {
this.sizeFunc = sizeFunc;
this.valueFunc = valueFunc::applyAsLong;
}
public StringSet(LongToIntFunction sizeFunc, LongToIntFunction valueFunc) {
this.sizeFunc = sizeFunc;
this.valueFunc = valueFunc::applyAsInt;
}
public StringSet(LongFunction<?> sizeFunc, LongFunction<Object> valueFunc) {
this.sizeFunc = checkSizeFunc(sizeFunc);
this.valueFunc = valueFunc;
}
public StringSet(LongFunction<?> sizeFunc, LongUnaryOperator valueFunc) {
this.sizeFunc = checkSizeFunc(sizeFunc);
this.valueFunc = valueFunc::applyAsLong;
}
public StringSet(LongFunction<?> sizeFunc, LongToIntFunction valueFunc) {
this.sizeFunc = checkSizeFunc(sizeFunc);
this.valueFunc = valueFunc::applyAsInt;
}
public StringSet(LongUnaryOperator sizeFunc, LongFunction<Object> valueFunc) {
this.sizeFunc = l -> (int) sizeFunc.applyAsLong(l);
this.valueFunc = valueFunc;
}
public StringSet(LongUnaryOperator sizeFunc, LongUnaryOperator valueFunc) {
this.sizeFunc = l -> (int) sizeFunc.applyAsLong(l);
this.valueFunc = valueFunc::applyAsLong;
}
public StringSet(LongUnaryOperator sizeFunc, LongToIntFunction valueFunc) {
this.sizeFunc = l -> (int) sizeFunc.applyAsLong(l);
this.valueFunc = valueFunc::applyAsInt;
}
private static LongToIntFunction checkSizeFunc(LongFunction<?> sizeFunc) {
Object sizeType = sizeFunc.apply(0);
if (int.class.isAssignableFrom(sizeType.getClass())) {
return value -> ((LongFunction<Integer>)sizeFunc).apply(value);
} else if (long.class.isAssignableFrom(sizeType.getClass())) {
return value -> ((LongFunction<Long>)sizeFunc).apply(value).intValue();
} else {
throw new BasicError("The size function produces " + sizeType.getClass().getCanonicalName() + ", which can't be used as an integer");
}
}
@Override
public java.util.Set<String> apply(long value) {

45
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_int/HashInterval.java Normal file
View File

@@ -0,0 +1,45 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_int;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import java.util.function.LongToIntFunction;
@ThreadSafeMapper
public class HashInterval implements LongToIntFunction {
private final long minValue;
private final long width;
private final Hash hash = new Hash();
/**
* Create a hash interval based on a minimum value of 0 and a specified width.
* @param width The maximum value, which is excluded.
*/
@Example({"HashInterval(4)","return values which could include 0, 1, 2, 3, but not 4"})
public HashInterval(int width) {
this.width=width;
this.minValue=0L;
}
/**
* Create a hash interval
* @param minIncl The minimum value, which is included
* @param maxExcl The maximum value, which is excluded
*/
@Example({"HashInterval(2,5)","return values which could include 2, 3, 4, but not 5"})
public HashInterval(int minIncl, int maxExcl) {
if (maxExcl<=minIncl) {
throw new BasicError("HashInterval must have min and max value in that order, where the min is less than the max.");
}
this.minValue = minIncl;
this.width = (maxExcl - minIncl);
}
@Override
public int applyAsInt(long operand) {
return (int) ((minValue + (hash.applyAsLong(operand) % width)) & Integer.MAX_VALUE);
}
}

10
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_int/HashRange.java
View File

@@ -1,5 +1,6 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_int;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
@@ -21,12 +22,11 @@ public class HashRange implements LongToIntFunction {
@Example({"HashRange(35L,39L)","map the input to a number in the range 35-38, inclusive, of type int"})
public HashRange(int minValue, int maxValue) {
this.minValue = minValue;
if (maxValue<=minValue) {
throw new RuntimeException("HashRange must have min and max value in that order.");
if (maxValue<minValue) {
throw new BasicError("HashRange must have min and max value in that order.");
}
this.width = maxValue - minValue;
this.minValue = minValue;
this.width = (maxValue - minValue) +1;
}
@Override

54
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/HashInterval.java Normal file
View File

@@ -0,0 +1,54 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_long;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.function.LongUnaryOperator;
/**
* Return a value within a range, pseudo-randomly, using interval semantics,
* where the range of values return does not include the last value.
* This function behaves exactly like HashRange except for the exclusion
* of the last value. This allows you to stack intervals using known
* reference points without duplicating or skipping any given value.
*
* You can specify hash intervals as small as a single-element range, like
* (5,6), or as wide as the relevant data type allows.
*/
@ThreadSafeMapper
public class HashInterval implements LongUnaryOperator {
private final long minValue;
private final long width;
private final Hash hash = new Hash();
/**
* Create a hash interval based on a minimum value of 0 and a specified width.
* @param width The maximum value, which is excluded.
*/
@Example({"HashInterval(4L)","return values which could include 0L, 1L, 2L, 3L, but not 4L"})
public HashInterval(long width) {
this.minValue=0L;
this.width=width;
}
/**
* Create a hash interval
* @param minIncl The minimum value, which is included
* @param maxExcl The maximum value, which is excluded
*/
@Example({"HashInterval(2L,5L)","return values which could include 2L, 3L, 4L, but not 5L"})
public HashInterval(long minIncl, long maxExcl) {
if (maxExcl<=minIncl) {
throw new BasicError("HashInterval must have min and max value in that order, where the min is less than the max.");
}
this.minValue = minIncl;
this.width = (maxExcl - minIncl);
}
@Override
public long applyAsLong(long operand) {
return minValue + (hash.applyAsLong(operand) % width);
}
}

13
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/HashRange.java
View File

@@ -1,5 +1,6 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_long;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.function.LongUnaryOperator;
@@ -8,6 +9,9 @@ import java.util.function.LongUnaryOperator;
* Return a value within a range, pseudo-randomly. This is equivalent to
* returning a value with in range between 0 and some maximum value, but
* with a minimum value added.
*
* You can specify hash ranges as small as a single-element range, like
* (5,5), or as wide as the relevant data type allows.
*/
@ThreadSafeMapper
public class HashRange implements LongUnaryOperator {
@@ -22,12 +26,11 @@ public class HashRange implements LongUnaryOperator {
}
public HashRange(long minValue, long maxValue) {
this.minValue = minValue;
if (maxValue<=minValue) {
throw new RuntimeException("HashRange must have min and max value in that order.");
if (maxValue<minValue) {
throw new BasicError("HashRange must have min and max value in that order.");
}
this.width = maxValue - minValue;
this.minValue = minValue;
this.width = (maxValue - minValue)+1;
}
@Override

2
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/LongFlow.java
View File

@@ -16,7 +16,7 @@ public class LongFlow implements LongUnaryOperator {
private final LongUnaryOperator[] ops;
@Example({"StringFlow(Add(3),Mul(6))","Create an integer operator which adds 3 and multiplies the result by 6"})
@Example({"LongFlow(Add(3),Mul(6))","Create an integer operator which adds 3 and multiplies the result by 6"})
public LongFlow(LongUnaryOperator... ops) {
this.ops = ops;
}

20
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/FixedValue.java Normal file
View File

@@ -0,0 +1,20 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.function.LongFunction;
@ThreadSafeMapper
public class FixedValue implements LongFunction<String> {
private final String value;
public FixedValue(String value) {
this.value = value;
}
@Override
public String apply(long ignored) {
return value;
}
}

10
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/HashedLineToString.java
View File

@@ -19,8 +19,9 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.HashRange;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.HashInterval;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -35,7 +36,7 @@ import java.util.function.LongFunction;
@ThreadSafeMapper
public class HashedLineToString implements LongFunction<String> {
private final static Logger logger = LogManager.getLogger(HashedLineToString.class);
private final HashRange indexRange;
private final HashInterval indexRange;
private List<String> lines = new ArrayList<>();
@@ -44,7 +45,10 @@ public class HashedLineToString implements LongFunction<String> {
public HashedLineToString(String filename) {
this.filename = filename;
this.lines = NBIO.readLines(filename);
this.indexRange = new HashRange(0, lines.size() - 2);
if (lines.size()<1) {
throw new BasicError("Read " + lines.size() + " lines from " + filename + ", empty files are not supported");
}
this.indexRange = new HashInterval(0, lines.size());
}
public String toString() {

3
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/HashedLoremExtractToString.java
View File

@@ -33,11 +33,12 @@ public class HashedLoremExtractToString implements LongFunction<String> {
private final HashedFileExtractToString randomFileExtractMapper;
public HashedLoremExtractToString(int minsize, int maxsize) {
randomFileExtractMapper = new HashedFileExtractToString("lorem-ipsum.txt", minsize, maxsize);
randomFileExtractMapper = new HashedFileExtractToString("data/lorem_ipsum_full.txt", minsize, maxsize);
}
@Override
public String apply(long input) {
return randomFileExtractMapper.apply(input);
}
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/JoinTemplate.java
View File

@@ -23,8 +23,8 @@ public class JoinTemplate extends Template implements LongFunction<String> {
super(templateFor(prefix,delimiter,suffix,funcs), funcs);
}
@Example({"JoinTemplate(Add(3),'[',';',']',NumberNameToString(),NumberNameToString(),NumberNameToString())",
"create values like '[zero;three,six]', '[one;four,seven]', ..."})
@Example({"JoinTemplate(Add(3),'<',';','>',NumberNameToString(),NumberNameToString(),NumberNameToString())",
"create values like '<zero;three,six>', '<one;four,seven>', ..."})
public JoinTemplate(LongUnaryOperator iterop, String prefix, String delimiter, String suffix, LongFunction<?>... funcs) {
super(iterop, templateFor(prefix,delimiter,suffix,funcs), funcs);

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ModuloCSVLineToString.java
View File

@@ -60,7 +60,9 @@ public class ModuloCSVLineToString implements LongFunction<String> {
}
for (CSVRecord strings : csvp) {
lines.add(strings.get(column));
if (strings.get(column) != null) {
lines.add(strings.get(column));
}
}
}

48
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/NumberNameToString.java
View File

@@ -32,15 +32,57 @@ import java.util.function.LongFunction;
@ThreadSafeMapper
public class NumberNameToString implements LongFunction<String> {
private final static ThreadLocal<StringBuilder> tlsb = ThreadLocal.withInitial(StringBuilder::new);
private final static String THOUSAND = "thousand";
private final static String MILLION = "million";
private final static String BILLION = "billion";
private final static String TRILLION = "trillion";
private final static String QUADRILLION = "quadrillion";
private final static String QUINTILLION = "quintillion";
private final NumberInWordsFormatter formatter = NumberInWordsFormatter.getInstance();
@Override
public String apply(long input) {
if (input==0L) {
if (input == 0L) {
return "zero";
} else if (input > 0 && input <= 999999999L) {
return formatter.format((int) input);
} else if (input < 0 && input >= -999999999L) {
return "negative " + formatter.format((int) -input);
} else {
StringBuilder sb = tlsb.get();
sb.setLength(0);
long value = input;
if (value < 0) {
value = -value;
sb.append("negative");
}
long higher = (value / 1000000000L);
if (higher > 1000000000L) {
long evenhigher = higher / 1000000000L;
sb.append(formatter.format((int) evenhigher)).append(" quintillion");
higher = higher % 1000000000L;
}
String val = formatter.format((int) higher)
.replaceAll(THOUSAND, TRILLION)
.replaceAll(MILLION, QUADRILLION)
.replaceAll(BILLION, QUINTILLION);
if (sb.length() > 0) {
sb.append(" ");
}
sb.append(val).append(" billion");
String rstr = formatter.format((int) (input % 1000000000L));
if (!rstr.isEmpty()) {
sb.append(" ").append(rstr);
}
return sb.toString();
}
String result = formatter.format((int) input % Integer.MAX_VALUE);
return result;
}
}

43
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_string/to_bigdecimal/ToBigDecimal.java Normal file
View File

@@ -0,0 +1,43 @@
package io.nosqlbench.virtdata.library.basics.shared.from_string.to_bigdecimal;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.util.MathContextReader;
import java.math.BigDecimal;
import java.math.MathContext;
import java.math.RoundingMode;
import java.util.function.Function;
@ThreadSafeMapper
@Categories(Category.conversion)
public class ToBigDecimal implements Function<String, BigDecimal> {
private final MathContext context;
@Example({"Convert strings to BigDecimal according to default precision (unlimited) and rounding (HALF_UP)"})
public ToBigDecimal() {
this.context = MathContext.UNLIMITED;
}
/**
* Convert all input values to BigDecimal values with a specific MathContext. This form is only
* supported for scale=0, meaning whole numbers. The value for context can be one of UNLIMITED,
* DECIMAL32, DECIMAL64, DECIMAL128, or any valid configuration supported by
* {@link MathContext#MathContext(String)}, such as {@code "precision=32 roundingMode=CEILING"}.
* In the latter form, roundingMode can be any valid value for {@link RoundingMode}, like
* UP, DOWN, CEILING, FLOOR, HALF_UP, HALF_DOWN, HALF_EVEN, or UNNECESSARY.
*/
public ToBigDecimal(String context) {
this.context =
MathContextReader.getMathContext(context);
}
@Override
public BigDecimal apply(String s) {
return new BigDecimal(s, context);
}
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/stateful/LoadElement.java
View File

@@ -42,8 +42,8 @@ public class LoadElement implements Function<Object,Object>, ConfigAware {
}
@Override
public void applyConfig(Map<String, ?> element) {
Map<String,?> vars = (Map<String, ?>) element.get(mapname);
public void applyConfig(Map<String, ?> elements) {
Map<String,?> vars = (Map<String, ?>) elements.get(mapname);
if (vars!=null) {
this.vars = vars;
}

45
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/unary_int/HashInterval.java Normal file
View File

@@ -0,0 +1,45 @@
package io.nosqlbench.virtdata.library.basics.shared.unary_int;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.function.IntUnaryOperator;
@ThreadSafeMapper
public class HashInterval implements IntUnaryOperator {
private final int minValue;
private final int width;
private final Hash hash = new Hash();
/**
* Create a hash interval based on a minimum value of 0 and a specified width.
* @param width The maximum value, which is excluded.
*/
@Example({"HashInterval(4)","return values which could include 0, 1, 2, 3, but not 4"})
public HashInterval(int width) {
this.minValue=0;
this.width=width;
}
/**
* Create a hash interval
* @param minIncl The minimum value, which is included
* @param maxExcl The maximum value, which is excluded
*/
@Example({"HashInterval(2,5)","return values which could include 2, 3, 4, but not 5"})
public HashInterval(int minIncl, int maxExcl) {
if (maxExcl<=minIncl) {
throw new BasicError("HashInterval must have min and max value in that order, where the min is less than the max.");
}
this.minValue = minIncl;
this.width = (maxExcl - minIncl);
}
@Override
public int applyAsInt(int operand) {
return minValue + (hash.applyAsInt(operand) & width);
}
}

10
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/unary_int/HashRange.java
View File

@@ -1,5 +1,6 @@
package io.nosqlbench.virtdata.library.basics.shared.unary_int;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.function.IntUnaryOperator;
@@ -17,12 +18,11 @@ public class HashRange implements IntUnaryOperator {
}
public HashRange(int minValue, int maxValue) {
this.minValue = minValue;
if (maxValue<=minValue) {
throw new RuntimeException("HashRange must have min and max value in that order.");
if (maxValue<minValue) {
throw new BasicError("HashRange must have min and max value in that order.");
}
this.width = maxValue - minValue;
this.minValue = minValue;
this.width = (maxValue - minValue) +1;
}
@Override

31
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/util/MathContextReader.java Normal file
View File

@@ -0,0 +1,31 @@
package io.nosqlbench.virtdata.library.basics.shared.util;
import io.nosqlbench.nb.api.errors.BasicError;
import java.math.MathContext;
public class MathContextReader {
public static MathContext getMathContext(String name) {
try {
switch (name.toLowerCase()) {
case "unlimited":
return MathContext.UNLIMITED;
case "decimal32":
return MathContext.DECIMAL32;
case "decimal64":
return MathContext.DECIMAL64;
case "decimal128":
return MathContext.DECIMAL128;
default:
return new MathContext(name);
}
} catch (IllegalArgumentException iae) {
throw new BasicError("'" + name + "' was not a valid format for a new MathContext(String), try something " +
"like 'precision=17 roundingMode=UP");
}
}
}

2
virtdata-lib-basics/src/main/resources/META-INF/services/javax.annotation.processing.Processor
View File

@@ -1,2 +1,2 @@
io.nosqlbench.virtdata.api.processors.FunctionDocInfoProcessor
io.nosqlbench.nb.api.processors.ServiceProcessor
io.nosqlbench.nb.annotations.ServiceProcessor

2
virtdata-lib-basics/src/main/resources/data/adventures.txt
View File

@@ -332,7 +332,7 @@ from his cigarette.
“Precisely. And the man who wrote the note is a German. Do you note the
peculiar construction of the sentence—‘This account of you we have
from all quarters received.’ A Frenchman or Russian could not have
written that. It is the German who is so uncourteous to his verbs. It
written that. It is the German who is so uncourteous to his verb. It
only remains, therefore, to discover what is wanted by this German
who writes upon Bohemian paper, and prefers wearing a mask to showing
his face. And here he comes, if I am not mistaken, to resolve all our

30
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_double/to_bigdecimal/ToBigDecimalTest.java Normal file
View File

@@ -0,0 +1,30 @@
package io.nosqlbench.virtdata.library.basics.shared.from_double.to_bigdecimal;
import org.assertj.core.data.Offset;
import org.junit.Test;
import java.math.BigDecimal;
import java.math.MathContext;
import static org.assertj.core.api.Assertions.assertThat;
public class ToBigDecimalTest {
@Test
public void demonstrateDoubleToBigDecimal() {
double big = 1234567890.098765d;
System.out.println(big);
ToBigDecimal unlimited = new ToBigDecimal();
BigDecimal bignum = unlimited.apply(big);
assertThat(bignum.doubleValue()).isCloseTo(big, Offset.offset(0.000001d));
assertThat(bignum).isEqualTo(new BigDecimal(big, MathContext.UNLIMITED));
ToBigDecimal p5rounded = new ToBigDecimal("precision=5 roundingMode=UP");
BigDecimal rounded = p5rounded.apply(big);
assertThat(rounded.doubleValue()).isCloseTo(1234600000.0D,Offset.offset(0.0000001d));
System.out.println(rounded);
}
}

31
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bigdecimal/ToBigDecimalTest.java Normal file
View File

@@ -0,0 +1,31 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_bigdecimal;
import org.assertj.core.data.Offset;
import org.junit.Test;
import java.math.BigDecimal;
import java.math.MathContext;
import static org.assertj.core.api.Assertions.assertThat;
public class ToBigDecimalTest {
@Test
public void demonstrateLongToBigDecimal() {
ToBigDecimal wholeValues = new ToBigDecimal();
BigDecimal whole12345 = wholeValues.apply(12345L);
assertThat(whole12345).isEqualTo(new BigDecimal(12345L));
ToBigDecimal pennies = new ToBigDecimal(2);
BigDecimal pennies12345 = pennies.apply(12345L);
assertThat(pennies12345).isEqualTo(BigDecimal.valueOf(12345,2));
ToBigDecimal custom = new ToBigDecimal("precision=5 roundingMode=CEILING");
BigDecimal c123456 = custom.apply(123456L);
assertThat(c123456).isEqualTo(new BigDecimal(123456L,new MathContext("precision=5 roundingMode=CEILING")));
double v = c123456.doubleValue();
assertThat(v).isCloseTo(123460d, Offset.offset(0.001d));
}
}

4
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListTest.java
View File

@@ -9,9 +9,9 @@ public class ListTest {
@Test
public void testList() {
io.nosqlbench.virtdata.library.basics.shared.from_long.to_collection.List lf = new io.nosqlbench.virtdata.library.basics.shared.from_long.to_collection.List(new HashRange(1, 3), (l) -> "_" + l);
io.nosqlbench.virtdata.library.basics.shared.from_long.to_collection.List lf = new io.nosqlbench.virtdata.library.basics.shared.from_long.to_collection.List(new HashRange(2, 3), (l) -> "_" + l);
java.util.List<Object> l1 = lf.apply(2L);
assertThat(l1).containsExactly("_2","_3");
assertThat(l1).containsExactly("_2","_3","_4");
}
@Test

7
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/SetTest.java
View File

@@ -2,20 +2,23 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_collection;
import org.junit.Test;
import java.util.function.LongFunction;
import java.util.function.LongToIntFunction;
import static org.assertj.core.api.Assertions.assertThat;
public class SetTest {
@Test
public void testSet() {
Set set = new Set(s -> 3, e -> e);
Set set = new Set((LongToIntFunction) s -> 3, (LongFunction<Object>) e -> e);
java.util.Set<Object> s1 = set.apply(15L);
assertThat(s1).containsOnly(15L,16L,17L);
}
@Test
public void testStringSet() {
StringSet set = new StringSet(s -> 3, e -> e);
StringSet set = new StringSet((LongToIntFunction) s -> 3, (LongToIntFunction) (e -> (int)e));
java.util.Set<String> s1 = set.apply(15L);
assertThat(s1).containsOnly("15","16","17");
}

22
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/HashIntervalTest.java Normal file
View File

@@ -0,0 +1,22 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_long;
import io.nosqlbench.nb.api.errors.BasicError;
import org.junit.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class HashIntervalTest {
@Test
public void testBasicRange() {
HashInterval hi = new HashInterval(3L, 5L);
long r1 = hi.applyAsLong(43L);
assertThat(r1).isEqualTo(4L);
}
@Test(expected = BasicError.class)
public void testRangeError() {
HashInterval hi = new HashInterval(3L, 3L);
}
}

27
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/NumberNameToStringTest.java Normal file
View File

@@ -0,0 +1,27 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import org.junit.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class NumberNameToStringTest {
@Test
public void testLargeNumbers() {
NumberNameToString fmt = new NumberNameToString();
assertThat(fmt.apply(Integer.MAX_VALUE))
.isEqualTo(
"two billion one hundred and forty seven million four hundred and eighty three thousand six hundred and forty seven"
);
assertThat(fmt.apply(999999999))
.isEqualTo(
"nine hundred and ninety nine million nine hundred and ninety nine thousand nine hundred and ninety nine"
);
assertThat(fmt.apply(1000000000L)).isEqualTo("one billion");
assertThat(fmt.apply(-1000000000L)).isEqualTo("negative one billion");
assertThat(fmt.apply(10000000000L)).isEqualTo("ten billion");
// 9, 223,372,036, 854,775,807
assertThat(fmt.apply(Long.MAX_VALUE)).isEqualTo("nine quintillion two hundred and twenty three quadrillion three hundred and seventy two trillion and thirty six billion eight hundred and fifty four million seven hundred and seventy five thousand eight hundred and seven");
}
}

1
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/tests/long_collections/HashedRangeToLongListTest.java
View File

@@ -13,6 +13,7 @@ public class HashedRangeToLongListTest {
public void longListRangeTest() {
HashedRangeToLongList gener = new HashedRangeToLongList(3, 6, 9, 12);
for (int i = 0; i < 100; i++) {
System.out.println("long list range test size: " + i);
List<Long> list= gener.apply(i);
assertThat(list.size()).isBetween(9,12);
for (Long longVal : list) {

7
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/tests/long_long/HashRangeTest.java
View File

@@ -13,4 +13,11 @@ public class HashRangeTest {
assertThat(hashRange.applyAsLong(32L)).isEqualTo(11L);
}
@Test
public void testSingleElementRange() {
HashRange hashRange = new HashRange(33L,33L);
long l = hashRange.applyAsLong(93874L);
assertThat(l).isEqualTo(33L);
}
}

18
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/tests/long_long/RangeTests.java
View File

@@ -13,11 +13,11 @@ public class RangeTests {
@Test
public void testHashRange() {
HashRange hr = new HashRange(10,13);
assertThat(hr.applyAsLong(0)).isEqualTo(11);
assertThat(hr.applyAsLong(10)).isEqualTo(12);
assertThat(hr.applyAsLong(100)).isEqualTo(12);
assertThat(hr.applyAsLong(1000)).isEqualTo(11);
assertThat(hr.applyAsLong(10000)).isEqualTo(10);
assertThat(hr.applyAsLong(0)).isEqualTo(13);
assertThat(hr.applyAsLong(10)).isEqualTo(11);
assertThat(hr.applyAsLong(100)).isEqualTo(13);
assertThat(hr.applyAsLong(1000)).isEqualTo(13);
assertThat(hr.applyAsLong(10000)).isEqualTo(11);
}
@Test
@@ -32,10 +32,10 @@ public class RangeTests {
@Test
public void testAddHashRange() {
AddHashRange ahr = new AddHashRange(14,17);
assertThat(ahr.applyAsLong(1)).isEqualTo(16);
assertThat(ahr.applyAsLong(2)).isEqualTo(16);
assertThat(ahr.applyAsLong(3)).isEqualTo(18);
assertThat(ahr.applyAsLong(4)).isEqualTo(20);
assertThat(ahr.applyAsLong(1)).isEqualTo(17);
assertThat(ahr.applyAsLong(2)).isEqualTo(17);
assertThat(ahr.applyAsLong(3)).isEqualTo(20);
assertThat(ahr.applyAsLong(4)).isEqualTo(19);
assertThat(ahr.applyAsLong(5)).isEqualTo(19);
}