IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-12-23 15:40:44 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into mwolters/reset_cmd

Browse Source
This commit is contained in:
Mark Wolters 2024-02-05 10:44:25 -04:00
commit 35c8538206
11 changed files with 442 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/simrate/RateLimiters.java
View File

@ -25,13 +25,9 @@ public enum RateLimiters {
private static final Logger logger = LogManager.getLogger(RateLimiters.class);
public static synchronized RateLimiter createOrUpdate(final NBComponent parent, final RateLimiter extant, final SimRateSpec spec) {
return createOrUpdate(parent, extant, spec, "cycle");
}
public static synchronized RateLimiter createOrUpdate(final NBComponent parent, final RateLimiter extant, final SimRateSpec spec,final String type) {
if (null == extant) {
final RateLimiter rateLimiter= new SimRate(parent, spec, type);
final RateLimiter rateLimiter= new SimRate(parent, spec);
RateLimiters.logger.info(() -> "Using rate limiter: " + rateLimiter);
return rateLimiter;

72
engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/simrate/SimRate.java
View File

@ -79,58 +79,34 @@ public class SimRate extends NBBaseComponent implements RateLimiter, Thread.Unca
private long startTime;
public SimRate(NBComponent parent, SimRateSpec spec) {
this(parent, spec, "cycle");
}
public SimRate(NBComponent parent, SimRateSpec spec, String type) {
super(parent, NBLabels.forKV());
super(parent, NBLabels.forKV().and("rateType",
(spec instanceof CycleRateSpec? "cycle" : "stride")));
this.spec = spec;
initMetrics(type);
initMetrics();
startFiller();
}
private void initMetrics(String type) {
if (type.equalsIgnoreCase("cycle")) {
create().gauge(
"cycles_waittime",
() -> (double) getWaitTimeDuration().get(ChronoUnit.NANOS),
MetricCategory.Core,
"The cumulative scheduling delay which accrues when" +
" an activity is not able to execute operations as fast as requested."
);
create().gauge(
"config_cyclerate",
() -> spec.opsPerSec,
MetricCategory.Config,
"The configured cycle rate in ops/s"
);
create().gauge(
"config_burstrate",
() -> spec.burstRatio,
MetricCategory.Config,
"the configured burst rate as a multiplier to the configured cycle rate. ex: 1.05 means 5% faster is allowed."
);
} else {
create().gauge(
"stride_waittime",
() -> (double) getWaitTimeDuration().get(ChronoUnit.NANOS),
MetricCategory.Core,
"The cumulative scheduling delay which accrues when" +
" an activity is not able to execute operations as fast as requested."
);
create().gauge(
"config_striderate",
() -> spec.opsPerSec,
MetricCategory.Config,
"The configured stride rate in ops/s"
);
create().gauge(
"config_burstrate",
() -> spec.burstRatio,
MetricCategory.Config,
"the configured burst rate as a multiplier to the configured cycle rate. ex: 1.05 means 5% faster is allowed."
);
}
private void initMetrics() {
String rateType = getLabels().valueOf("rateType");
create().gauge(
rateType + "s_waittime",
() -> (double) getWaitTimeDuration().get(ChronoUnit.NANOS),
MetricCategory.Core,
"The cumulative scheduling delay which accrues when" +
" an activity is not able to execute operations as fast as requested."
);
create().gauge(
"config_" + rateType + "rate",
() -> spec.opsPerSec,
MetricCategory.Config,
"The configured cycle rate in ops/s"
);
create().gauge(
rateType + "_config_burstrate",
() -> spec.burstRatio,
MetricCategory.Config,
"the configured burst rate as a multiplier to the configured cycle rate. ex: 1.05 means 5% faster is allowed."
);
}
public long refill() {

6
engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -89,8 +89,6 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok
private ActivityMetricProgressMeter progressMeter;
private String workloadSource = "unspecified";
private final RunStateTally tally = new RunStateTally();
public static final String STRIDE = "stride";
public static final String CYCLE = "cycle";
public SimpleActivity(NBComponent parent, ActivityDef activityDef) {
super(parent, NBLabels.forKV("activity", activityDef.getAlias()).and(activityDef.auxLabels()));
@ -317,11 +315,11 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok
}
public void createOrUpdateStrideLimiter(SimRateSpec spec) {
strideLimiter = RateLimiters.createOrUpdate(this, strideLimiter, spec, STRIDE);
strideLimiter = RateLimiters.createOrUpdate(this, strideLimiter, spec);
}
public void createOrUpdateCycleLimiter(SimRateSpec spec) {
cycleLimiter = RateLimiters.createOrUpdate(this, cycleLimiter, spec, CYCLE);
cycleLimiter = RateLimiters.createOrUpdate(this, cycleLimiter, spec);
}
/**

17
virtdata-api/src/main/java/io/nosqlbench/virtdata/api/bindings/VirtDataConversions.java
View File

@ -20,10 +20,7 @@ import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.lang.reflect.TypeVariable;
import java.lang.reflect.*;
import java.security.InvalidParameterException;
import java.util.*;
import java.util.function.LongFunction;
@ -70,6 +67,18 @@ public class VirtDataConversions {
private static final Logger logger = LogManager.getLogger(VirtDataConversions.class);
public static <F,T> T[] adaptFunctionArray(F[] functionObjects, Class<T> functionType, Class<Object>... resultSignature) {
T[] functions = (T[]) Array.newInstance(functionType, functionObjects.length);
for (int i = 0; i < functionObjects.length; i++) {
F func = functionObjects[i];
T adapted = adaptFunction(func, functionType, resultSignature);
functions[i]=adapted;
}
return functions;
}
public static <F, T> List<T> adaptFunctionList(F[] funcs, Class<T> functionType, Class<Object>... resultSignature) {
List<T> functions = new ArrayList<>();
for (Object func : funcs) {

126
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/Concat.java Normal file
View File

@ -0,0 +1,126 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.api.bindings.VirtDataConversions;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
import java.util.function.LongBinaryOperator;
import java.util.function.LongFunction;
import java.util.regex.Matcher;
/**
* <P>This is the core implementation of the Concat style of String
* binding. It is the newer and recommended version of {@link Template}.</P>
*
* <P>Users should use one of these wrappers:
* <UL>
* <LI>{@link ConcatCycle} - all inputs are the cycle value</LI>
* <LI>{@link ConcatHashed} - all inputs are a hash of (cycle+step)</LI>
* <LI>{@link ConcatStepped} - all inputs are (cycle+step)</LI>
* <LI>{@link ConcatFixed} - all inputs are always the same value</LI>
* <LI>{@link ConcatChained} - all inputs are chained hashes of the previous one</LI>
* </UL></P>
* <p>
* <HR/>
*
* <P>This implementation is available for specialized use when needed, but the
* above versions are much more self-explanatory and easy to use.</P>
*
* <P>As with previous implementations, the basic input which is fed to the functions
* is the sum of the input cycle and the step, where the step is simply the index of
* the insertion point within the template string. These start at 0, so a template string
* which contains "{}-{}" will have two steps, 0, and 1. For cycle 35, the first
* function will take input 35, and the second 36. This can create some local neighborhood
* similarity in test data, so other forms are provided which can hash the values for
* an added degree of (effective) randomness and one that chains these so that each
* set of values from a Concat binding are quite distinct from each other.</P>
* <p>
* <HR/>
* <P>Binding functions used to populate each step of the template may have their own bounds
* of output values like {@link Combinations}. These are easy to use internally since they
* work well with the hashing. However, some other functions may operate over the whole space
* of long values, and come with no built-in cardinality constraints. It is recommended to
* use those with built-in constraints when you want to render a discrete population of values.</P>
*/
@ThreadSafeMapper
@Categories(Category.general)
public class Concat implements LongFunction<String> {
private final static Logger logger = LogManager.getLogger(Concat.class);
protected final LongBinaryOperator cycleStepMapper;
protected final String[] literals;
protected final LongFunction<String>[] functions;
public Concat(String template, Object... functions) {
this(Long::sum, template, functions);
}
public Concat(LongBinaryOperator cycleStepMapper, String template, Object... functions) {
this.cycleStepMapper = cycleStepMapper;
this.literals = parseTemplate(template);
if (literals.length > 1 && functions.length < 1) {
logger.warn("You provided zero functions to go with concat template " + template + ", inserting diagnostic function for \"v:\"(cycle+step)");
this.functions = new LongFunction[]{l -> "v:" + l};
} else {
this.functions = VirtDataConversions.adaptFunctionArray(functions, LongFunction.class, Object.class);
}
}
@Override
public String apply(long cycle) {
StringBuilder buffer = new StringBuilder();
buffer.setLength(0);
for (int i = 0; i < literals.length - 1; i++) {
buffer.append(literals[i]);
long value = cycleStepMapper.applyAsLong(cycle, i);
int funcIdx = Math.min(functions.length - 1, i);
LongFunction<String> selectedFunction = functions[funcIdx];
String string = selectedFunction.apply(value);
buffer.append(string);
}
buffer.append(literals[literals.length - 1]);
return buffer.toString();
}
private String[] parseTemplate(String template) {
try {
List<String> literals = new ArrayList<>();
java.util.regex.Pattern p = java.util.regex.Pattern.compile("\\{}");
Matcher m = p.matcher(template);
int pos = 0;
while (m.find()) {
literals.add(template.substring(pos, m.start()));
pos = m.end();
}
String partial = template.substring(pos);
literals.add(partial);
return literals.toArray(new String[0]);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

60
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ConcatChained.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import java.util.function.LongFunction;
/**
* <P>This is a variant of Concat which chains the hash values
* from step to step so that each of the provided functions will
* yield unrelated values. The first input value to a function
* is a hash of the cycle input value, the next is a hash of the
* first input value, and so on.</P>
*/
@ThreadSafeMapper
@Categories(Category.general)
public class ConcatChained extends Concat {
private final static Hash hash = new Hash();
public ConcatChained(String template, Object... functions) {
super((c,s) -> hash.applyAsLong(c+s), template, functions);
}
@Override
public String apply(long cycle) {
StringBuilder buffer = new StringBuilder();
buffer.setLength(0);
buffer.append(literals[0]);
long value = cycle;
for (int i = 0; i < literals.length-1; i++) {
value = hash.applyAsLong(value);
buffer.append(literals[i]);
int funcIdx = Math.min(functions.length - 1, i);
LongFunction<String> selectedFunction = functions[funcIdx];
String string = selectedFunction.apply(value);
buffer.append(string);
}
buffer.append(literals[literals.length-1]);
return buffer.toString();
}
}

34
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ConcatCycle.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
/**
* <P>This is a variant of Concat which always uses the input cycle value
* as the input for all the functions provided.</P>
*/
@ThreadSafeMapper
@Categories(Category.general)
public class ConcatCycle extends Concat {
public ConcatCycle(String template, Object... functions) {
super((c,s) -> c, template, functions);
}
}

37
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ConcatFixed.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.api.bindings.VirtDataConversions;
import java.util.function.LongUnaryOperator;
/**
* <P>This is a variant of Concat which always uses the same value
* as input for the functions provided.</P>
*/
@ThreadSafeMapper
@Categories(Category.general)
public class ConcatFixed extends Concat {
public ConcatFixed(long value, String template, Object... functions) {
super((c,s) -> value, template, functions);
}
}

35
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ConcatHashed.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
/**
* <P>This is a variant of Concat which always hashes the cycle+step value
* for each function provided.</P>
*/
@ThreadSafeMapper
@Categories(Category.general)
public class ConcatHashed extends Concat {
private final static Hash hash = new Hash();
public ConcatHashed(String template, Object... functions) {
super((c,s) -> hash.applyAsLong(c+s), template, functions);
}
}

41
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ConcatStepped.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.api.bindings.VirtDataConversions;
import java.util.ArrayList;
import java.util.List;
import java.util.function.LongBinaryOperator;
import java.util.function.LongFunction;
import java.util.regex.Matcher;
/**
* <P>This is a variant of Concat which uses the cycle+step sum for each
* of the functions provided.</P>
*/
@ThreadSafeMapper
@Categories(Category.general)
public class ConcatStepped extends Concat {
public ConcatStepped(String template, Object... functions) {
super(Long::sum, template, functions);
}
}

69
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ConcatTest.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.library.basics.shared.conversions.from_long.ToHexString;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.*;
class ConcatTest {
@Test
public void testEmptyString() {
Concat p = new Concat("");
assertThat(p.apply(3L)).isEqualTo("");
}
@Test
public void testMismatchedInserts() {
Concat c = new Concat("{}");
assertThat(c.apply(3L)).isEqualTo("v:3");
}
@Test
public void testOnlyLiteral() {
Concat l = new Concat(Long::sum, "literal");
assertThat(l.apply(3L)).isEqualTo("literal");
}
@Test
public void testSimpleValue() {
Concat p = new Concat(Long::sum, "{}", new NumberNameToString());
assertThat(p.apply(1)).isEqualTo("one");
}
@Test
public void testFixedCycle() {
Concat p = new Concat((c, s) -> c, "{}-{}", new NumberNameToString());
assertThat(p.apply(3L)).isEqualTo("three-three");
}
@Test
public void testSteppedCycle() {
Concat p = new Concat(Long::sum, "{}-{}", new NumberNameToString());
assertThat(p.apply(3L)).isEqualTo("three-four");
}
@Test
public void testConcatChained() {
ConcatChained cc = new ConcatChained("{}-{}", new ToHexString());
assertThat(cc.apply(1)).isEqualTo("5752fae69d1653da-7dfbf78ca62528b5");
}
}