IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

replace ActivityDef with NBConfigurable

Browse Source
This commit is contained in:
Jonathan Shook 2025-01-02 12:15:11 -06:00
parent 3ab18635bb
commit 97e7b16d77
61 changed files with 1316 additions and 1373 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
nb-adapters/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java
View File

@ -70,14 +70,14 @@ public class DiagDriverAdapter extends BaseDriverAdapter<DiagOp, DiagSpace> impl
@Override
public NBConfigModel getConfigModel() {
NBConfigModel model = super.getConfigModel();
model.add(DiagSpace.getConfigModel());
NBConfigModel model = getConfigModel();
model.add(DiagSpace.getStaticConfigModel());
return model;
}
@Override
public NBConfigModel getReconfigModel() {
NBConfigModel model = super.getReconfigModel();
NBConfigModel model = getReconfigModel();
NBConfigModel mapperModel = NBReconfigurable.collectModels(DiagDriverAdapter.class, List.of(mapper));
return model.add(mapperModel);
}

22
nb-adapters/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagOpDispenser.java
View File

@ -20,6 +20,7 @@ import io.nosqlbench.adapter.diag.optasks.DiagTask;
import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigurable;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.config.standard.NBReconfigurable;
import io.nosqlbench.nb.api.components.core.NBParentComponentInjection;
@ -102,6 +103,16 @@ public class DiagOpDispenser extends BaseOpDispenser<DiagOp,DiagSpace> implement
return opFunc.getReconfigModel();
}
@Override
public void applyConfig(NBConfiguration cfg) {
}
@Override
public NBConfigModel getConfigModel() {
return null;
}
private final static class OpFunc implements LongFunction<DiagOp>, NBReconfigurable {
private final List<DiagTask> tasks;
private final LongFunction<DiagSpace> spaceF;
@ -126,6 +137,17 @@ public class DiagOpDispenser extends BaseOpDispenser<DiagOp,DiagSpace> implement
public NBConfigModel getReconfigModel() {
return NBReconfigurable.collectModels(DiagTask.class, tasks);
}
@Override
public void applyConfig(NBConfiguration cfg) {
NBConfigurable.applyMatching(cfg, tasks);
}
@Override
public NBConfigModel getConfigModel() {
return NBConfigurable.collectModels(DiagTask.class, tasks);
}
}
@Override

15
nb-adapters/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagOpMapper.java
View File

@ -20,9 +20,7 @@ import io.nosqlbench.adapters.api.activityimpl.OpDispenser;
import io.nosqlbench.adapters.api.activityimpl.OpMapper;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.config.standard.NBReconfigurable;
import io.nosqlbench.nb.api.config.standard.*;
import java.util.ArrayList;
import java.util.LinkedHashMap;
@ -32,6 +30,7 @@ import java.util.function.LongFunction;
public class DiagOpMapper implements OpMapper<DiagOp,DiagSpace>, NBReconfigurable {
private final Map<String,DiagOpDispenser> dispensers = new LinkedHashMap<>();
private final DiagDriverAdapter adapter;
private NBConfiguration config;
public DiagOpMapper(DiagDriverAdapter adapter) {
this.adapter = adapter;
@ -45,6 +44,16 @@ public class DiagOpMapper implements OpMapper<DiagOp,DiagSpace>, NBReconfigurabl
return dispenser;
}
@Override
public NBConfigModel getConfigModel() {
return ConfigModel.of(DiagOpMapper.class).asReadOnly();
}
@Override
public void applyConfig(NBConfiguration cfg) {
this.config = cfg;
NBConfigurable.applyMatching(cfg, dispensers.values());
}
@Override
public void applyReconfig(NBConfiguration recfg) {

20
nb-adapters/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagSpace.java
View File

@ -17,17 +17,12 @@
package io.nosqlbench.adapter.diag;
import io.nosqlbench.adapters.api.activityimpl.uniform.BaseSpace;
import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.simrate.RateLimiter;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.config.standard.Param;
import io.nosqlbench.nb.api.config.standard.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class DiagSpace extends BaseSpace<DiagSpace> implements ActivityDefObserver {
public class DiagSpace extends BaseSpace<DiagSpace> implements NBConfigurable {
private final Logger logger = LogManager.getLogger(DiagSpace.class);
private final NBConfiguration cfg;
@ -47,7 +42,10 @@ public class DiagSpace extends BaseSpace<DiagSpace> implements ActivityDefObserv
this.errorOnClose = cfg.get("erroronclose",boolean.class);
}
public static NBConfigModel getConfigModel() {
public NBConfigModel getConfigModel() {
return getStaticConfigModel();
}
public static NBConfigModel getStaticConfigModel() {
return ConfigModel.of(DiagSpace.class)
.add(Param.defaultTo("interval",1000))
.add(Param.defaultTo("erroronclose", false))
@ -60,11 +58,6 @@ public class DiagSpace extends BaseSpace<DiagSpace> implements ActivityDefObserv
}
}
@Override
public void onActivityDefUpdate(ActivityDef activityDef) {
NBConfiguration cfg = getConfigModel().apply(activityDef.getParams().getStringStringMap());
this.applyConfig(cfg);
}
@Override
public void close() throws Exception {
@ -73,4 +66,5 @@ public class DiagSpace extends BaseSpace<DiagSpace> implements ActivityDefObserv
throw new RuntimeException("diag space was configured to throw this error when it was configured.");
}
}
}

75
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java
View File

@ -144,55 +144,6 @@ public abstract class BaseDriverAdapter<RESULT
return cfg;
}
@Override
public void applyConfig(NBConfiguration cfg) {
this.cfg = cfg;
}
@Override
public void applyReconfig(NBConfiguration reconf) {
this.cfg = getReconfigModel().apply(reconf.getMap());
}
/**
* In order to be provided with config information, it is required
* that the driver adapter specify the valid configuration options,
* their types, and so on.
*/
@Override
public NBConfigModel getConfigModel() {
return ConfigModel.of(BaseDriverAdapter.class)
.add(Param.optional("alias"))
.add(Param.optional("labels", String.class, "Labels which will apply to metrics and annotations for this activity only"))
.add(Param.defaultTo("strict", true, "strict op field mode, which requires that provided op fields are recognized and used"))
.add(Param.optional(List.of("op", "stmt", "statement"), String.class, "op template in statement form"))
.add(Param.optional("tags", String.class, "tags to be used to filter operations"))
.add(Param.defaultTo("errors", "stop", "error handler configuration"))
.add(Param.optional("threads").setRegex("\\d+|\\d+x|auto").setDescription("number of concurrent operations, controlled by threadpool"))
.add(Param.optional("stride").setRegex("\\d+"))
.add(Param.optional("striderate", String.class, "rate limit for strides per second"))
.add(Param.optional("cycles").setRegex("\\d+[KMBGTPE]?|\\d+[KMBGTPE]?\\.\\.\\d+[KMBGTPE]?").setDescription("cycle interval to use"))
.add(Param.optional("recycles").setDescription("allow cycles to be re-used this many times"))
.add(Param.optional(List.of("cyclerate", "targetrate", "rate"), String.class, "rate limit for cycles per second"))
.add(Param.optional("seq", String.class, "sequencing algorithm"))
.add(Param.optional("instrument", Boolean.class))
.add(Param.optional(List.of("workload", "yaml"), String.class, "location of workload yaml file"))
.add(Param.optional("driver", String.class))
.add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|emit|none)"))
.add(Param.optional("maxtries", Integer.class))
.asReadOnly();
}
@Override
public NBConfigModel getReconfigModel() {
return ConfigModel.of(BaseDriverAdapter.class)
.add(Param.optional("threads").setRegex("\\d+|\\d+x|auto").setDescription("number of concurrent operations, controlled by threadpool"))
.add(Param.optional("striderate", String.class, "rate limit for strides per second"))
.add(Param.optional(List.of("cyclerate", "targetrate", "rate"), String.class, "rate limit for cycles per second"))
.asReadOnly();
}
@Override
public LongFunction<SPACE> getSpaceFunc(ParsedOp pop) {
@ -233,4 +184,30 @@ public abstract class BaseDriverAdapter<RESULT
}
super.beforeDetach();
}
@Override
public void applyConfig(NBConfiguration cfg) {
this.cfg = cfg;
}
@Override
public void applyReconfig(NBConfiguration reconf) {
this.cfg = getReconfigModel().apply(reconf.getMap());
}
/// These are implemented here as _unit_ values, meaning, you shouldn't be asking
/// "Does this element have a configuration model", but instead you should be asking
/// "What is the (possibly empty?) configuration model of this element?"
@Override
public NBConfigModel getConfigModel() {
return ConfigModel.of(BaseDriverAdapter.class).asReadOnly();
}
/// These are implemented here as _unit_ values, meaning, you shouldn't be asking
/// "Does this element have a reconfiguration model", but instead you should be asking
/// "What is the (possibly empty?) reconfiguration model of this element?"
@Override
public NBConfigModel getReconfigModel() {
return ConfigModel.of(BaseDriverAdapter.class).asReadOnly();
}
}

32
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java
View File

@ -394,28 +394,24 @@ public class ParsedOp extends NBBaseComponent implements LongFunction<Map<String
private final List<CapturePoint> captures = new ArrayList<>();
private final OpTemplate _opTemplate;
private final NBConfiguration activityCfg;
private final Map<String,Object> activityCfg;
private final ParsedTemplateMap tmap;
private final NBLabels labels;
private final List<Function<Map<String, Object>, Map<String, Object>>> preprocessors;
/**
Create a parsed command from an Op template. This version is exactly like
except that it allows
preprocessors. Preprocessors are all applied to the the op template before
it is applied to the parsed command fields, allowing you to combine or destructure
fields from more tha one representation into a single canonical representation
for processing.
@param opTemplate
The OpTemplate as provided by a user via YAML, JSON, or API (data structure)
@param activityCfg
The activity configuration, used to resolve nested config parameters
@param preprocessors
Map->Map transformers.
*/
public ParsedOp(ParsedOp pop, NBConfiguration config) {
this(pop._opTemplate,new LinkedHashMap<>(pop.activityCfg) {{ this.putAll(config.getMap());}},List.of(),pop.parent);
}
/// Create a parsed command from an Op template. Preprocessors are all applied to the the op template before
/// it is applied to the parsed command fields, allowing you to combine or destructure
/// fields from more tha one representation into a single canonical representation
/// for processing.
/// @param opTemplate The OpTemplate as provided by a user via YAML, JSON, or API (data structure)
/// @param activityCfg The activity configuration, used to resolve nested config parameters
/// @param preprocessors Map->Map transformers.
public ParsedOp(
OpTemplate opTemplate, NBConfiguration activityCfg,
OpTemplate opTemplate,
Map<String,Object> activityCfg,
List<Function<Map<String, Object>, Map<String, Object>>> preprocessors,
NBComponent parent
) {
@ -439,7 +435,7 @@ public class ParsedOp extends NBBaseComponent implements LongFunction<Map<String
this.tmap = new ParsedTemplateMap(
getName(), map, opTemplate.getBindings(),
List.of(opTemplate.getParams(), activityCfg.getMap())
List.of(opTemplate.getParams(), activityCfg)
);
NBLabels opLabels = parent.getLabels().and(

6
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java
View File

@ -20,7 +20,7 @@ import com.google.gson.Gson;
import io.nosqlbench.nb.api.advisor.NBAdvisorBuilder;
import io.nosqlbench.nb.api.advisor.NBAdvisorPoint;
import io.nosqlbench.nb.api.advisor.conditions.Conditions;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.errors.OpConfigError;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
@ -53,9 +53,9 @@ public class StrInterpolator implements Function<String, String> {
private final Pattern COMMENT = Pattern.compile("^\\s*#.*");
private final Pattern INSERT = Pattern.compile("^(\\s*)INSERT:\\s+(.+)$");
public StrInterpolator(ActivityDef... activityDefs) {
public StrInterpolator(NBConfiguration... activityDefs) {
Arrays.stream(activityDefs)
.map(ad -> ad.getParams().getStringStringMap())
.map(ad -> ad.getMap())
.forEach(multimap::add);
}

140
nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/ParsedOpTest.java
View File

@ -37,39 +37,27 @@ import static org.assertj.core.api.Assertions.assertThat;
public class ParsedOpTest {
private NBComponent getParent() {
return new TestComponent("opparent","opparent");
return new TestComponent("opparent", "opparent");
}
private ParsedOp getOp() {
ParsedOp pc = new ParsedOp(
new OpData().applyFields(
Map.of(
"op", Map.of(
"stmt", "test",
"dyna1", "{dyna1}",
"dyna2", "{{NumberNameToString()}}",
"identity", "{{Identity()}}"
),
"bindings", Map.of(
"dyna1", "NumberNameToString()"
)
)
),
ConfigModel.of(ParsedOpTest.class)
.add(Param.defaultTo("testcfg", "testval"))
.asReadOnly()
.apply(Map.of()),
List.of(),
getParent()
);
OpData opTemplate = new OpData().applyFields(Map.of(
"op", Map.of(
"stmt", "test", "dyna1", "{dyna1}", "dyna2", "{{NumberNameToString()}}", "identity",
"{{Identity()}}"), "bindings", Map.of("dyna1", "NumberNameToString()")));
NBConfiguration nbcfg = ConfigModel.of(ParsedOpTest.class)
.add(Param.defaultTo("testcfg", "testval")).asReadOnly().apply(Map.of());
ParsedOp pc = new ParsedOp(opTemplate, nbcfg.getMap(), List.of(), getParent());
return pc;
}
@Test
public void testFieldDelegationFromDynamicToStaticToConfig() {
final NBConfiguration cfg = ConfigModel.of(ParsedOpTest.class)
.add(Param.defaultTo("puppy", "dog"))
.add(Param.required("surname", String.class))
.add(Param.defaultTo("puppy", "dog")).add(Param.required("surname", String.class))
.asReadOnly().apply(Map.of("surname", "yes"));
final String opt = """
@ -80,10 +68,11 @@ public class ParsedOpTest {
params:
ps1: "param-one"
""";
final OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null);
assertThat(stmtsDocs.getOps().matching("",true).size()).isEqualTo(1);
final OpTemplate opTemplate = stmtsDocs.getOps().matching("",true).get(0);
final ParsedOp parsedOp = new ParsedOp(opTemplate, cfg, List.of(), getParent());
final OpsDocList stmtsDocs = OpsLoader.loadString(
opt, OpTemplateFormat.yaml, cfg.getMap(), null);
assertThat(stmtsDocs.getOps().matching("", true).size()).isEqualTo(1);
final OpTemplate opTemplate = stmtsDocs.getOps().matching("", true).get(0);
final ParsedOp parsedOp = new ParsedOp(opTemplate, cfg.getMap(), List.of(), getParent());
assertThat(parsedOp.getAsFunctionOr("d1", "invalid").apply(1L)).isEqualTo("one");
assertThat(parsedOp.getAsFunctionOr("s1", "invalid").apply(1L)).isEqualTo("static-one");
@ -104,31 +93,20 @@ public class ParsedOpTest {
final ParsedOp parsedOp = new ParsedOp(
new OpData().applyFields(Map.of(
"op", Map.of(
"field1-literal", "literalvalue1",
"field2-object", "{{NumberNameToString()}}",
"field3-template", "pre-{dyna1}-post",
"field4-map-template", Map.of(
"subfield1-object", "{{Identity(); ToString()}}"
), "field5-map-literal", Map.of(
"subfield2-literal", "LiteralValue"
)
),
"bindings", Map.of(
"dyna1", "NumberNameToString()"
))
),
ConfigModel.of(ParsedOpTest.class)
.add(Param.defaultTo("testcfg", "testval"))
.asReadOnly()
.apply(Map.of()),
List.of(),
getParent()
);
"field1-literal", "literalvalue1", "field2-object", "{{NumberNameToString()}}",
"field3-template", "pre-{dyna1}-post", "field4-map-template",
Map.of("subfield1-object", "{{Identity(); ToString()}}"), "field5-map-literal",
Map.of("subfield2-literal", "LiteralValue")), "bindings",
Map.of("dyna1", "NumberNameToString()"))),
ConfigModel.of(ParsedOpTest.class).add(Param.defaultTo("testcfg", "testval"))
.asReadOnly().apply(Map.of()).getMap(), List.of(), getParent());
final LongFunction<? extends String> f1 = parsedOp.getAsRequiredFunction("field1-literal");
final LongFunction<? extends String> f2 = parsedOp.getAsRequiredFunction("field2-object");
final LongFunction<? extends String> f3 = parsedOp.getAsRequiredFunction("field3-template");
final LongFunction<? extends Map> f4 = parsedOp.getAsRequiredFunction("field4-map-template", Map.class);
final LongFunction<? extends Map> f5 = parsedOp.getAsRequiredFunction("field5-map-literal", Map.class);
final LongFunction<? extends Map> f4 = parsedOp.getAsRequiredFunction(
"field4-map-template", Map.class);
final LongFunction<? extends Map> f5 = parsedOp.getAsRequiredFunction(
"field5-map-literal", Map.class);
assertThat(f1.apply(1)).isNotNull();
assertThat(f2.apply(2)).isNotNull();
assertThat(f3.apply(3)).isNotNull();
@ -148,21 +126,25 @@ public class ParsedOpTest {
@Test
public void testNewListBinder() {
final LongFunction<List<Object>> lb = getOp().newListBinder("dyna1", "identity", "dyna2", "identity");
final LongFunction<List<Object>> lb = getOp().newListBinder(
"dyna1", "identity", "dyna2", "identity");
final List<Object> objects = lb.apply(1);
assertThat(objects).isEqualTo(List.of("one", 1L, "one", 1L));
}
@Test
public void testNewMapBinder() {
final LongFunction<Map<String, Object>> mb = getOp().newOrderedMapBinder("dyna1", "identity", "dyna2");
final LongFunction<Map<String, Object>> mb = getOp().newOrderedMapBinder(
"dyna1", "identity", "dyna2");
final Map<String, Object> objects = mb.apply(2);
assertThat(objects).isEqualTo(Map.<String, Object>of("dyna1", "two", "identity", 2L, "dyna2", "two"));
assertThat(objects).isEqualTo(
Map.<String, Object>of("dyna1", "two", "identity", 2L, "dyna2", "two"));
}
@Test
public void testNewAryBinder() {
final LongFunction<Object[]> ab = getOp().newArrayBinder("dyna1", "dyna1", "identity", "identity");
final LongFunction<Object[]> ab = getOp().newArrayBinder(
"dyna1", "dyna1", "identity", "identity");
final Object[] objects = ab.apply(3);
assertThat(objects).isEqualTo(new Object[]{"three", "three", 3L, 3L});
}
@ -170,45 +152,21 @@ public class ParsedOpTest {
@Test
public void testLayeredListBinder() {
ParsedOp pc = new ParsedOp(
new OpData().applyFields(
Map.of(
"op", Map.of(
"alist", List.of(
List.of(
"item1",
"item2-{dyna1}"
),
Map.of(
"akey", "avalue",
"akey2", "a {dyna1} value2"
)
)
),
"bindings", Map.of(
"dyna1", "NumberNameToString()"
)
)
),
ConfigModel.of(ParsedOpTest.class)
.add(Param.defaultTo("testcfg", "testval"))
.asReadOnly()
.apply(Map.of()),
List.of(),
getParent()
);
new OpData().applyFields(Map.of(
"op", Map.of(
"alist",
List.of(
List.of("item1", "item2-{dyna1}"),
Map.of("akey", "avalue", "akey2", "a {dyna1} value2"))), "bindings",
Map.of("dyna1", "NumberNameToString()"))),
ConfigModel.of(ParsedOpTest.class).add(Param.defaultTo("testcfg", "testval"))
.asReadOnly().apply(Map.of()).getMap(), List.of(), getParent());
Map<String, Object> result = pc.getTemplateMap().apply(1);
assertThat(result).isEqualTo(
Map.of(
"alist", List.of(
List.of("item1", "item2-one"),
Map.of(
"akey", "avalue",
"akey2", "a one value2"
)
)
)
);
assertThat(result).isEqualTo(Map.of(
"alist", List.of(
List.of("item1", "item2-one"),
Map.of("akey", "avalue", "akey2", "a one value2"))));
}

10
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBConfigurable.java
View File

@ -18,6 +18,7 @@ package io.nosqlbench.nb.api.config.standard;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
/**
* All implementation types which wish to have a type-marshalled configuration
@ -65,6 +66,15 @@ public interface NBConfigurable extends NBCanConfigure, NBConfigModelProvider {
}
}
static void applyMatchingCollection(NBConfiguration cfg, Collection<?> configurables) {
for (Object configurable : configurables) {
if (configurable instanceof NBConfigurable c) {
NBConfiguration partial = c.getConfigModel().matchConfig(cfg);
c.applyConfig(partial);
}
}
}
static NBConfigModel collectModels(Class<?> of, Collection<?> configurables) {
ConfigModel model = ConfigModel.of(of);
for (Object configurable : configurables) {

161
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBConfiguration.java
View File

@ -18,25 +18,22 @@ package io.nosqlbench.nb.api.config.standard;
import io.nosqlbench.nb.api.system.NBEnvironment;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
import java.util.*;
public class NBConfiguration {
private final LinkedHashMap<String, Object> data;
private final NBConfigModel model;
private final List<NBReconfigurable> listeners = new ArrayList<>();
/**
* Create a NBConfigReader from a known valid configuration and a config model.
* This method is restricted to encourage construction of readers only by passing
* through the friendly {@link NBConfigModel#apply(Map)} method.
*
* @param model
* A configuration model, describing what is allowed to be configured by name and type.
* @param validConfig
* A valid config reader.
Create a NBConfigReader from a known valid configuration and a config model.
This method is restricted to encourage construction of readers only by passing
through the friendly {@link NBConfigModel#apply(Map)} method.
@param model
A configuration model, describing what is allowed to be configured by name and type.
@param validConfig
A valid config reader.
*/
protected NBConfiguration(NBConfigModel model, LinkedHashMap<String, Object> validConfig) {
this.data = validConfig;
@ -48,16 +45,17 @@ public class NBConfiguration {
}
public static NBConfiguration empty() {
return new NBConfiguration(ConfigModel.of(Object.class).asReadOnly(), new LinkedHashMap<>());
return new NBConfiguration(
ConfigModel.of(Object.class).asReadOnly(),
new LinkedHashMap<>());
}
/**
* Returns the value of the named parameter as {@link #getOptional(String)}, so long
* as no env vars were reference OR all env var references were found.
*
* @param name
* The name of the variable to look up
* @return An optional value, if present and (optionally) interpolated correctly from the environment
Returns the value of the named parameter as {@link #getOptional(String)}, so long
as no env vars were reference OR all env var references were found.
@param name
The name of the variable to look up
@return An optional value, if present and (optionally) interpolated correctly from the environment
*/
public Optional<String> getEnvOptional(String name) {
Optional<String> optionalValue = getOptional(name);
@ -84,44 +82,56 @@ public class NBConfiguration {
if (value instanceof String) {
Optional<String> interpolated = NBEnvironment.INSTANCE.interpolate(value.toString());
if (interpolated.isEmpty()) {
throw new NBConfigError("Unable to interpolate env and sys props in '" + value + "'");
throw new NBConfigError("Unable to interpolate env and sys props in '" +
value +
"'");
}
String result = interpolated.get();
return ConfigModel.convertValueTo(this.getClass().getSimpleName(), name, result, vclass);
return ConfigModel.convertValueTo(
this.getClass().getSimpleName(),
name,
result,
vclass);
} else {
return value;
}
}
/**
* Get a config value or object by name. This uses type inference (as a generic method)
* in addition to the internal model for type checking and ergonomic use. If you do not
* call this within an assignment or context where the Java compiler knows what type you
* are expecting, then use {@link #get(String, Class)} instead.
*
* @param name
* The name of the configuration parameter
* @param <T>
* The (inferred) generic type of the configuration value
* @return The value of type T, matching the config model type for the provided field name
Get a config value or object by name. This uses type inference (as a generic method)
in addition to the internal model for type checking and ergonomic use. If you do not
call this within an assignment or context where the Java compiler knows what type you
are expecting, then use {@link #get(String, Class)} instead.
@param name
The name of the configuration parameter
@param <T>
The (inferred) generic type of the configuration value
@return The value of type T, matching the config model type for the provided field name
*/
public <T> T get(String name) {
Param<T> param = (Param<T>) model.getNamedParams().get(name);
if (param == null) {
throw new NBConfigError("Attempted to get parameter for name '" + name + "' but this parameter has no " +
"model defined for " + this.getModel().getOf());
throw new NBConfigError("Attempted to get parameter for name '" +
name +
"' but this parameter has no " +
"model defined for " +
this.getModel().getOf());
}
// if (param.isRequired() && (param.getDefaultValue()==null) && )
// if (param.isRequired() && (param.getDefaultValue()==null) && )
Object object = this.data.get(name);
object = object != null ? object : param.getDefaultValue();
if (object == null && param.isRequired()) {
throw new NBConfigError("An object by name '" + name + "' was requested as required, and no value was" +
" defined for it. This user provided value must be set or otherwise marked optional or given a" +
" default value in the parameter model.");
throw new NBConfigError("An object by name '" +
name +
"' was requested as required, and no value was" +
" defined for it. This user provided value must be set or otherwise marked optional or given a" +
" default value in the parameter model.");
} else if (object == null && !param.isRequired()) {
throw new NBConfigError("An object by name '" + name + "' was requested as given by the config layer," +
" but no value was present, and no default was found in the config model. This is an ambiguous " +
"scenario. Either access the object as optional, or give it a default value. (code change)");
throw new NBConfigError("An object by name '" +
name +
"' was requested as given by the config layer," +
" but no value was present, and no default was found in the config model. This is an ambiguous " +
"scenario. Either access the object as optional, or give it a default value. (code change)");
}
if (param.type.isInstance(object)) {
return (T) object;
@ -130,7 +140,14 @@ public class NBConfiguration {
} else if (NBTypeConverter.canConvert(object, param.type)) {
return NBTypeConverter.convert(object, param.type);
} else {
throw new NBConfigError("Unable to assign config value for field '" + name + "' of type '" + object.getClass().getCanonicalName() + "' to the required return type '" + param.type.getCanonicalName() + "' as specified in the config model for '" + model.getOf().getCanonicalName());
throw new NBConfigError("Unable to assign config value for field '" +
name +
"' of type '" +
object.getClass().getCanonicalName() +
"' to the required return type '" +
param.type.getCanonicalName() +
"' as specified in the config model for '" +
model.getOf().getCanonicalName());
}
}
@ -138,12 +155,20 @@ public class NBConfiguration {
Param<T> param = model.getParam(name);
if (param == null) {
throw new NBConfigError("Parameter named '" + name + "' is not valid for " + model.getOf().getSimpleName() + ".");
throw new NBConfigError("Parameter named '" +
name +
"' is not valid for " +
model.getOf().getSimpleName() +
".");
}
if ((!param.isRequired()) && param.getDefaultValue() == null) {
throw new RuntimeException("Non-optional get on optional parameter " + name + "' which has no default value while configuring " + model.getOf() + "." +
"\nTo avoid user impact, ensure that ConfigModel and NBConfigurable usage are aligned.");
throw new RuntimeException("Non-optional get on optional parameter " +
name +
"' which has no default value while configuring " +
model.getOf() +
"." +
"\nTo avoid user impact, ensure that ConfigModel and NBConfigurable usage are aligned.");
}
Object o = data.get(name);
@ -178,7 +203,9 @@ public class NBConfiguration {
}
}
} else {
throw new NBConfigError("Parameter definition was not found for " + Arrays.toString(names) + ".");
throw new NBConfigError("Parameter definition was not found for " +
Arrays.toString(names) +
".");
}
}
if (o == null) {
@ -195,7 +222,11 @@ public class NBConfiguration {
} else if (NBTypeConverter.canConvert(o, type)) {
return Optional.of((T) NBTypeConverter.convert(o, type));
} else {
throw new NBConfigError("config param " + Arrays.toString(names) + " was not assignable to class '" + type.getCanonicalName() + "'");
throw new NBConfigError("config param " +
Arrays.toString(names) +
" was not assignable to class '" +
type.getCanonicalName() +
"'");
}
}
@ -208,7 +239,11 @@ public class NBConfiguration {
if (defaultValue.getClass().isAssignableFrom(o.getClass())) {
return (T) o;
}
throw new NBConfigError("config parameter '" + name + "' is not assignable to required type '" + defaultValue.getClass() + "'");
throw new NBConfigError("config parameter '" +
name +
"' is not assignable to required type '" +
defaultValue.getClass() +
"'");
}
public <T> T param(String name, Class<? extends T> vclass) {
@ -238,4 +273,36 @@ public class NBConfiguration {
return data;
}
/// see [#update(Map)]
public <T> NBConfiguration update(String fieldName, T value) {
return update(Map.of(fieldName,value));
}
/// This will create a new configuration without modifying the existing one,
/// retaining the same config model and all other values except for the modified ones.
/// Further, any reconfig listeners which are registered will be notified via the
/// [NBReconfigurable#applyConfig(NBConfiguration)] method
///
/// This eventing will occur whether or not the value was actually changed. Spurious
/// evenging of duplicate values should be considered an design bug.
///
/// Any holders of an updated configurations must maintain their own copies if necessary for
/// deltas.
public <T> NBConfiguration update(Map<String,Object> entries) {
NBConfiguration updated = model.apply(new LinkedHashMap<>(this.data) {
{
putAll(entries);
}
});
for (NBReconfigurable listener : this.listeners) {
listener.applyReconfig(updated);
}
return updated;
}
public NBConfiguration addListener(NBReconfigurable reconfigurable) {
this.listeners.add(reconfigurable);
return this;
}
}

2
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBReconfigurable.java
View File

@ -39,7 +39,7 @@ import java.util.Collection;
* {@link #applyMatching(NBConfiguration, Collection)} can be used to apply
* reconfigurations to groups of elements with a shared configuration model.
*/
public interface NBReconfigurable extends NBCanReconfigure, NBReconfigModelProvider {
public interface NBReconfigurable extends NBConfigurable, NBCanReconfigure, NBReconfigModelProvider {
/**
* This applies a configuration to an element <EM>AFTER</EM> the initial

113
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/engine/activityimpl/ActivityConfig.java Normal file
View File

@ -0,0 +1,113 @@
package io.nosqlbench.nb.api.engine.activityimpl;
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.labels.NBLabelSpec;
import io.nosqlbench.nb.api.labels.NBLabels;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.LinkedHashMap;
import java.util.Optional;
public class ActivityConfig extends NBConfiguration {
public static final String DEFAULT_ALIAS = "UNNAMEDACTIVITY";
public static final String DEFAULT_ATYPE = "stdout";
public static final String DEFAULT_CYCLES = "0";
public static final String DEFAULT_RECYCLES = "1";
public static final int DEFAULT_THREADS = 1;
public static final Logger logger = LogManager.getLogger(ActivityConfig.class);
// an alias with which to control the activity while it is running
public static final String FIELD_ALIAS = "alias";
// a file or URL containing the activity: op templates, generator bindings, ...
public static final String FIELD_ATYPE = "type";
// cycles for this activity in either "M" or "N..M" form. "M" form implies "0..M"
public static final String FIELD_CYCLES = "cycles";
public static final String FIELD_RECYCLES = "recycles";
// initial thread concurrency for this activity
public static final String FIELD_THREADS = "threads";
public static final String FIELD_LABELS = "labels";
public static final String[] field_list = {
FIELD_ALIAS, FIELD_ATYPE, FIELD_CYCLES, FIELD_THREADS, FIELD_RECYCLES
};
public ActivityConfig(NBConfiguration config) {
this(config.getModel(), config.getMap());
}
public ActivityConfig(NBConfigModel model, LinkedHashMap<String, Object> validConfig)
{
super(model, validConfig);
Optional<String> directAlias = getOptional("alias");
if (!directAlias.isPresent()) {
String indirectAlias = getOptional(ActivityConfig.FIELD_ALIAS).or(
() -> getOptional("workload")).or(() -> getOptional("driver"))
.orElse("ACTIVITYNAME");
getMap().put("alias", indirectAlias);
}
}
public String getAlias() {
return get("alias");
}
public NBLabels auxLabels() {
Optional<String> auxLabelSpec = getOptional(FIELD_LABELS);
if (auxLabelSpec.isPresent()) {
return NBLabelSpec.parseLabels(auxLabelSpec.get());
}
return NBLabels.forKV();
}
public Optional<String> getDriver() {
return getOptional("driver", "type");
// .orElseThrow(() -> new BasicError("The parameter " +
// "'driver=' is required."));
}
public void setThreads(int i) {
update("threads", i);
}
public int getThreads() {
return get(FIELD_THREADS, Integer.class);
}
public String summary() {
return String.valueOf(this);
}
public void updateLastCycle(long maxValue) {
CyclesSpec spec = CyclesSpec.parse(get("cycles", String.class));
spec = spec.withLast(maxValue);
update("cycles", spec.toString());
}
public CyclesSpec getCyclesSpec() {
return CyclesSpec.parse(get("cycles", String.class));
}
}

263
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/engine/activityimpl/ActivityDef.java
View File

@ -1,263 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.api.engine.activityimpl;
import io.nosqlbench.nb.api.components.core.NBNamedElement;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.Param;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.nb.api.labels.NBLabelSpec;
import io.nosqlbench.nb.api.labels.NBLabels;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.security.InvalidParameterException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
/**
* <p>A runtime definition for an activity.</p>
* <p>Instances of ActivityDef hold control values for the execution of a single activity.
* Each thread of the related activity is initialized with the associated ActivityDef.
* When the ActivityDef is modified, interested activity threads are notified so that
* they can dynamically adjust.</p>
* <p>The canonical values for all parameters are kept internally in the parameter map.
* Essentially, ActivityDef is just a type-aware wrapper around a thread-safe parameter map,
* with an atomic change counter which can be used to signal changes to observers.</p>
*/
public class ActivityDef implements NBNamedElement {
// milliseconds between cycles per thread, for slow tests only
public static final String DEFAULT_ALIAS = "UNNAMEDACTIVITY";
public static final String DEFAULT_ATYPE = "stdout";
public static final String DEFAULT_CYCLES = "0";
public static final String DEFAULT_RECYCLES = "1";
public static final int DEFAULT_THREADS = 1;
public static final Logger logger = LogManager.getLogger(ActivityDef.class);
// an alias with which to control the activity while it is running
public static final String FIELD_ALIAS = "alias";
// a file or URL containing the activity: op templates, generator bindings, ...
public static final String FIELD_ATYPE = "type";
// cycles for this activity in either "M" or "N..M" form. "M" form implies "0..M"
public static final String FIELD_CYCLES = "cycles";
public static final String FIELD_RECYCLES = "recycles";
// initial thread concurrency for this activity
public static final String FIELD_THREADS = "threads";
public static final String[] field_list = {
FIELD_ALIAS, FIELD_ATYPE, FIELD_CYCLES, FIELD_THREADS, FIELD_RECYCLES
};
// parameter map has its own internal atomic map
private final ParameterMap parameterMap;
private CyclesSpec cyclesSpec;
private CyclesSpec reCyclesSpec;
public ActivityDef(ParameterMap parameterMap) {
this.parameterMap = parameterMap;
}
//public static Optional<ActivityDef> parseActivityDefOptionally(String namedActivitySpec) {
// try {
// ActivityDef activityDef = parseActivityDef(namedActivitySpec);
// return Optional.of(activityDef);
// } catch (Exception e) {
// return Optional.empty();
// }
//}
public static ActivityDef parseActivityDef(String namedActivitySpec) {
Optional<ParameterMap> activityParameterMap = ParameterMap.parseParams(namedActivitySpec);
ActivityDef activityDef = new ActivityDef(activityParameterMap.orElseThrow(
() -> new RuntimeException("Unable to parse:" + namedActivitySpec)
));
logger.info("parsed activityDef {} to-> {}", namedActivitySpec, activityDef);
return activityDef;
}
public String toString() {
return "ActivityDef:" + parameterMap.toString();
}
/**
* The alias that the associated activity instance is known by.
*
* @return the alias
*/
public String getAlias() {
return parameterMap.getOptionalString("alias").orElse(DEFAULT_ALIAS);
}
/**
* The first cycle that will be used for execution of this activity, inclusive.
* If the value is provided as a range as in 0..10, then the first number is the start cycle
* and the second number is the end cycle +1. Effectively, cycle ranges
* are [closed,open) intervals, as in [min..max)
*
* @return the long start cycle
*/
public long getStartCycle() {
return getCyclesSpec().first_inclusive();
}
public void setStartCycle(long firstCycleInclusive) {
cyclesSpec=getCyclesSpec().withFirst(firstCycleInclusive);
}
public void setStartCycle(String firstCycleInclusive) {
cyclesSpec=getCyclesSpec().withFirst(firstCycleInclusive);
}
public void setEndCycle(long lastCycleExclusive) {
cyclesSpec=getCyclesSpec().withLast(lastCycleExclusive);
}
public void setEndCycle(String lastCycleExclusive) {
cyclesSpec=getCyclesSpec().withLast(lastCycleExclusive);
}
/**
* The last cycle that will be used for execution of this activity, inclusive.
*
* @return the long end cycle
*/
public long getEndCycle() {
return getCyclesSpec().last_exclusive();
}
/**
* The number of threads (AKA slots) that the associated activity should currently be using.
*
* @return target thread count
*/
public int getThreads() {
return parameterMap.getOptionalInteger(FIELD_THREADS).orElse(DEFAULT_THREADS);
}
public void setThreads(int threads) {
parameterMap.set(FIELD_THREADS, threads);
}
/**
* Get the parameter map, which is the backing-store for all data within an ActivityDef.
*
* @return the parameter map
*/
public ParameterMap getParams() {
return parameterMap;
}
public AtomicLong getChangeCounter() {
return parameterMap.getChangeCounter();
}
public void setCycles(String cycles) {
parameterMap.set(FIELD_CYCLES, cycles);
this.cyclesSpec=CyclesSpec.parse(cycles);
checkInvariants();
}
public String getCycleSummary() {
return getCyclesSpec().summary();
}
public synchronized long getCycleCount() {
return getCyclesSpec().cycle_count();
}
public synchronized CyclesSpec getCyclesSpec() {
if (this.cyclesSpec==null) {
this.cyclesSpec = CyclesSpec.parse(parameterMap.getOptionalString(FIELD_CYCLES).orElse(DEFAULT_CYCLES));
}
return this.cyclesSpec;
}
public synchronized CyclesSpec getRecyclesSpec() {
if (this.reCyclesSpec==null) {
this.reCyclesSpec = CyclesSpec.parse(parameterMap.getOptionalString(FIELD_RECYCLES).orElse(DEFAULT_RECYCLES));
}
return this.reCyclesSpec;
}
private void checkInvariants() {
if (getStartCycle() >= getEndCycle()) {
throw new InvalidParameterException("Start cycle must be strictly less than end cycle, but they are [" + getStartCycle() + ',' + getEndCycle() + ')');
}
}
@Override
public String getName() {
return getAlias();
}
public ActivityDef deprecate(String deprecatedName, String newName) {
Object deprecatedParam = this.parameterMap.get(deprecatedName);
if (null == deprecatedParam) {
return this;
}
if (deprecatedParam instanceof CharSequence chars) {
if (this.parameterMap.containsKey(newName)) {
throw new BasicError("You have specified activity param '" + deprecatedName + "' in addition to the valid name '" + newName + "'. Remove '" + deprecatedName + "'.");
}
if (!newName.equals("driver")) {
logger.warn("Auto replacing deprecated activity param '{}={}' with new '{}={}'.", deprecatedName, chars, newName, chars);
}
parameterMap.put(newName, parameterMap.remove(deprecatedName));
} else {
throw new BasicError("Can't replace deprecated name with value of type " + deprecatedName.getClass().getCanonicalName());
}
return this;
}
public NBLabels auxLabels() {
Optional<String> auxLabelSpec = getParams().getOptionalString("labels");
if (auxLabelSpec.isPresent()) {
return NBLabelSpec.parseLabels(auxLabelSpec.get());
}
return NBLabels.forKV();
}
public NBConfigModel getConfigModel() {
ConfigModel cfgmodel = ConfigModel.of(this.getClass());
Map<String, String> params = parameterMap.getStringStringMap();
params.forEach((k, v) -> {
cfgmodel.add(Param.defaultTo(k, v, "activity parameter found on command line"));
});
cfgmodel.add(Param.defaultTo(FIELD_ALIAS, DEFAULT_ALIAS).setDescription("The alias for the operations"));
cfgmodel.add(Param.defaultTo(FIELD_ATYPE, DEFAULT_ATYPE).setDescription("The default adapter type is 'stdout'"));
cfgmodel.add(Param.defaultTo(FIELD_CYCLES, DEFAULT_CYCLES).setDescription("The default number of cycles to test is '0'"));
cfgmodel.add(Param.defaultTo(FIELD_THREADS, DEFAULT_THREADS).setDescription("The default number of threads for testing is '1'"));
cfgmodel.add(Param.defaultTo(FIELD_RECYCLES, DEFAULT_RECYCLES).setDescription("The default number of recycles to test is '1'"));
cfgmodel.add(Param.optional("labels", String.class).setDescription("Metric labels for this activity"));
cfgmodel.add(Param.optional("tags", String.class).setDescription("Tags for selecting workload op templates"));
cfgmodel.add(Param.defaultTo("driver", DEFAULT_ATYPE).setDescription("The default adapter driver is 'stdout'"));
cfgmodel.add(Param.optional("workload", String.class).setDescription("The test workload"));
cfgmodel.add(Param.optional("yaml", String.class).setDescription("The test workload"));
cfgmodel.add(Param.defaultTo("async", 1,"Inflight Ops"));
cfgmodel.add(Param.defaultTo("maxtries", 10,"Maximum number of retries"));
cfgmodel.add(Param.defaultTo("interval", 1000,"Action interval"));
cfgmodel.add(Param.defaultTo("hdr_digits", 4,"HDR Digits"));
cfgmodel.add(Param.optional("errors").setDescription("Error handling method"));
cfgmodel.add(Param.optional("striderate").setDescription("Rate limiting stride"));
List<String> rates = Arrays.asList("cyclerate", "targetrate", "rate");
cfgmodel.add(Param.optional(rates, String.class, "Rate limit"));
return cfgmodel.asReadOnly();
}
}

3
nb-engine/nb-engine-clients/src/main/java/io/nosqlbench/engine/clients/grafana/analyzer/GrafanaRegionAnalyzer.java
View File

@ -74,7 +74,8 @@ public class GrafanaRegionAnalyzer implements Runnable {
//[2020-12-15T05:04:37.232Z[GMT] - 2020-12-15T05:04:37.232Z[GMT]]
//span:interval
//details:
// params: ActivityDef:(4)/{keycount=5000000000L, hosts=node1, main-cycles=500, threads=1, workload=./keyvalue.yaml, cycles=2, stride=2, tags=block:'schema.*', password=cassandra, rf=3, pooling=16:16:500, driver=cql, rampup-cycles=5000000000, alias=keyvalue_default_schema, valuecount=5000000000L, errors=count, username=cassandra}
// params: ActivityConfig:(4)/{keycount=5000000000L, hosts=node1, main-cycles=500,
// threads=1, workload=./keyvalue.yaml, cycles=2, stride=2, tags=block:'schema.*', password=cassandra, rf=3, pooling=16:16:500, driver=cql, rampup-cycles=5000000000, alias=keyvalue_default_schema, valuecount=5000000000L, errors=count, username=cassandra}
//labels:
// layer: StandardActivity
// alias: keyvalue_default_schema

36
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/core/ActivityDefObserver.java
View File

@ -1,36 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityapi.core;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
/**
* Decorator interface for getting notified when an activities parameters are changed at runtime.
*
* This can be optionally implemented by Any Motor, Input, or Action. The eventing is mediated
* through the ActivityExecutor in order to isolate the programmatic API from the internal API.
*/
public interface ActivityDefObserver {
void onActivityDefUpdate(ActivityDef activityDef);
static void apply(ActivityDef def, Object... candidates) {
for (Object candidate : candidates) {
if (candidate instanceof ActivityDefObserver observer) {
observer.onActivityDefUpdate(def);
}
}
}
}

5
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/core/ComponentActivityInstrumentation.java
View File

@ -19,7 +19,6 @@ package io.nosqlbench.engine.api.activityapi.core;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ParameterMap;
import io.nosqlbench.nb.api.engine.metrics.instruments.*;
@ -30,8 +29,6 @@ public class ComponentActivityInstrumentation {
private static final String RESPONSE_TIME = "_responsetime";
private final Activity activity;
private final ActivityDef def;
private final ParameterMap params;
private final int hdrdigits;
private NBMetricTimer readInputTimer;
private NBMetricTimer stridesServiceTimer;
@ -53,9 +50,7 @@ public class ComponentActivityInstrumentation {
public ComponentActivityInstrumentation(final Activity activity) {
this.activity = activity;
def = activity.getActivityDef();
this.hdrdigits = activity.getComponentProp("hdr_digits").map(Integer::parseInt).orElse(3);
params = this.def.getParams();
initMetrics();
}

6
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/core/IActivityWiring.java
View File

@ -23,7 +23,7 @@ import io.nosqlbench.engine.api.activityapi.input.InputDispenser;
import io.nosqlbench.engine.api.activityapi.output.OutputDispenser;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import java.io.InputStream;
import java.io.PrintWriter;
@ -32,9 +32,9 @@ import java.io.PrintWriter;
* Provides the components needed to build and run an activity a runtime.
* The easiest way to build a useful StandardActivity is to extend {@link Activity}.
*/
public interface IActivityWiring extends Comparable<IActivityWiring>, ActivityDefObserver, ProgressCapable, StateCapable, NBComponent {
public interface IActivityWiring extends Comparable<IActivityWiring>, ProgressCapable, StateCapable, NBComponent {
ActivityDef getActivityDef();
ActivityConfig getActivityConfig();
MotorDispenser<?> getMotorDispenserDelegate();

8
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/core/MotorDispenser.java
View File

@ -15,7 +15,8 @@
*/
package io.nosqlbench.engine.api.activityapi.core;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
/**
* A MotorDispenser is created for each StandardActivity instance within a scenario.
@ -29,9 +30,10 @@ public interface MotorDispenser<T> {
* Resolve (find or create) a Motor instance for the slot specified.
* The motor is not required to be per-slot (per-thread), but any shared inputs motors be thread safe.
*
* @param activityDef the ActivityDef which will be used to parameterize the returned motor
* @param activityConfig the activity config which will be used to parameterize the returned
* motor
* @param slot The numbered slot within the activity instance for this motor
* @return A new or cached Motor for the specified slot.
*/
Motor<T> getMotor(ActivityDef activityDef, int slot);
Motor<T> getMotor(ActivityConfig activityConfig, int slot);
}

4
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/cyclelog/inputs/cyclelog/CycleLogInput.java
View File

@ -52,9 +52,9 @@ public class CycleLogInput extends NBBaseComponent implements Input, AutoCloseab
public CycleLogInput(Activity activity) {
super(activity, NBLabels.forKV("input","cyclelog"));
SimpleConfig conf = new SimpleConfig(activity.getActivityDef(), "input");
SimpleConfig conf = new SimpleConfig(activity, "input");
mbb =
initMappedBuffer(conf.getString("file").orElse(activity.getActivityDef().getAlias()) +
initMappedBuffer(conf.getString("file").orElse(activity.getAlias()) +
".cyclelog");
cycleResultSegmentIterator = iterator();
segmentIter = cycleResultSegmentIterator.next().iterator();

2
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/cyclelog/outputs/cyclelog/CycleLogOutput.java
View File

@ -68,7 +68,7 @@ public class CycleLogOutput extends NBBaseComponent implements Output, CanFilter
SimpleConfig conf = new SimpleConfig(wiring, "output");
this.extentSizeInSpans = conf.getInteger("extentSize").orElse(1000);
this.outputFile = new File(conf.getString("file").orElse(wiring.getActivityDef().getAlias()) +
this.outputFile = new File(conf.getString("file").orElse(wiring.getConfig().getAlias()) +
".cyclelog");

2
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/cyclelog/outputs/logger/LoggingMarkerDispenser.java
View File

@ -35,7 +35,7 @@ public class LoggingMarkerDispenser implements OutputDispenser {
@Override
public Output getOutput(long slot) {
return new LoggingOutput(activity.getActivityDef(), slot);
return new LoggingOutput(activity.getConfig() , slot);
}
}

8
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/cyclelog/outputs/logger/LoggingOutput.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.engine.api.activityapi.cyclelog.outputs.logger;
import io.nosqlbench.engine.api.activityapi.cyclelog.buffers.results.ResultReadable;
import io.nosqlbench.engine.api.activityapi.cyclelog.inputs.cyclelog.CanFilterResultValue;
import io.nosqlbench.engine.api.activityapi.output.Output;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@ -28,12 +28,12 @@ import java.util.function.Predicate;
public class LoggingOutput implements Output, CanFilterResultValue {
private final static Logger logger = LogManager.getLogger(LoggingOutput.class);
private final ActivityDef def;
private final NBConfiguration def;
private final long slot;
private final ThreadLocal<StringBuilder> sb = ThreadLocal.withInitial(StringBuilder::new);
private Predicate<ResultReadable> filter;
public LoggingOutput(ActivityDef def, long slot) {
public LoggingOutput(NBConfiguration def, long slot) {
this.def = def;
this.slot = slot;
}
@ -45,7 +45,7 @@ public class LoggingOutput implements Output, CanFilterResultValue {
}
sb.get().setLength(0);
sb.get()
.append("activity=").append(def.getAlias())
.append("activity=").append(def.get("alias",String.class))
.append(",cycle=").append(completedCycle)
.append(",result=").append((byte) (result & 127));
logger.info(() -> sb.get().toString());

6
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/ErrorMetrics.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityapi.errorhandling;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.engine.api.metrics.ExceptionCountMetrics;
import io.nosqlbench.engine.api.metrics.ExceptionExpectedResultVerificationMetrics;
@ -46,7 +46,7 @@ public class ErrorMetrics {
public synchronized ExceptionHistoMetrics getExceptionHistoMetrics() {
if (null == exceptionHistoMetrics)
this.exceptionHistoMetrics = new ExceptionHistoMetrics(this.parent, ActivityDef.parseActivityDef(""));
this.exceptionHistoMetrics = new ExceptionHistoMetrics(this.parent, Activity.configFor(""));
return this.exceptionHistoMetrics;
}
@ -57,7 +57,7 @@ public class ErrorMetrics {
public synchronized ExceptionTimerMetrics getExceptionTimerMetrics() {
if (null == exceptionTimerMetrics)
this.exceptionTimerMetrics = new ExceptionTimerMetrics(this.parent, ActivityDef.parseActivityDef(""));
this.exceptionTimerMetrics = new ExceptionTimerMetrics(this.parent, Activity.configFor(""));
return this.exceptionTimerMetrics;
}

8
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/CoreServices.java
View File

@ -53,8 +53,8 @@ public class CoreServices {
}
public static <A> Optional<Predicate<ResultReadable>> getOutputFilter(ActivityWiring activity) {
String paramdata= activity.getParams().getOptionalString("of")
.orElse(activity.getParams().getOptionalString("outputfilter").orElse(null));
String paramdata= activity.getConfig().getOptional("of")
.orElse(activity.getConfig().getOptional("outputfilter").orElse(null));
if (paramdata==null) {
return Optional.empty();
}
@ -81,8 +81,8 @@ public class CoreServices {
}
public static <A> Optional<Predicate<ResultReadable>> getInputFilter(Activity activity) {
String paramdata= activity.getParams().getOptionalString("if")
.orElse(activity.getParams().getOptionalString("inputfilter").orElse(null));
String paramdata= activity.getConfig().getOptional("if")
.orElse(activity.getConfig().getOptional("inputfilter").orElse(null));
if (paramdata==null) {
return Optional.empty();
}

8
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/action/CoreAction.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.activityimpl.action;
import io.nosqlbench.engine.api.activityapi.core.SyncAction;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@ -26,12 +26,12 @@ public class CoreAction implements SyncAction {
private final int interval;
private final int slot;
private final ActivityDef activityDef;
private final ActivityConfig activityDef;
public CoreAction(ActivityDef activityDef, int slot) {
public CoreAction(ActivityConfig activityDef, int slot) {
this.activityDef = activityDef;
this.slot = slot;
this.interval = activityDef.getParams().getOptionalInteger("interval").orElse(1000);
this.interval = activityDef.getOptional("interval").map(Integer::parseInt).orElse(1000);
}
@Override

2
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/action/CoreActionDispenser.java
View File

@ -36,6 +36,6 @@ public class CoreActionDispenser implements ActionDispenser {
@Override
public SyncAction getAction(int slot) {
return new CoreAction(activity.getActivityDef(), slot);
return new CoreAction(activity.getConfig(), slot);
}
}

215
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/input/AtomicInput.java
View File

@ -16,11 +16,11 @@
package io.nosqlbench.engine.api.activityimpl.input;
import com.codahale.metrics.Gauge;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.config.standard.*;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import io.nosqlbench.nb.api.engine.activityimpl.CyclesSpec;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.cyclelog.buffers.results.CycleSegment;
import io.nosqlbench.engine.api.activityapi.input.Input;
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
@ -30,20 +30,21 @@ import org.apache.logging.log4j.Logger;
import java.util.concurrent.atomic.AtomicLong;
/**
* <p>TODO: This documentation is out of date as of 2.0.0
* <p>This input will provide threadsafe access to a sequence of long values.</p>
* <p>Changes to the cycles or the targetrate will affect the provided inputs.
* If the min or max cycle is changed, then these are re-applied first to the
* max cycle and then to the min cycle. If the min cycle is changed, then the
* next cycle value is set to the assigned min value. Otherwise, the cycle
* will continue as usual till it reaches the max value. The ability to start
* the input while running by applying a new set of parameters makes it possible
* to re-trigger a sequence of inputs during a test.</p>
* <p>This input, and Inputs in general do not actively prevent usage of values
* after the max value. They simply expose it to callers. It is up to the
* caller to check the value to determine when the input is deemed "used up."</p>
*/
public class AtomicInput extends NBBaseComponent implements Input, ActivityDefObserver, Gauge<Long> {
<p>TODO: This documentation is out of date as of 2.0.0
<p>This input will provide threadsafe access to a sequence of long values.</p>
<p>Changes to the cycles or the targetrate will affect the provided inputs.
If the min or max cycle is changed, then these are re-applied first to the
max cycle and then to the min cycle. If the min cycle is changed, then the
next cycle value is set to the assigned min value. Otherwise, the cycle
will continue as usual till it reaches the max value. The ability to start
the input while running by applying a new set of parameters makes it possible
to re-trigger a sequence of inputs during a test.</p>
<p>This input, and Inputs in general do not actively prevent usage of values
after the max value. They simply expose it to callers. It is up to the
caller to check the value to determine when the input is deemed "used up."</p> */
public class AtomicInput extends NBBaseComponent
implements Input, NBConfigurable, NBReconfigurable, Gauge<Long>
{
private final static Logger logger = LogManager.getLogger(AtomicInput.class);
private final AtomicLong cycle_value = new AtomicLong(0L);
@ -54,69 +55,43 @@ public class AtomicInput extends NBBaseComponent implements Input, ActivityDefOb
private final AtomicLong recycle_value = new AtomicLong(0L);
private final AtomicLong recycles_max = new AtomicLong(0L);
private final long startedAt = System.currentTimeMillis();
private NBConfiguration config;
private final ActivityDef activityDef;
public AtomicInput(NBComponent parent, ActivityDef activityDef) {
public AtomicInput(Activity parent) {
super(parent);
this.activityDef = activityDef;
onActivityDefUpdate(activityDef);
applyConfig(parent.getConfig());
create().gauge(
"input_cycles_first",
() -> (double) this.cycles_min.get(),
MetricCategory.Config,
"The first cycle of the cycle interval, inclusive"
);
"input_cycles_first", () -> (double) this.cycles_min.get(), MetricCategory.Config,
"The first cycle of the cycle interval, inclusive");
create().gauge(
"input_cycles_last",
() -> (double) this.cycles_max.get(),
MetricCategory.Config,
"The last cycle of the cycle interval, exclusive"
);
"input_cycles_last", () -> (double) this.cycles_max.get(), MetricCategory.Config,
"The last cycle of the cycle interval, exclusive");
create().gauge(
"input_cycle",
() -> (double) this.cycle_value.get(),
MetricCategory.Core,
"The next input cycle that will be dispatched to a thread"
);
"input_cycle", () -> (double) this.cycle_value.get(), MetricCategory.Core,
"The next input cycle that will be dispatched to a thread");
create().gauge(
"input_cycles_total",
this::getTotalCycles,
MetricCategory.Config,
"The total number of cycles to be executed"
);
"input_cycles_total", this::getTotalCycles, MetricCategory.Config,
"The total number of cycles to be executed");
create().gauge(
"input_recycles_first",
() -> (double) this.recycles_min.get(),
MetricCategory.Config,
"The first recycle value, inclusive"
);
"input_recycles_first", () -> (double) this.recycles_min.get(), MetricCategory.Config,
"The first recycle value, inclusive");
create().gauge(
"input_recycles_last",
() -> (double) this.recycles_max.get(),
MetricCategory.Config,
"The last recycle value, exclusive"
);
"input_recycles_last", () -> (double) this.recycles_max.get(), MetricCategory.Config,
"The last recycle value, exclusive");
create().gauge(
"input_recycle",
() -> (double) this.recycle_value.get(),
MetricCategory.Core,
"The next recycle value that will be dispatched once cycles are completed"
);
"input_recycle", () -> (double) this.recycle_value.get(), MetricCategory.Core,
"The next recycle value that will be dispatched once cycles are completed");
create().gauge(
"input_recycles_total",
this::getTotalRecycles,
MetricCategory.Config,
"The total number of recycles to be executed, within which each set of cycles will be executed"
);
"input_recycles_total", this::getTotalRecycles, MetricCategory.Config,
"The total number of recycles to be executed, within which each set of cycles will be executed");
}
private double getTotalRecycles() {
return ((double)this.recycles_max.get())-((double)this.recycles_min.get());
return ((double) this.recycles_max.get()) - ((double) this.recycles_min.get());
}
private double getTotalCycles() {
return ((double)this.cycles_max.get())-((double)this.cycles_min.get());
return ((double) this.cycles_max.get()) - ((double) this.cycles_min.get());
}
@Override
@ -124,60 +99,114 @@ public class AtomicInput extends NBBaseComponent implements Input, ActivityDefOb
while (true) {
long currentStrideStart = this.cycle_value.get();
long nextStrideStart = currentStrideStart + stride;
if (nextStrideStart > cycles_max.get()) { // This indicates a stride boundary crossing the end
if (nextStrideStart >
cycles_max.get())
{ // This indicates a stride boundary crossing the end
recycle_value.getAndIncrement();
if (recycle_value.get() >= recycles_max.get()) {
logger.trace(() -> "Exhausted input for " + activityDef.getAlias() + " at " + currentStrideStart + ", recycle " +
"count " + recycle_value.get());
logger.trace(() -> "Exhausted input for " +
description() +
" at " +
currentStrideStart +
", recycle " +
"count " +
recycle_value.get());
return null;
} else {
cycle_value.set(cycles_min.get());
logger.trace(() -> "recycling input for " + activityDef.getAlias() + " recycle:" + recycle_value.get());
logger.trace(() -> "recycling input for " +
description() +
" recycle:" +
recycle_value.get());
continue;
}
}
if (cycle_value.compareAndSet(currentStrideStart, nextStrideStart)) {
return new InputInterval.Segment(recycle_value.get(), currentStrideStart, nextStrideStart);
return new InputInterval.Segment(
recycle_value.get(), currentStrideStart,
nextStrideStart);
}
}
}
@Override
public String toString() {
return "AtomicInput{" +
"cycleValue=" + cycle_value +
", min=" + cycles_min +
", max=" + cycles_max +
", activity=" + activityDef.getAlias() +
'}';
return "AtomicInput(" +
description() +
"){cycleValue=" +
cycle_value +
", min=" +
cycles_min +
", max=" +
cycles_max +
'}';
}
@Override
public void onActivityDefUpdate(ActivityDef activityDef) {
CyclesSpec recyclesSpec = activityDef.getRecyclesSpec();
CyclesSpec cyclesSpec = activityDef.getCyclesSpec();
public void applyConfig(NBConfiguration cfg) {
this.config = this.getConfigModel().matchConfig(cfg);
CyclesSpec recyclesSpec = CyclesSpec.parse(cfg.get(ActivityConfig.FIELD_RECYCLES));
CyclesSpec cyclesSpec = CyclesSpec.parse(cfg.get(ActivityConfig.FIELD_CYCLES));
if (cyclesSpec.cycle_count() == 0) {
throw new RuntimeException("You specified cycles, but the range specified means zero " +
"cycles: " +
cyclesSpec);
}
if (recyclesSpec.cycle_count() == 0) {
throw new RuntimeException("You specified recycles, but the range specified means " +
"zero " +
"recycles: " +
recyclesSpec);
}
cycles_max.set(cyclesSpec.last_exclusive());
if (cycles_min.get() != cyclesSpec.first_inclusive()) {
logger.info(() -> "resetting first cycle (inclusive) value to: cycle[" + cycles_min.get() + "->" + cyclesSpec.first_inclusive() + "] " +
" start[" + cycle_value.get() + "->" + cycles_min.get() + "]");
logger.info(() -> "resetting first cycle (inclusive) value to: cycle[" +
cycles_min.get() +
"->" +
cyclesSpec.first_inclusive() +
"] " +
" start[" +
cycle_value.get() +
"->" +
cycles_min.get() +
"]");
cycles_min.set(cyclesSpec.first_inclusive());
cycle_value.set(cycles_min.get());
}
if (cycles_max.get() != cyclesSpec.last_exclusive()) {
logger.info(() -> "resetting last cycle (exclusive) value to: cycle[" + cycles_max.get() + "->" + cyclesSpec.last_exclusive() + "]");
logger.info(() -> "resetting last cycle (exclusive) value to: cycle[" +
cycles_max.get() +
"->" +
cyclesSpec.last_exclusive() +
"]");
cycles_max.set(cyclesSpec.last_exclusive());
}
recycles_max.set(recyclesSpec.last_exclusive());
if (recycles_min.get() != recyclesSpec.first_inclusive()) {
logger.info(() -> "resetting recycle value to new start: recycle[" + recycles_min.get() + "->" + recyclesSpec.first_inclusive() + "] " +
" start[" + recycle_value.get() + "->" + recycles_min.get() + "]");
logger.info(() -> "resetting recycle value to new start: recycle[" +
recycles_min.get() +
"->" +
recyclesSpec.first_inclusive() +
"] " +
" start[" +
recycle_value.get() +
"->" +
recycles_min.get() +
"]");
recycles_min.set(recyclesSpec.first_inclusive());
recycle_value.set(recyclesSpec.first_inclusive());
}
if (recycles_max.get() != recyclesSpec.last_exclusive()) {
logger.info(() -> "resetting last recycle (exclusive) value to: recycle[" + recycles_max.get() + "->" + recyclesSpec.last_exclusive() + "]");
logger.info(() -> "resetting last recycle (exclusive) value to: recycle[" +
recycles_max.get() +
"->" +
recyclesSpec.last_exclusive() +
"]");
recycles_max.set(recyclesSpec.last_exclusive());
}
@ -196,4 +225,22 @@ public class AtomicInput extends NBBaseComponent implements Input, ActivityDefOb
public Long getValue() {
return this.cycle_value.get();
}
@Override
public void applyReconfig(NBConfiguration recfg) {
this.applyConfig(recfg);
}
@Override
public NBConfigModel getReconfigModel() {
return getConfigModel();
}
@Override
public NBConfigModel getConfigModel() {
return ConfigModel.of(AtomicInput.class)
.add(Param.required(ActivityConfig.FIELD_CYCLES, String.class))
.add(Param.required(ActivityConfig.FIELD_RECYCLES, String.class)).asReadOnly();
}
}

2
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/input/CoreInputDispenser.java
View File

@ -44,7 +44,7 @@ public class CoreInputDispenser implements InputDispenser, ActivitiesAware {
}
private synchronized Input createInput(long slot) {
SimpleConfig conf = new SimpleConfig(activity.getActivityDef(), "input");
SimpleConfig conf = new SimpleConfig(activity, "input");
String inputType = conf.getString("type").orElse("atomicseq");
InputType inputTypeImpl = InputType.FINDER.getOrThrow(inputType);
InputDispenser inputDispenser = inputTypeImpl.getInputDispenser(activity);

2
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/input/TargetRateInputType.java
View File

@ -37,7 +37,7 @@ public class TargetRateInputType implements InputType {
public Dispenser(Activity activity) {
this.activity = activity;
this.input = new AtomicInput(activity, activity.getActivityDef());
this.input = new AtomicInput(activity);
}
@Override

6
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/marker/ContiguousOutputChunker.java
View File

@ -80,9 +80,9 @@ public class ContiguousOutputChunker implements Output {
// // efficient marker extent handling. The ability to use segmented inputs with markers will
// // come in a future append.
// }
this.min = new AtomicLong(activity.getActivityDef().getStartCycle());
this.nextMin = new AtomicLong(activity.getActivityDef().getEndCycle());
long stride = activity.getParams().getOptionalLong("stride").orElse(1L);
this.min = new AtomicLong(activity.getCyclesSpec().first_inclusive());
this.nextMin = new AtomicLong(activity.getCyclesSpec().last_exclusive());
long stride = activity.getConfig().getOptional(Long.class,"stride").orElse(1L);
long cycleCount = nextMin.get() - min.get();
if ((cycleCount % stride) != 0) {
throw new RuntimeException("stride must evenly divide into cycles.");

104
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java
View File

@ -27,7 +27,7 @@ import io.nosqlbench.engine.api.activityapi.output.Output;
import io.nosqlbench.engine.api.activityapi.simrate.RateLimiter;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.config.standard.*;
import io.nosqlbench.nb.api.labels.NBLabels;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@ -47,7 +47,7 @@ import static io.nosqlbench.engine.api.activityapi.core.RunState.*;
This motor implementation splits the handling of sync and async actions with a hard
fork in the middle to limit potential breakage of the prior sync implementation
with new async logic. */
public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver, Motor<D>, Stoppable {
public class CoreMotor<D> extends NBBaseComponent implements Motor<D>, Stoppable, NBReconfigurable {
private static final Logger logger = LogManager.getLogger(CoreMotor.class);
@ -74,7 +74,7 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
private int stride = 1;
private OpTracker<D> opTracker;
private NBConfiguration config;
/**
@ -87,15 +87,20 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
A LongSupplier which provides the cycle number inputs.
*/
public CoreMotor(
Activity activity, long slotId, Input input, SyncAction action,
Output output) {
super(activity, NBLabels.forKV("motor", "coremotor"));
Activity activity,
long slotId,
Input input,
SyncAction action,
Output output
)
{
super(activity, NBLabels.forKV("motor", slotId));
this.activity = activity;
this.slotId = slotId;
setInput(input);
setResultOutput(output);
motorState = new MotorState(slotId, activity.getRunStateTally());
onActivityDefUpdate(activity.getActivityDef());
applyConfig(activity.getConfig());
this.action = action;
int hdrdigits = activity.getComponentProp("hdr_digits").map(Integer::parseInt).orElse(3);
@ -150,8 +155,9 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
cycleRateLimiter = activity.getCycleLimiter();
if (motorState.get() == Finished) {
logger.warn(
() -> "Input was already exhausted for slot " + slotId + ", remaining in finished state.");
logger.warn(() -> "Input was already exhausted for slot " +
slotId +
", remaining in finished state.");
}
action.init();
@ -174,13 +180,15 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
CycleSegment cycleSegment = null;
CycleResultSegmentBuffer segBuffer = new CycleResultSegmentBuffer(stride);
try (Timer.Context inputTime = inputTimer.time()) {
try (Timer.Context inputTime = activity.metrics.inputTimer.time()) {
cycleSegment = input.getInputSegment(stride);
}
if (cycleSegment == null) {
logger.trace(
() -> "input exhausted (input " + input + ") via null segment, stopping motor thread " + slotId);
logger.trace(() -> "input exhausted (input " +
input +
") via null segment, stopping motor thread " +
slotId);
motorState.enterState(Finished);
continue;
}
@ -198,16 +206,20 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
long cyclenum = cycleSegment.nextCycle();
if (cyclenum < 0) {
if (cycleSegment.isExhausted()) {
logger.trace(
() -> "input exhausted (input " + input + ") via negative read, stopping motor thread " + slotId);
logger.trace(() -> "input exhausted (input " +
input +
") via negative read, stopping motor thread " +
slotId);
motorState.enterState(Finished);
continue;
}
}
if (motorState.get() != Running) {
logger.trace(
() -> "motor stopped after input (input " + cyclenum + "), stopping motor thread " + slotId);
logger.trace(() -> "motor stopped after input (input " +
cyclenum +
"), stopping motor thread " +
slotId);
continue;
}
int result = -1;
@ -226,7 +238,7 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
throw e;
} finally {
long cycleEnd = System.nanoTime();
cycleServiceTimer.update(
activity.metrics.cycleServiceTimer.update(
(cycleEnd - cycleStart) + cycleDelay, TimeUnit.NANOSECONDS);
}
segBuffer.append(cyclenum, result);
@ -234,10 +246,9 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
} finally {
long strideEnd = System.nanoTime();
stridesServiceTimer.update(
activity.metrics.stridesServiceTimer.update(
(strideEnd - strideStart) + strideDelay,
TimeUnit.NANOSECONDS
);
TimeUnit.NANOSECONDS);
}
if (output != null) {
@ -245,8 +256,12 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
try {
output.onCycleResultSegment(outputBuffer);
} catch (Exception t) {
logger.error(
() -> "Error while feeding result segment " + outputBuffer + " to output '" + output + "', error:" + t);
logger.error(() -> "Error while feeding result segment " +
outputBuffer +
" to output '" +
output +
"', error:" +
t);
throw t;
}
}
@ -254,11 +269,13 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
if (motorState.get() == Stopping) {
motorState.enterState(Stopped);
logger.trace(
() -> Thread.currentThread().getName() + " shutting down as " + motorState.get());
logger.trace(() -> Thread.currentThread().getName() +
" shutting down as " +
motorState.get());
} else if (motorState.get() == Finished) {
logger.trace(
() -> Thread.currentThread().getName() + " shutting down as " + motorState.get());
logger.trace(() -> Thread.currentThread().getName() +
" shutting down as " +
motorState.get());
} else {
logger.warn(
() -> "Unexpected motor state for CoreMotor shutdown: " + motorState.get());
@ -277,18 +294,11 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
}
@Override
public void onActivityDefUpdate(ActivityDef activityDef) {
for (Object component : (new Object[]{input, opTracker, action, output})) {
if (component instanceof ActivityDefObserver) {
((ActivityDefObserver) component).onActivityDefUpdate(activityDef);
}
}
this.stride = activityDef.getParams().getOptionalInteger("stride").orElse(1);
public void applyConfig(NBConfiguration cfg) {
NBConfigurable.applyMatching(cfg, new Object[]{input, opTracker, action, output});
this.config = getConfigModel().matchConfig(cfg);
strideRateLimiter = activity.getStrideLimiter();
cycleRateLimiter = activity.getCycleLimiter();
}
@Override
@ -298,8 +308,10 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
Stoppable.stop(input, action);
motorState.enterState(Stopping);
} else {
logger.warn(
() -> "attempted to stop motor " + this.getSlotId() + ": from non Running state:" + currentState);
logger.warn(() -> "attempted to stop motor " +
this.getSlotId() +
": from non Running state:" +
currentState);
}
}
@ -307,4 +319,20 @@ public class CoreMotor<D> extends NBBaseComponent implements ActivityDefObserver
this.output = resultOutput;
}
@Override
public NBConfigModel getConfigModel() {
return ConfigModel.of(CoreMotor.class).add(Param.required("stride", Integer.class))
.asReadOnly();
}
@Override
public void applyReconfig(NBConfiguration recfg) {
applyConfig(recfg);
}
@Override
public NBConfigModel getReconfigModel() {
return getConfigModel();
}
}

4
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotorDispenser.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityimpl.motor;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import io.nosqlbench.engine.api.activityapi.core.*;
import io.nosqlbench.engine.api.activityapi.input.Input;
import io.nosqlbench.engine.api.activityapi.input.InputDispenser;
@ -49,7 +49,7 @@ public class CoreMotorDispenser<D> implements MotorDispenser<D> {
}
@Override
public Motor<D> getMotor(ActivityDef activityDef, int slotId) {
public Motor<D> getMotor(ActivityConfig activityConfig, int slotId) {
SyncAction action = actionDispenser.getAction(slotId);
Input input = inputDispenser.getInput(slotId);
Output output = null;

308
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/Activity.java
View File

@ -97,11 +97,20 @@ import org.apache.logging.log4j.Logger;
///
/// The config parameters for an activity are standard, and custom behaviors afforded to activities
/// work the same across all op types.
public class Activity<R extends java.util.function.LongFunction, S> extends NBStatusComponent implements InvokableResult, SyntheticOpTemplateProvider, ActivityDefObserver, StateCapable, ProgressCapable, Comparable<Activity>, MotorDispenser {
public class Activity<R extends java.util.function.LongFunction, S> extends NBStatusComponent
implements InvokableResult,
SyntheticOpTemplateProvider,
StateCapable,
ProgressCapable,
Comparable<Activity>,
MotorDispenser,
NBConfigurable,
NBReconfigurable
{
private static final Logger logger = LogManager.getLogger("ACTIVITY");
private final OpSequence<OpDispenser<? extends CycleOp<?>>> sequence;
private final ConcurrentHashMap<String, DriverAdapter<CycleOp<?>, Space>> adapters = new ConcurrentHashMap<>();
protected final ActivityDef activityDef;
private final ConcurrentHashMap<String, DriverAdapter<CycleOp<?>, Space>> adapters
= new ConcurrentHashMap<>();
public final ActivityMetrics metrics;
private ActivityMetricProgressMeter progressMeter;
@ -116,28 +125,33 @@ public class Activity<R extends java.util.function.LongFunction, S> extends NBSt
private ErrorMetrics errorMetrics;
private Input input;
private StandardAction<?, ?> action;
private ActivityConfig config;
public Activity(NBComponent parent, ActivityDef activityDef) {
public Activity(NBComponent parent, ActivityConfig config) {
super(parent, NBLabels.forKV("activity", config.getAlias()).and(config.auxLabels()));
// NBConfiguration validConfig = getConfigModel().apply(config.getMap());
this.applyConfig(config);
this.sequence = initSequence();
this.metrics = new ActivityMetrics(this);
}
public static ActivityConfig configFor(String s) {
return configFor(ParameterMap.parseParams(s).orElseThrow());
}
getParams().set(
"alias", Optional.ofNullable(activityDef.getAlias()).or(
() -> getParams().getOptionalString("workload")).or(
() -> getParams().getOptionalString("driver")).orElseThrow(
() -> new RuntimeException(
"Unable to determine name of activity from " + activityDef))
);
private OpSequence<OpDispenser<? extends CycleOp<?>>> initSequence() {
// this.activityDef = activityDef;
// this.metrics = new ActivityMetrics(this);
// OpsDocList workload;
Optional<String> yaml_loc = config.getOptional("yaml", "workload");
// TODO: avoid having to load this duplicitously to parse the template variables in a separate phase
NBConfigModel yamlmodel = yaml_loc.map(path -> {
return OpsLoader.loadPath(
path, new LinkedHashMap<>(activityDef.getParams()), "activities").getConfigModel();
return OpsLoader.loadPath(path, new LinkedHashMap<>(config.getMap()), "activities")
.getConfigModel();
}).orElse(ConfigModel.of(Activity.class).asReadOnly());
@ -418,18 +432,6 @@ public class Activity<R extends java.util.function.LongFunction, S> extends NBSt
// return adapter.getErrorNameMapper();
// }
@Override
public synchronized void onActivityDefUpdate(ActivityDef activityDef) {
for (DriverAdapter<?, ?> adapter : adapters.values()) {
if (adapter instanceof NBReconfigurable configurable) {
NBConfigModel cfgModel = configurable.getReconfigModel();
NBConfiguration cfg = cfgModel.matchConfig(activityDef.getParams());
NBReconfigurable.applyMatching(cfg, List.of(configurable));
}
}
}
@Override
public OpTemplates getSyntheticOpTemplates(OpTemplates opsDocList, Map<String, Object> cfg) {
OpTemplates accumulator = new OpTemplates();
@ -469,7 +471,7 @@ public class Activity<R extends java.util.function.LongFunction, S> extends NBSt
switch (event) {
case ParamChange<?> pc -> {
switch (pc.value()) {
case SetThreads st -> activityDef.setThreads(st.threads);
case SetThreads st -> config.update(ActivityConfig.FIELD_THREADS, st.threads);
case CycleRateSpec crs -> createOrUpdateCycleLimiter(crs);
case StrideRateSpec srs -> createOrUpdateStrideLimiter(srs);
default -> super.onEvent(event);
@ -532,91 +534,94 @@ public class Activity<R extends java.util.function.LongFunction, S> extends NBSt
// }
/**
Modify the provided ActivityDef with defaults for stride and cycles, if they haven't been
Modify the provided activity config with defaults for stride and cycles, if they haven't been
provided, based on the
length of the sequence as determined by the provided ratios. Also, modify the ActivityDef with
length of the sequence as determined by the provided ratios. Also, modify the activity config
with
reasonable
defaults when requested.
@param seq
- The {@link OpSequence} to derive the defaults from
*/
private synchronized void setDefaultsFromOpSequence(OpSequence<?> seq) {
Optional<String> strideOpt = getParams().getOptionalString("stride");
if (strideOpt.isEmpty()) {
String stride = String.valueOf(seq.getSequence().length);
logger.info(() -> "defaulting stride to " + stride + " (the sequence length)");
// getParams().set("stride", stride);
getParams().setSilently("stride", stride);
Map<String, Object> updates = new LinkedHashMap<>(config.getMap());
updates.computeIfAbsent(
"stride", k -> {
String stride = String.valueOf(seq.getSequence().length);
logger.info(() -> "defaulting stride to " + stride + " (the sequence length)");
return stride;
});
updates.computeIfAbsent(
"cycles", k -> {
String cycles = (String) updates.get("stride");
logger.info(() -> "defaulting cycles to " + cycles + " (the stride length)");
return cycles;
});
long cycles = CyclesSpec.parse(updates.get("cycles").toString()).cycle_count();
long stride = Long.parseLong(updates.get("stride").toString());
if (cycles < stride) {
throw new RuntimeException("The specified cycles (" +
cycles +
") are less than the stride (" +
stride +
"). This means there aren't enough cycles to cause a stride to be" +
" executed. If this was intended, then set stride low enough to" +
" allow it.");
}
// CYCLES
Optional<String> cyclesOpt = getParams().getOptionalString("cycles");
if (cyclesOpt.isEmpty()) {
String cycles = getParams().getOptionalString("stride").orElseThrow();
logger.info(() -> "defaulting cycles to " + cycles + " (the stride length)");
this.getActivityDef().setCycles(getParams().getOptionalString("stride").orElseThrow());
} else {
if (0 == activityDef.getCycleCount()) {
throw new RuntimeException(
"You specified cycles, but the range specified means zero cycles: " + getParams().get(
"cycles"));
}
long stride = getParams().getOptionalLong("stride").orElseThrow();
long cycles = this.activityDef.getCycleCount();
if (cycles < stride) {
throw new RuntimeException(
"The specified cycles (" + cycles + ") are less than the stride (" + stride + "). This means there aren't enough cycles to cause a stride to be executed." + " If this was intended, then set stride low enough to allow it.");
}
}
Optional<String> threadSpec = Optional.ofNullable(updates.get("threads"))
.map(String::valueOf);
long cycleCount = this.activityDef.getCycleCount();
long stride = this.activityDef.getParams().getOptionalLong("stride").orElseThrow();
if (0 < stride && 0 != cycleCount % stride) {
logger.warn(
() -> "The stride does not evenly divide cycles. Only full strides will be executed," + "leaving some cycles unused. (stride=" + stride + ", cycles=" + cycleCount + ')');
}
Optional<String> threadSpec = activityDef.getParams().getOptionalString("threads");
if (threadSpec.isPresent()) {
String spec = threadSpec.get();
int processors = Runtime.getRuntime().availableProcessors();
int threads = 0;
if ("auto".equalsIgnoreCase(spec)) {
int threads = processors * 10;
if (threads > activityDef.getCycleCount()) {
threads = (int) activityDef.getCycleCount();
threads = processors * 10;
if (threads > cycles) {
threads = (int) cycles;
logger.info(
"setting threads to {} (auto) [10xCORES, cycle count limited]", threads);
} else {
logger.info("setting threads to {} (auto) [10xCORES]", threads);
}
// activityDef.setThreads(threads);
activityDef.getParams().setSilently("threads", threads);
} else if (spec.toLowerCase().matches("\\d+x")) {
String multiplier = spec.substring(0, spec.length() - 1);
int threads = processors * Integer.parseInt(multiplier);
logger.info(() -> "setting threads to " + threads + " (" + multiplier + "x)");
// activityDef.setThreads(threads);
activityDef.getParams().setSilently("threads", threads);
threads = processors * Integer.parseInt(multiplier);
int finalThreads = threads;
logger.info(() -> "setting threads to " + finalThreads + " (" + multiplier + "x)");
} else if (spec.toLowerCase().matches("\\d+")) {
logger.info(() -> "setting threads to " + spec + " (direct)");
// activityDef.setThreads(Integer.parseInt(spec));
activityDef.getParams().setSilently("threads", Integer.parseInt(spec));
} else {
throw new RuntimeException("Unrecognized format for threads:" + spec);
}
updates.put("threads", threads);
if (threads > cycles) {
int finalThreads1 = threads;
logger.warn(() -> "threads=" +
finalThreads1 +
" and cycles=" +
updates.get("cycles").toString() +
", you should have more cycles than threads.");
}
if (activityDef.getThreads() > activityDef.getCycleCount()) {
logger.warn(
() -> "threads=" + activityDef.getThreads() + " and cycles=" + activityDef.getCycleSummary() + ", you should have more cycles than threads.");
}
} else if (1000 < cycleCount) {
logger.warn(
() -> "For testing at scale, it is highly recommended that you " + "set threads to a value higher than the default of 1." + " hint: you can use threads=auto for reasonable default, or" + " consult the topic on threads with `help threads` for" + " more information.");
} else if (1000 < cycles) {
logger.warn(() -> "For testing at scale, it is highly recommended that you " +
"set threads to a value higher than the default of 1." +
" hint: you can use threads=auto for reasonable default, or" +
" consult the topic on threads with `help threads` for" +
" more information.");
}
if (0 < this.activityDef.getCycleCount() && seq.getOps().isEmpty()) {
if (0 < cycles && seq.getOps().isEmpty()) {
throw new BasicError(
"You have configured a zero-length sequence and non-zero cycles. It is not possible to continue with this activity.");
"You have configured a zero-length sequence and non-zero cycles. It is not" +
" possible to continue with this activity.");
}
}
@ -696,25 +701,23 @@ public class Activity<R extends java.util.function.LongFunction, S> extends NBSt
} else if (workload != null && OpsLoader.isJson(workload)) {
workloadSource = "commandline: (workload/json):" + workload;
opsDocs = OpsLoader.loadString(
workload, OpTemplateFormat.json, activityDef.getParams(), null);
workload, OpTemplateFormat.json, config.getMap(), null);
} else if (workload != null && OpsLoader.isYaml(workload)) {
workloadSource = "commandline: (workload/yaml):" + workload;
opsDocs= OpsLoader.loadString(
workload, OpTemplateFormat.yaml, activityDef.getParams(), null);
opsDocs = OpsLoader.loadString(
workload, OpTemplateFormat.yaml, config.getMap(), null);
} else if (workload != null) {
opsDocs= OpsLoader.loadPath(workload, activityDef.getParams(), "activities");
opsDocs = OpsLoader.loadPath(workload, config.getMap(), "activities");
} else if (stmt != null) {
workloadSource = "commandline: (stmt/inline): '" + stmt + "'";
opsDocs= OpsLoader.loadString(
stmt, OpTemplateFormat.inline, activityDef.getParams(), null);
opsDocs = OpsLoader.loadString(
stmt, OpTemplateFormat.inline, config.getMap(), null);
} else if (op != null && OpsLoader.isJson(op)) {
workloadSource = "commandline: (op/json): '" + op + "'";
opsDocs= OpsLoader.loadString(
op, OpTemplateFormat.json, activityDef.getParams(), null);
opsDocs = OpsLoader.loadString(op, OpTemplateFormat.json, config.getMap(), null);
} else if (op != null) {
workloadSource = "commandline: (op/inline): '" + op + "'";
opsDocs= OpsLoader.loadString(
op, OpTemplateFormat.inline, activityDef.getParams(), null);
opsDocs = OpsLoader.loadString(op, OpTemplateFormat.inline, config.getMap(), null);
}
return new OpTemplates(opsDocs);
@ -747,24 +750,19 @@ public class Activity<R extends java.util.function.LongFunction, S> extends NBSt
return startedAtMillis;
}
public ActivityDef getActivityDef() {
return activityDef;
}
public String toString() {
return (activityDef != null ? activityDef.getAlias() : "unset_alias") + ':' + this.runState + ':' + this.tally;
return config.getAlias() + ':' + this.runState + ':' + this.tally;
}
public synchronized void initOrUpdateRateLimiters(ActivityDef activityDef) {
public synchronized void initOrUpdateRateLimiters() {
// cycleratePerThread = activityDef.getParams().takeBoolOrDefault("cyclerate_per_thread", false);
activityDef.getParams().getOptionalNamedParameter("striderate").map(
StrideRateSpec::new).ifPresent(sr -> this.onEvent(new ParamChange<>(sr)));
activityDef.getParams().getOptionalNamedParameter("cyclerate", "targetrate", "rate").map(
CycleRateSpec::new).ifPresent(sr -> this.onEvent(new ParamChange<>(sr)));
config.getOptional("striderate").map(StrideRateSpec::new)
.ifPresent(sr -> this.onEvent(new ParamChange<>(sr)));
config.getOptional("cyclerate", "targetrate", "rate").map(CycleRateSpec::new)
.ifPresent(sr -> this.onEvent(new ParamChange<>(sr)));
}
public void createOrUpdateStrideLimiter(SimRateSpec spec) {
@ -806,7 +804,7 @@ public class Activity<R extends java.util.function.LongFunction, S> extends NBSt
@Override
public Map<String, String> asResult() {
return Map.of("activity", this.getActivityDef().getAlias());
return Map.of("activity", config.getAlias());
}
/**
@ -816,15 +814,14 @@ public class Activity<R extends java.util.function.LongFunction, S> extends NBSt
@return The number of allowable retries
*/
public int getMaxTries() {
return this.activityDef.getParams().getOptionalInteger("maxtries").orElse(10);
return config.getOptional(Integer.class, "maxtries").orElse(10);
}
public synchronized NBErrorHandler getErrorHandler() {
if (null == this.errorHandler) {
errorHandler = new NBErrorHandler(
() -> activityDef.getParams().getOptionalString("errors").orElse("stop"),
this::getExceptionMetrics
);
() -> config.getOptional("errors").orElse("stop"),
this::getExceptionMetrics);
}
return errorHandler;
}
@ -844,7 +841,7 @@ public class Activity<R extends java.util.function.LongFunction, S> extends NBSt
@Override
public int compareTo(Activity o) {
return this.getActivityDef().getAlias().compareTo(o.getActivityDef().getAlias());
return getAlias().compareTo(o.getAlias());
}
// public void registerAutoCloseable(AutoCloseable closeable) {
@ -860,17 +857,17 @@ public class Activity<R extends java.util.function.LongFunction, S> extends NBSt
public String getAlias() {
return getActivityDef().getAlias();
return config.getAlias();
}
@Override
public Motor getMotor(ActivityDef activityDef, int slot) {
public Motor getMotor(ActivityConfig activityConfig, int slot) {
return new CoreMotor(this, slot, getInput(), getAction(), getOutput());
}
public synchronized Input getInput() {
if (input == null) {
this.input = new AtomicInput(this, this.getActivityDef());
this.input = new AtomicInput(this);
}
return this.input;
}
@ -891,5 +888,90 @@ public class Activity<R extends java.util.function.LongFunction, S> extends NBSt
cycleLimiterSource = ThreadLocalRateLimiters.createOrUpdate(this, cycleLimiterSource, spec);
}
public ActivityConfig getConfig() {
return this.config;
}
public static ActivityConfig configFor(Map<String, ?> params) {
return new ActivityConfig(configModel.apply(params));
}
private static NBConfigModel configModel = ConfigModel.of(Activity.class)
.add(Param.optional("alias")).add(Param.optional(
"labels", String.class,
"Labels which will apply to metrics and annotations for this activity only"))
.add(Param.defaultTo(
"strict", true,
"strict op field mode, which requires that provided op fields are recognized and used"))
.add(Param.optional("op", String.class, "op template in statement form")).add(
Param.optional(
List.of("stmt", "statement"), String.class,
"op template in statement " + "form"))
.add(Param.defaultTo("tags", "", "tag filter to be used to filter operations"))
.add(Param.defaultTo("errors", "stop", "error handler configuration")).add(
Param.defaultTo("threads","1").setRegex("\\d+|\\d+x|auto")
.setDescription("number of concurrent operations, controlled by threadpool"))
.add(Param.optional("stride").setRegex("\\d+"))
.add(Param.optional("striderate", String.class, "rate limit for strides per second")).add(
Param.defaultTo("cycles", "1")
.setRegex("\\d+[KMBGTPE]?|\\d+[KMBGTPE]?\\.\\" + ".\\d+[KMBGTPE]?")
.setDescription("cycle interval to use")).add(Param.defaultTo("recycles", "1")
.setDescription("allow cycles to be re-used this many " + "times")).add(Param.optional(
List.of("cyclerate", "targetrate", "rate"), String.class,
"rate limit for cycles per second"))
.add(Param.optional("seq", String.class, "sequencing algorithm"))
.add(Param.optional("instrument", Boolean.class)).add(
Param.optional(
List.of("workload", "yaml"), String.class, "location of workload yaml file"))
.add(Param.optional("driver", String.class))
.add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|emit|none)"))
.add(Param.optional("maxtries", Integer.class)).add(
Param.defaultTo(
"input", "type=atomicseq", "The type of cycle input to use for this " + "activity"))
.add(Param.optional(List.of("if","inputfilter"),String.class,"an input filter"))
.add(Param.optional("output",String.class))
.asReadOnly();
@Override
public NBConfigModel getConfigModel() {
return configModel;
}
@Override
public void applyConfig(NBConfiguration config) {
Optional<String> directAlias = config.getOptional("alias");
// if (!directAlias.isPresent()) {
// String indirectAlias = config.getOptional(ActivityConfig.FIELD_ALIAS)
// .or(() -> config.getOptional("workload")).or(() -> config.getOptional("driver"))
// .orElse("ACTIVITYNAME");
//
// config.getMap().put("alias", indirectAlias);
// }
//
NBConfigurable.applyMatchingCollection(config, adapters.values());
this.config = new ActivityConfig(config);
}
@Override
public void applyReconfig(NBConfiguration reconf) {
this.config = new ActivityConfig(getReconfigModel().apply(reconf.getMap()));
}
@Override
public NBConfigModel getReconfigModel() {
return ConfigModel.of(Activity.class).add(
Param.optional("threads").setRegex("\\d+|\\d+x|auto")
.setDescription("number of concurrent operations, controlled by threadpool"))
.add(Param.optional("striderate", String.class, "rate limit for strides per second"))
.add(Param.optional(
List.of("cyclerate", "targetrate", "rate"), String.class,
"rate limit for cycles per second")).asReadOnly();
}
public CyclesSpec getCyclesSpec() {
return CyclesSpec.parse(config.get(ActivityConfig.FIELD_CYCLES));
}
}

23
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/ActivityDefAware.java
View File

@ -1,23 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityimpl.uniform;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
public interface ActivityDefAware {
void setActivityDef(ActivityDef activiytDef);
}

23
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/ActivityWiring.java
View File

@ -2,13 +2,13 @@ package io.nosqlbench.engine.api.activityimpl.uniform;
/*
* Copyright (c) nosqlbench
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@ -23,29 +23,26 @@ import io.nosqlbench.engine.api.activityapi.core.MotorDispenser;
import io.nosqlbench.engine.api.activityapi.cyclelog.filters.IntPredicateDispenser;
import io.nosqlbench.engine.api.activityapi.input.InputDispenser;
import io.nosqlbench.engine.api.activityapi.output.OutputDispenser;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ParameterMap;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
public class ActivityWiring {
private final ActivityDef activityDef;
private final ActivityConfig activityDef;
private MotorDispenser<?> motorDispenser;
private InputDispenser inputDispenser;
private ActionDispenser actionDispenser;
private OutputDispenser markerDispenser;
private IntPredicateDispenser resultFilterDispenser;
public ActivityWiring(ActivityDef activityDef) {
public ActivityWiring(ActivityConfig activityDef) {
this.activityDef = activityDef;
}
public static ActivityWiring of(ActivityDef activityDef) {
public static ActivityWiring of(ActivityConfig activityDef) {
return new ActivityWiring(activityDef);
}
public ActivityDef getActivityDef() {
public ActivityConfig getConfig() {
return activityDef;
}
@ -89,8 +86,4 @@ public class ActivityWiring {
this.markerDispenser = outputDispenser;
}
public ParameterMap getParams() {
return activityDef.getParams();
}
}

18
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.engine.api.activityimpl.uniform;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import io.nosqlbench.engine.api.activityapi.core.ActivitiesAware;
import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
import io.nosqlbench.engine.api.activityapi.core.MotorDispenser;
@ -41,9 +41,10 @@ public class StandardActivityType<A extends Activity<?,?>> {
private final Map<String, DriverAdapter> adapters = new HashMap<>();
private final NBComponent parent;
// private final DriverAdapter<?, ?> adapter;
private final ActivityDef activityDef;
private final ActivityConfig activityDef;
public StandardActivityType(final DriverAdapter<?,?> adapter, final ActivityDef activityDef, final NBComponent parent) {
public StandardActivityType(final DriverAdapter<?,?> adapter, final ActivityConfig activityDef,
final NBComponent parent) {
this.parent = parent;
// this.adapter = adapter;
this.activityDef = activityDef;
@ -52,10 +53,9 @@ public class StandardActivityType<A extends Activity<?,?>> {
// .deprecate("yaml", "workload")
// );
adapters.put(adapter.getAdapterName(),adapter);
if (adapter instanceof ActivityDefAware) ((ActivityDefAware) adapter).setActivityDef(activityDef);
}
public StandardActivityType(final ActivityDef activityDef, final NBComponent parent) {
public StandardActivityType(final ActivityConfig activityDef, final NBComponent parent) {
this.parent = parent;
this.activityDef = activityDef;
@ -69,11 +69,9 @@ public class StandardActivityType<A extends Activity<?,?>> {
* @return a distinct StandardActivity instance for each call
*/
@SuppressWarnings("unchecked")
public A getActivity(final ActivityDef activityDef,
public A getActivity(final ActivityConfig activityDef,
final NBComponent parent,
final ActivityWiring wiring) {
if (activityDef.getParams().getOptionalString("async").isPresent())
throw new RuntimeException("This driver does not support async mode yet.");
return (A) new Activity(parent, activityDef);
}
@ -95,7 +93,7 @@ public class StandardActivityType<A extends Activity<?,?>> {
* @return a distinct activity instance for each call
*/
public Activity getAssembledActivity(
final NBComponent parent, final ActivityDef activityDef,
final NBComponent parent, final ActivityConfig activityDef,
final Map<String, Activity> activities
) {
// final A activity = this.getActivity(activityDef, parent);
@ -122,7 +120,7 @@ public class StandardActivityType<A extends Activity<?,?>> {
if (motorDispenser instanceof ActivitiesAware) ((ActivitiesAware) motorDispenser).setActivitiesMap(activities);
wiring.setMotorDispenserDelegate(motorDispenser);
return this.getActivity(activityDef,parent,wiring);
return activity;
}
/**

10
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/actions/StandardAction.java
View File

@ -22,11 +22,9 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.*;
import io.nosqlbench.adapters.api.evalctx.CycleFunction;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricHistogram;
import io.nosqlbench.nb.api.errors.ResultVerificationError;
import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.core.SyncAction;
import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorDetail;
import io.nosqlbench.engine.api.activityapi.errorhandling.modular.NBErrorHandler;
@ -47,7 +45,7 @@ import java.util.concurrent.TimeUnit;
The type of activity
@param <R>
The type of operation */
public class StandardAction<A extends Activity<R, ?>, R extends java.util.function.LongFunction> extends NBBaseComponent implements SyncAction, ActivityDefObserver {
public class StandardAction<A extends Activity<R, ?>, R extends java.util.function.LongFunction> extends NBBaseComponent implements SyncAction {
private final static Logger logger = LogManager.getLogger("ACTION");
private final NBErrorHandler errorHandler;
private final OpSequence<OpDispenser<? extends CycleOp<?>>> opsequence;
@ -142,7 +140,7 @@ public class StandardAction<A extends Activity<R, ?>, R extends java.util.functi
}
}
}
this.triesHistogram.update(tries);
activity.metrics.triesHistogram.update(tries);
if (op instanceof OpGenerator) {
logger.trace(() -> "GEN OP for cycle(" + cycle + ")");
@ -155,8 +153,4 @@ public class StandardAction<A extends Activity<R, ?>, R extends java.util.functi
return code;
}
@Override
public void onActivityDefUpdate(ActivityDef activityDef) {
}
}

14
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionHistoMetrics.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.metrics;
import com.codahale.metrics.Histogram;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
@ -34,14 +34,15 @@ public class ExceptionHistoMetrics {
private final ConcurrentHashMap<String, Histogram> histos = new ConcurrentHashMap<>();
private final Histogram allerrors;
private final NBComponent parent;
private final ActivityDef activityDef;
private final ActivityConfig activityDef;
public ExceptionHistoMetrics(final NBComponent parent, final ActivityDef activityDef) {
public ExceptionHistoMetrics(final NBComponent parent, final ActivityConfig config) {
this.parent = parent;
this.activityDef = activityDef;
this.activityDef = config;
int hdrdigits = parent.getComponentProp("hdr_digits").map(Integer::parseInt).orElse(4);
this.allerrors = parent.create().histogram(
"errorhistos_ALL",
activityDef.getParams().getOptionalInteger("hdr_digits").orElse(4),
hdrdigits,
MetricCategory.Errors,
"A histogram for all exceptions"
);
@ -50,11 +51,12 @@ public class ExceptionHistoMetrics {
public void update(final String name, final long magnitude) {
Histogram h = this.histos.get(name);
if (null == h) synchronized (this.histos) {
int hdrdigits = parent.getComponentProp("hdr_digits").map(Integer::parseInt).orElse(4);
h = this.histos.computeIfAbsent(
name,
errName -> parent.create().histogram(
"errorhistos_"+errName,
this.activityDef.getParams().getOptionalInteger("hdr_digits").orElse(4),
hdrdigits,
MetricCategory.Errors,
"error histogram for exception '" + errName + "'"
)

8
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionTimerMetrics.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.metrics;
import com.codahale.metrics.Timer;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
@ -32,11 +32,11 @@ import java.util.concurrent.TimeUnit;
public class ExceptionTimerMetrics {
private final ConcurrentHashMap<String, Timer> timers = new ConcurrentHashMap<>();
private final Timer allerrors;
private final ActivityDef activityDef;
private final ActivityConfig activityDef;
private final NBComponent parentLabels;
public ExceptionTimerMetrics(final NBComponent parent, final ActivityDef activityDef) {
this.activityDef = activityDef;
public ExceptionTimerMetrics(final NBComponent parent, final ActivityConfig config) {
this.activityDef = config;
this.parentLabels = parent;
this.allerrors=parent.create().timer(

2
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/util/ConfigTuples.java
View File

@ -35,7 +35,7 @@ public class ConfigTuples implements Iterable<ConfigTuples.Section> {
}
public ConfigTuples(Activity activity, String param) {
this(activity.getParams().getOptionalString(param).orElse(""));
this(activity.getConfig().getOptional(param).orElse(""));
}
private List<Section> parseParams(String configdata) {

8
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/util/SimpleConfig.java
View File

@ -18,7 +18,6 @@ package io.nosqlbench.engine.api.util;
import io.nosqlbench.engine.api.activityimpl.uniform.ActivityWiring;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ParameterMap;
import java.util.Arrays;
@ -36,13 +35,10 @@ public class SimpleConfig {
}
public SimpleConfig(Activity activity, String params) {
this(activity.getActivityDef(),params);
this(activity.getConfig().get(params));
}
public SimpleConfig(ActivityWiring wiring, String param) {
this(wiring.getParams().getOptionalString(param).orElse(""));
}
public SimpleConfig(ActivityDef activityDef, String param) {
this(activityDef.getParams().getOptionalString(param).orElse(""));
this(wiring.getConfig().getOptional(param).orElse(""));
}
public SimpleConfig(ParameterMap parameters, String param) {
this(parameters.getOptionalString(param).orElse(""));

2
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExceptionHandler.java
View File

@ -27,7 +27,7 @@ public class ActivityExceptionHandler implements Thread.UncaughtExceptionHandler
public ActivityExceptionHandler(ActivityExecutor executor) {
this.executor = executor;
logger.debug(() -> "Activity executor exception handler starting up for executor '" + executor.getActivityDef().getAlias() + "'");
logger.debug(() -> "Activity executor exception handler starting up for executor '" + executor.getActivity().getAlias() + "'");
}

145
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java
View File

@ -16,9 +16,12 @@
package io.nosqlbench.engine.core.lifecycle.activity;
import com.codahale.metrics.Gauge;
import io.nosqlbench.engine.api.activityimpl.motor.CoreMotorDispenser;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.engine.core.lifecycle.IndexedThreadFactory;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.config.standard.NBReconfigurable;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
@ -28,15 +31,12 @@ import io.nosqlbench.engine.api.activityapi.core.*;
import io.nosqlbench.engine.api.activityimpl.MotorState;
import io.nosqlbench.nb.api.annotations.Annotation;
import io.nosqlbench.nb.api.annotations.Layer;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ParameterMap;
import io.nosqlbench.engine.api.activityapi.core.progress.ProgressCapable;
import io.nosqlbench.engine.api.activityapi.core.progress.ProgressMeterDisplay;
import io.nosqlbench.engine.api.activityimpl.motor.RunStateImage;
import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
import io.nosqlbench.engine.core.annotation.Annotators;
import io.nosqlbench.engine.core.lifecycle.ExecutionResult;
//import io.nosqlbench.virtdata.userlibs.apps.valuechecker.IndexedThreadFactory;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -45,6 +45,8 @@ import java.util.List;
import java.util.concurrent.*;
import java.util.stream.Collectors;
/// TODO: Make this use nbreconfigurable events for live updates
/**
* <p>An ActivityExecutor is an execution harness for a single activity instance.
* It is responsible for managing threads and activity settings which may be changed while the activity is running.</p>
@ -61,7 +63,8 @@ import java.util.stream.Collectors;
* This allows the state tracking to work consistently for all observers.</p>
*/
public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener, ProgressCapable, Callable<ExecutionResult> {
public class ActivityExecutor implements NBReconfigurable, NBLabeledElement, ProgressCapable,
Callable<ExecutionResult> {
// TODO Encapsulate valid state transitions to be only modifiable within the appropriate type view.
@ -70,7 +73,7 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
private final LinkedList<Motor<?>> motors = new LinkedList<>();
private final Activity activity;
private final ActivityDef activityDef;
private final ActivityConfig config;
private final RunStateTally tally;
private final MotorDispenser motorSource;
private ExecutorService executorService;
@ -84,9 +87,9 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
public ActivityExecutor(Activity activity) {
this.activity = activity;
this.activityDef = activity.getActivityDef();
this.config = activity.getConfig();
this.motorSource = activity;
activity.getActivityDef().getParams().addListener(this);
// activity.getConfig().addListener(this);
this.tally = activity.getRunStateTally();
}
@ -99,7 +102,7 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
* Simply stop the motors
*/
public void stopActivity() {
logger.info(() -> "stopping activity in progress: " + this.getActivityDef().getAlias());
logger.info(() -> "stopping activity in progress: " + activity.getAlias());
activity.setRunState(RunState.Stopping);
motors.forEach(Motor::requestStop);
@ -109,13 +112,13 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
tally.awaitNoneOther(RunState.Stopped, RunState.Finished, RunState.Errored);
activity.setRunState(RunState.Stopped);
logger.info(() -> "stopped: " + this.getActivityDef().getAlias() + " with " + motors.size() + " slots");
logger.info(() -> "stopped: " + activity.getAlias() + " with " + motors.size() + " slots");
Annotators.recordAnnotation(Annotation.newBuilder()
.element(this)
.interval(this.startedAt, this.stoppedAt)
.layer(Layer.Activity)
.addDetail("params", getActivityDef().toString())
.addDetail("params", String.valueOf(activity.getConfig()))
.build()
);
}
@ -124,7 +127,7 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
* Force stop the motors without trying to wait for the activity to reach stopped/finished state
*/
public void forceStopActivity() {
logger.info(() -> "force stopping activity in progress: " + this.getActivityDef().getAlias());
logger.info(() -> "force stopping activity in progress: " + activity.getAlias());
activity.setRunState(RunState.Stopping);
motors.forEach(Motor::requestStop);
@ -133,21 +136,20 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
tally.awaitNoneOther(RunState.Stopped, RunState.Finished);
activity.setRunState(RunState.Stopped);
logger.info(() -> "stopped: " + this.getActivityDef().getAlias() + " with " + motors.size() + " slots");
logger.info(() -> "stopped: " + activity.getAlias() + " with " + motors.size() + " slots");
Annotators.recordAnnotation(Annotation.newBuilder()
.element(this)
.interval(this.startedAt, this.stoppedAt)
.layer(Layer.Activity)
.addDetail("params", getActivityDef().toString())
.addDetail("params", String.valueOf(activity.getConfigModel()))
.build()
);
}
public Exception forceStopActivity(int initialMillisToWait) {
activitylogger.debug("FORCE STOP/before alias=(" + activity.getActivityDef().getAlias() +
")");
activitylogger.debug("FORCE STOP/before alias=(" + activity.getAlias() + ")");
activity.setRunState(RunState.Stopped);
executorService.shutdownNow();
@ -205,36 +207,10 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
}
}
/**
* Listens for changes to parameter maps, maps them to the activity instance, and notifies all eligible listeners of
* changes.
*/
@Override
public void handleParameterMapUpdate(ParameterMap parameterMap) {
activity.onActivityDefUpdate(activityDef);
// An activity must be initialized before the motors and other components are
// considered ready to handle parameter map changes. This is signaled in an activity
// by the RunState.
if (activity.getRunState() != RunState.Uninitialized) {
if (activity.getRunState() == RunState.Running) {
adjustMotorCountToThreadParam(activity.getActivityDef());
}
motors.stream()
.filter(m -> (m instanceof ActivityDefObserver))
// .filter(m -> m.getSlotStateTracker().getSlotState() != RunState.Uninitialized)
// .filter(m -> m.getSlotStateTracker().getSlotState() != RunState.Starting)
.forEach(m -> ((ActivityDefObserver) m).onActivityDefUpdate(activityDef));
}
}
public ActivityDef getActivityDef() {
return activityDef;
}
public String toString() {
return getClass().getSimpleName() + "~" + activityDef.getAlias();
return getClass().getSimpleName() + "~" + config.getAlias();
}
private String getSlotStatus() {
@ -243,17 +219,12 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
.collect(Collectors.joining(",", "[", "]"));
}
/**
* Stop extra motors, start missing motors
*
* @param activityDef
* the activityDef for this activity instance
*/
private void adjustMotorCountToThreadParam(ActivityDef activityDef) { // TODO: Ensure that threads area allowed to complete their current op gracefully
private void adjustMotorCountToThreadParam(int threadCount) { // TODO: Ensure that threads area
// allowed to complete their current op gracefully
logger.trace(() -> ">-pre-adjust->" + getSlotStatus());
reduceActiveMotorCountDownToThreadParam(activityDef);
increaseActiveMotorCountUpToThreadParam(activityDef);
reduceActiveMotorCountDownToThreadParam();
increaseActiveMotorCountUpToThreadParam();
alignMotorStateToIntendedActivityState();
awaitAlignmentOfMotorStateToActivityState();
@ -261,11 +232,11 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
}
private void increaseActiveMotorCountUpToThreadParam(ActivityDef activityDef) {
private void increaseActiveMotorCountUpToThreadParam() {
// Create motor slots
try {
while (motors.size() < activityDef.getThreads()) {
Motor motor = motorSource.getMotor(activityDef, motors.size());
while (motors.size() < config.getThreads()) {
Motor motor = motorSource.getMotor(config, motors.size());
logger.trace(() -> "Starting cycle motor thread:" + motor);
motors.add(motor);
}
@ -276,24 +247,12 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
}
}
private void reduceActiveMotorCountDownToThreadParam(ActivityDef activityDef) {
private void reduceActiveMotorCountDownToThreadParam() {
// Stop and remove extra motor slots
if (activityDef.getThreads()==0) {
if (config.getThreads()==0) {
logger.warn("setting threads to zero is not advised. At least one thread has to be active to keep the activity alive.");
}
// LinkedList<Motor<?>> toremove = new LinkedList<>();
// while (activityDef.getThreads()>motors.size()) {
// Motor<?> motor = motors.removeLast();
// toremove.addFirst(motor);
// }
// for (Motor<?> motor : toremove) {
// motor.requestStop();
// }
// for (Motor<?> motor : toremove) {
// motor.removeState();
// }
//
while (motors.size() > activityDef.getThreads()) {
while (motors.size() > config.getThreads()) {
Motor motor = motors.get(motors.size() - 1);
logger.trace(() -> "Stopping cycle motor thread:" + motor);
motor.requestStop();
@ -408,7 +367,7 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
.now()
.layer(Layer.Activity)
.addDetail("event", "start-activity")
.addDetail("params", activityDef.toString())
.addDetail("params", String.valueOf(activity.getConfig()))
.build());
try {
@ -419,7 +378,7 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
registerMetrics();
startRunningActivityThreads();
awaitMotorsAtLeastRunning();
logger.debug("STARTED " + activityDef.getAlias());
logger.debug("STARTED " + config.getAlias());
awaitActivityCompletion();
} catch (Exception e) {
this.exception = e;
@ -544,12 +503,12 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
*/
private void startRunningActivityThreads() {
logger.info(() -> "starting activity " + activity.getAlias() + " for cycles " + activity.getActivityDef().getCycleSummary());
logger.info(() -> "starting activity " + activity.getAlias() + " for cycles " + activity.getCyclesSpec().summary());
Annotators.recordAnnotation(Annotation.newBuilder()
.element(this)
.now()
.layer(Layer.Activity)
.addDetail("params", getActivityDef().toString())
.addDetail("params", activity.getConfig().summary())
.build()
);
@ -558,13 +517,12 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
try {
activity.setRunState(RunState.Starting);
this.startedAt = System.currentTimeMillis();
activity.onActivityDefUpdate(activityDef);
} catch (Exception e) {
this.exception = new RuntimeException("Error initializing activity '" + activity.getAlias() + "':\n" + e.getMessage(), e);
activitylogger.error(() -> "error initializing activity '" + activity.getAlias() + "': " + exception);
throw new RuntimeException(exception);
}
adjustMotorCountToThreadParam(activity.getActivityDef());
adjustMotorCountToThreadParam(activity.getConfig().getThreads());
tally.awaitAny(RunState.Running, RunState.Finished, RunState.Stopped);
activity.setRunState(RunState.Running);
activitylogger.debug("START/after alias=(" + activity.getAlias() + ")");
@ -592,7 +550,7 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
.interval(startedAt, stoppedAt)
.layer(Layer.Activity)
.addDetail("event", "stop-activity")
.addDetail("params", activityDef.toString())
.addDetail("params", config.toString())
.build());
}
@ -605,6 +563,39 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
return image.isNoneOther(RunState.Running);
}
@Override
public void applyReconfig(NBConfiguration recfg) {
// An activity must be initialized before the motors and other components are
// considered ready to handle parameter map changes. This is signaled in an activity
// by the RunState.
if (activity.getRunState() != RunState.Uninitialized) {
if (activity.getRunState() == RunState.Running) {
adjustMotorCountToThreadParam(activity.getConfig().getThreads());
}
motors.stream()
.filter(m -> (m instanceof NBReconfigurable reconf))
.forEach(r -> ((NBReconfigurable) r).applyReconfig(recfg));
}
}
@Override
public NBConfigModel getReconfigModel() {
return null;
}
@Override
public void applyConfig(NBConfiguration cfg) {
}
@Override
public NBConfigModel getConfigModel() {
return null;
}
private class ThreadsGauge implements Gauge<Double> {
public ThreadsGauge(ActivityExecutor activityExecutor) {
ActivityExecutor ae = activityExecutor;

21
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityLoader.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.core.lifecycle.activity;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.engine.api.activityimpl.uniform.StandardActivityType;
import org.apache.logging.log4j.LogManager;
@ -27,10 +27,9 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* Consolidates the activity type and activity instantiation logic into one place
* per scope. Within the lifetime of this ActivityLoader, all activities may
* see each other by name.
*/
Consolidates the activity type and activity instantiation logic into one place
per scope. Within the lifetime of this ActivityLoader, all activities may
see each other by name. */
public class ActivityLoader {
private static final Logger logger = LogManager.getLogger("ACTIVITIES");
private final Map<String, Activity> activityMap = new ConcurrentHashMap<>();
@ -38,10 +37,14 @@ public class ActivityLoader {
public ActivityLoader() {
}
public synchronized Activity loadActivity(ActivityDef activityDef, final NBComponent parent) {
activityDef= activityDef.deprecate("yaml","workload").deprecate("type","driver");
final Activity activity =
new StandardActivityType<>(activityDef, parent).getAssembledActivity(parent, activityDef, this.activityMap);this.activityMap.put(activity.getAlias(),activity);
public synchronized Activity loadActivity(
ActivityConfig activityDef,
final NBComponent parent
)
{
final Activity activity = new StandardActivityType<>(
activityDef, parent).getAssembledActivity(parent, activityDef, this.activityMap);
this.activityMap.put(activity.getAlias(), activity);
ActivityLoader.logger.debug("Resolved activity for alias '{}'", activityDef.getAlias());
return activity;
}

36
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityTypeLoader.java
View File

@ -19,9 +19,9 @@ package io.nosqlbench.engine.core.lifecycle.activity;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.nb.api.spi.SimpleServiceLoader;
import io.nosqlbench.nb.api.system.NBEnvironment;
@ -118,29 +118,33 @@ public class ActivityTypeLoader {
return urlsToAdd;
}
public Optional<StandardActivityType> load(final ActivityDef activityDef, final NBComponent parent) {
public Optional<StandardActivityType> load(
final ActivityConfig activityDef,
final NBComponent parent
)
{
String driverName = activityDef.getParams()
.getOptionalString("driver", "type")
String driverName = activityDef.getDriver()
.orElseThrow(() -> new BasicError("The parameter 'driver=' is required."));
activityDef.getParams()
.getOptionalString("jar")
.map(jar -> {
final Set<URL> urls = NBIO.local().search(jar)
.list()
.stream().map(Content::getURL)
.collect(Collectors.toSet());
return urls;
})
.ifPresent(this::extendClassLoader);
activityDef.getOptional("jar").map(jar -> {
final Set<URL> urls = NBIO.local().search(jar).list().stream().map(Content::getURL)
.collect(Collectors.toSet());
return urls;
}).ifPresent(this::extendClassLoader);
return getDriverAdapter(driverName,activityDef,parent);
}
private Optional<StandardActivityType> getDriverAdapter(final String activityTypeName, final ActivityDef activityDef, final NBComponent parent) {
final Optional<DriverAdapter> oda = this.DRIVERADAPTER_SPI_FINDER.getOptionally(activityTypeName);
private Optional<StandardActivityType> getDriverAdapter(
final String activityTypeName,
final ActivityConfig activityDef,
final NBComponent parent
)
{
final Optional<DriverAdapter> oda = this.DRIVERADAPTER_SPI_FINDER.getOptionally(
activityTypeName);
if (oda.isPresent()) {
final DriverAdapter<?, ?> driverAdapter = oda.get();

2
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/CMD_stop.java
View File

@ -44,7 +44,7 @@ public class CMD_stop extends NBBaseCommand {
= params.maybeGet("activity").orElseThrow(
() -> new RuntimeException("The stop command requires an 'activity' parameter")
);
Optional<Activity> activity = controller.getActivity(activityName);
Optional<Activity> activity = controller.getOptionalActivity(activityName);
if (activity.isEmpty()) {
BasicError error = new BasicError("Activity '" + activityName + "' was not found for stop command.");
logger.warn(error);

159
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/container/ContainerActivitiesController.java
View File

@ -16,7 +16,8 @@
package io.nosqlbench.engine.core.lifecycle.scenario.container;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import io.nosqlbench.nb.api.engine.activityimpl.ParameterMap;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
@ -46,9 +47,7 @@ public class ContainerActivitiesController extends NBBaseComponent {
private static final Logger scenariologger = LogManager.getLogger("SCENARIO");
private final ActivityLoader activityLoader;
private final Map<String, ActivityRuntimeInfo> activityInfoMap = new ConcurrentHashMap<>();
private final ExecutorService executorService;
public ContainerActivitiesController(NBComponent parent) {
@ -63,17 +62,17 @@ public class ContainerActivitiesController extends NBBaseComponent {
* Start an activity, given the activity definition for it. The activity will be known in the scenario
* by the alias parameter.
*
* @param activityDef string in alias=value1;driver=value2;... format
* @param ActivityConfig string in alias=value1;driver=value2;... format
*/
public Activity start(ActivityDef activityDef) {
ActivityRuntimeInfo ari = doStartActivity(activityDef);
public Activity start(ActivityConfig ActivityConfig) {
ActivityRuntimeInfo ari = doStartActivity(ActivityConfig);
return ari.getActivity();
}
private ActivityRuntimeInfo doStartActivity(ActivityDef activityDef) {
if (!this.activityInfoMap.containsKey(activityDef.getAlias())) {
Activity activity = this.activityLoader.loadActivity(activityDef, this);
private ActivityRuntimeInfo doStartActivity(ActivityConfig ActivityConfig) {
if (!this.activityInfoMap.containsKey(ActivityConfig.getAlias())) {
Activity activity = this.activityLoader.loadActivity(ActivityConfig, this);
activity.initActivity();
ActivityExecutor executor = new ActivityExecutor(activity);
Future<ExecutionResult> startedActivity = executorService.submit(executor);
@ -82,17 +81,17 @@ public class ContainerActivitiesController extends NBBaseComponent {
this.activityInfoMap.put(activity.getAlias(), activityRuntimeInfo);
}
return this.activityInfoMap.get(activityDef.getAlias());
return this.activityInfoMap.get(ActivityConfig.getAlias());
}
/**
* Start an activity, given a map which holds the activity definition for it. The activity will be known in
* the scenario by the alias parameter.
*
* @param activityDefMap A map containing the activity definition
* @param activityParams A map containing the activity definition
*/
public Activity start(Map<String, String> activityDefMap) {
ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap));
public Activity start(Map<String, String> activityParams) {
ActivityConfig ad = Activity.configFor(activityParams);
Activity started = start(ad);
awaitAllThreadsOnline(started,30000L);
return started;
@ -105,11 +104,11 @@ public class ContainerActivitiesController extends NBBaseComponent {
* @param alias the alias of an activity that is already known to the scenario
*/
public Activity start(String alias) {
return start(ActivityDef.parseActivityDef(alias));
return start(Activity.configFor(Map.of("alias",alias)));
}
public synchronized void run(int timeout, Map<String, String> activityDefMap) {
ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap));
public synchronized void run(int timeout, Map<String, String> activityParams) {
ActivityConfig ad = Activity.configFor(activityParams);
run(ad, timeout);
}
@ -117,30 +116,32 @@ public class ContainerActivitiesController extends NBBaseComponent {
* Synchronously run the defined activity with a timeout in seconds.
*
* @param timeoutMs seconds to await completion of the activity.
* @param activityDef A definition for an activity to run
* @param ActivityConfig A definition for an activity to run
*/
public synchronized void run(ActivityDef activityDef, long timeoutMs) {
public synchronized void run(ActivityConfig ActivityConfig, long timeoutMs) {
doStartActivity(activityDef);
awaitActivity(activityDef, timeoutMs);
doStartActivity(ActivityConfig);
awaitActivity(ActivityConfig, timeoutMs);
}
public synchronized void run(int timeout, String activityDefString) {
ActivityDef activityDef = ActivityDef.parseActivityDef(activityDefString);
run(activityDef, timeout);
public synchronized void run(int timeout, String ActivityConfigString) {
Map<String, String> stringStringMap = ParameterMap.parseParams(ActivityConfigString)
.map(p -> p.getStringStringMap()).orElseThrow();
ActivityConfig activityConfig = Activity.configFor(stringStringMap);
run(activityConfig, timeout);
}
public synchronized void run(Map<String, String> activityDefMap) {
run(Integer.MAX_VALUE, activityDefMap);
public synchronized void run(Map<String, String> ActivityConfigMap) {
run(Integer.MAX_VALUE, ActivityConfigMap);
}
public synchronized void run(String activityDefString) {
run(Integer.MAX_VALUE, activityDefString);
public synchronized void run(String ActivityConfigString) {
run(Integer.MAX_VALUE, ActivityConfigString);
}
public synchronized void run(ActivityDef activityDef) {
run(activityDef, Long.MAX_VALUE);
public synchronized void run(ActivityConfig ActivityConfig) {
run(ActivityConfig, Long.MAX_VALUE);
}
@ -148,13 +149,13 @@ public class ContainerActivitiesController extends NBBaseComponent {
return isRunningActivity(aliasToDef(alias));
}
public boolean isRunningActivity(ActivityDef activityDef) {
ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(activityDef.getAlias());
public boolean isRunningActivity(ActivityConfig ActivityConfig) {
ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(ActivityConfig.getAlias());
return (null != runtimeInfo) && runtimeInfo.isRunning();
}
public boolean isRunningActivity(Map<String, String> activityDefMap) {
ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap));
public boolean isRunningActivity(Map<String, String> activityParams) {
ActivityConfig ad = Activity.configFor(activityParams);
return isRunningActivity(ad);
}
@ -163,36 +164,36 @@ public class ContainerActivitiesController extends NBBaseComponent {
* alias parameter. This method retains the activity def signature to provide convenience for scripting.</p>
* <p>For example, sc.stop("alias=foo")</p>
*
* @param activityDef An activity def, including at least the alias parameter.
* @param ActivityConfig An activity def, including at least the alias parameter.
*/
public synchronized void stop(ActivityDef activityDef) {
public synchronized void stop(ActivityConfig ActivityConfig) {
ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(activityDef.getAlias());
ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(ActivityConfig.getAlias());
if (null == runtimeInfo) {
throw new RuntimeException("could not stop missing activity:" + activityDef);
throw new RuntimeException("could not stop missing activity:" + ActivityConfig);
}
scenariologger.debug("STOP {}", activityDef.getAlias());
scenariologger.debug("STOP {}", ActivityConfig.getAlias());
runtimeInfo.stopActivity();
}
public boolean awaitAllThreadsOnline(ActivityDef activityDef, long timeoutMs) {
ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(activityDef.getAlias());
public boolean awaitAllThreadsOnline(ActivityConfig ActivityConfig, long timeoutMs) {
ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(ActivityConfig.getAlias());
if (null == runtimeInfo) {
throw new RuntimeException("could not stop missing activity:" + activityDef);
throw new RuntimeException("could not stop missing activity:" + ActivityConfig);
}
scenariologger.debug("STOP {}", activityDef.getAlias());
scenariologger.debug("STOP {}", ActivityConfig.getAlias());
return runtimeInfo.awaitAllThreadsOnline(timeoutMs);
}
public synchronized void stop(Activity activity) {
stop(activity.getActivityDef());
stop(activity.getConfig());
}
public boolean awaitAllThreadsOnline(Activity activity, long timeoutMs) {
return awaitAllThreadsOnline(activity.getActivityDef(), timeoutMs);
return awaitAllThreadsOnline(activity.getConfig(), timeoutMs);
}
@ -201,10 +202,10 @@ public class ContainerActivitiesController extends NBBaseComponent {
* <p>Stop an activity, given an activity def map. The only part of the map that is important is the
* alias parameter. This method retains the map signature to provide convenience for scripting.</p>
*
* @param activityDefMap A map, containing at least the alias parameter
* @param paramsMap A map, containing at least the alias parameter
*/
public synchronized void stop(Map<String, String> activityDefMap) {
ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap));
public synchronized void stop(Map<String, String> paramsMap) {
ActivityConfig ad = Activity.configFor(paramsMap);
stop(ad);
}
@ -225,7 +226,7 @@ public class ContainerActivitiesController extends NBBaseComponent {
.filter(s -> !s.isEmpty())
.flatMap(aspec -> getMatchingAliases(aspec).stream()).collect(Collectors.toList());
for (String alias : matched) {
ActivityDef adef = aliasToDef(alias);
ActivityConfig adef = aliasToDef(alias);
scenariologger.debug("STOP {}", adef.getAlias());
stop(adef);
}
@ -236,16 +237,16 @@ public class ContainerActivitiesController extends NBBaseComponent {
* alias parameter. This method retains the activity def signature to provide convenience for scripting.</p>
* <p>For example, sc.forceStop("alias=foo")</p>
*
* @param activityDef An activity def, including at least the alias parameter.
* @param ActivityConfig An activity def, including at least the alias parameter.
*/
public synchronized void forceStop(ActivityDef activityDef) {
public synchronized void forceStop(ActivityConfig ActivityConfig) {
ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(activityDef.getAlias());
ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(ActivityConfig.getAlias());
if (null == runtimeInfo) {
throw new RuntimeException("could not force stop missing activity:" + activityDef);
throw new RuntimeException("could not force stop missing activity:" + ActivityConfig);
}
scenariologger.debug("FORCE STOP {}", activityDef.getAlias());
scenariologger.debug("FORCE STOP {}", ActivityConfig.getAlias());
runtimeInfo.forceStopActivity();
}
@ -254,10 +255,10 @@ public class ContainerActivitiesController extends NBBaseComponent {
* <p>Stop an activity, given an activity def map. The only part of the map that is important is the
* alias parameter. This method retains the map signature to provide convenience for scripting.</p>
*
* @param activityDefMap A map, containing at least the alias parameter
* @param activityParams A map, containing at least the alias parameter
*/
public synchronized void forceStop(Map<String, String> activityDefMap) {
ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap));
public synchronized void forceStop(Map<String, String> activityParams) {
ActivityConfig ad = Activity.configFor(activityParams);
forceStop(ad);
}
@ -278,7 +279,7 @@ public class ContainerActivitiesController extends NBBaseComponent {
.filter(s -> !s.isEmpty())
.flatMap(aspec -> getMatchingAliases(aspec).stream()).collect(Collectors.toList());
for (String alias : matched) {
ActivityDef adef = aliasToDef(alias);
ActivityConfig adef = aliasToDef(alias);
scenariologger.debug("STOP {}", adef.getAlias());
forceStop(adef);
}
@ -376,19 +377,17 @@ public class ContainerActivitiesController extends NBBaseComponent {
return completed;
}
private ActivityDef aliasToDef(String alias) {
if (alias.contains("=")) {
return ActivityDef.parseActivityDef(alias);
}
return ActivityDef.parseActivityDef("alias=" + alias + ';');
private ActivityConfig aliasToDef(String alias) {
String cfg = alias.contains("=") ? alias : "alias=" + alias;
return Activity.configFor(ParameterMap.parseOrException(cfg).getStringStringMap());
}
public void await(Map<String, String> activityDefMap) {
this.awaitActivity(activityDefMap);
public void await(Map<String, String> ActivityConfigMap) {
this.awaitActivity(ActivityConfigMap);
}
public boolean awaitActivity(Map<String, String> activityDefMap) {
ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap));
public boolean awaitActivity(Map<String, String> activityParams) {
ActivityConfig ad = Activity.configFor(activityParams);
return awaitActivity(ad, Long.MAX_VALUE);
}
@ -397,21 +396,21 @@ public class ContainerActivitiesController extends NBBaseComponent {
}
public boolean awaitActivity(String alias, long timeoutMs) {
ActivityDef toAwait = aliasToDef(alias);
ActivityConfig toAwait = aliasToDef(alias);
return awaitActivity(toAwait, Long.MAX_VALUE);
}
public void await(ActivityDef activityDef, long timeoutMs) {
this.awaitActivity(activityDef, timeoutMs);
public void await(ActivityConfig ActivityConfig, long timeoutMs) {
this.awaitActivity(ActivityConfig, timeoutMs);
}
public boolean awaitActivity(ActivityDef activityDef, long timeoutMs) {
ActivityRuntimeInfo ari = this.activityInfoMap.get(activityDef.getAlias());
public boolean awaitActivity(ActivityConfig ActivityConfig, long timeoutMs) {
ActivityRuntimeInfo ari = this.activityInfoMap.get(ActivityConfig.getAlias());
if (null == ari) {
throw new RuntimeException("Could not await missing activity: " + activityDef.getAlias());
throw new RuntimeException("Could not await missing activity: " + ActivityConfig.getAlias());
}
scenariologger.debug("AWAIT/before alias={}", activityDef.getAlias());
scenariologger.debug("AWAIT/before alias={}", ActivityConfig.getAlias());
ExecutionResult result = null;
Future<ExecutionResult> future=null;
try {
@ -434,8 +433,8 @@ public class ContainerActivitiesController extends NBBaseComponent {
return Collections.unmodifiableMap(activityInfoMap);
}
public List<ActivityDef> getActivityDefs() {
return activityInfoMap.values().stream().map(ari -> ari.getActivity().getActivityDef()).toList();
public List<ActivityConfig> getActivityConfigs() {
return activityInfoMap.values().stream().map(ari -> ari.getActivity().getConfig()).toList();
}
public void reportMetrics() {
@ -448,7 +447,11 @@ public class ContainerActivitiesController extends NBBaseComponent {
}
return Optional.empty();
}
public Optional<Activity> getActivity(String activityName) {
public Activity getActivity(String activityName) {
return getOptionalActivity(activityName).orElseThrow(() -> new RuntimeException("Unable " +
"to find required activity by name: '" + activityName + "'"));
}
public Optional<Activity> getOptionalActivity(String activityName) {
return Optional.ofNullable(this.activityInfoMap.get(activityName)).map(ActivityRuntimeInfo::getActivity);
}
@ -469,8 +472,8 @@ public class ContainerActivitiesController extends NBBaseComponent {
throw new RuntimeException(e);
}
public ActivityDef getActivityDef(String alias) {
return activityInfoMap.get(alias).getActivity().getActivityDef();
public ActivityConfig getActivityConfig(String alias) {
return activityInfoMap.get(alias).getActivity().getConfig();
}
public void shutdown() {

14
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/bindings/PolyglotScenarioController.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.core.lifecycle.scenario.script.bindings;
import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.graalvm.polyglot.Value;
@ -88,8 +88,8 @@ public class PolyglotScenarioController {
controller.run(timeout, spec.as(Map.class));
} else if (spec.isHostObject()) {
Object o = spec.asHostObject();
if (o instanceof ActivityDef) {
controller.run((ActivityDef) o, timeout);
if (o instanceof ActivityConfig) {
controller.run((ActivityConfig) o, timeout);
} else {
throw new RuntimeException("unrecognized polyglot host object type for run: " + spec);
}
@ -113,7 +113,7 @@ public class PolyglotScenarioController {
private synchronized void startValue(Value spec) {
if (spec.isHostObject()) {
controller.start((ActivityDef) spec.asHostObject());
controller.start((ActivityConfig) spec.asHostObject());
} else if (spec.isString()) {
controller.start(spec.asString());
} else if (spec.hasMembers()) {
@ -137,7 +137,7 @@ public class PolyglotScenarioController {
private synchronized void stopValue(Value spec) {
if (spec.isHostObject()) {
controller.stop((ActivityDef) spec.asHostObject());
controller.stop((ActivityConfig) spec.asHostObject());
} else if (spec.isString()) {
controller.stop(spec.asString());
} else if (spec.hasMembers()) {
@ -162,7 +162,7 @@ public class PolyglotScenarioController {
private synchronized void forceStopValue(Value spec) {
if (spec.isHostObject()) {
controller.forceStop((ActivityDef) spec.asHostObject());
controller.forceStop((ActivityConfig) spec.asHostObject());
} else if (spec.isString()) {
controller.forceStop(spec.asString());
} else if (spec.hasMembers()) {
@ -239,7 +239,7 @@ public class PolyglotScenarioController {
private synchronized boolean isRunningActivityValue(Value spec) {
if (spec.isHostObject()) {
return controller.isRunningActivity((ActivityDef) spec.asHostObject());
return controller.isRunningActivity((ActivityConfig) spec.asHostObject());
} else if (spec.isString()) {
return controller.isRunningActivity(spec.asString());
} else if (spec.hasMembers()) {

4
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java
View File

@ -25,7 +25,7 @@ import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBCommandResult;
import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
import io.nosqlbench.nb.api.components.decorators.NBTokenWords;
import io.nosqlbench.nb.api.components.status.NBHeartbeatComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
import org.apache.logging.log4j.LogManager;
@ -165,7 +165,7 @@ public class NBSession extends NBHeartbeatComponent implements Function<List<Cmd
for (String containerName : containers.keySet()) {
NBBufferedContainer ctx = containers.get(containerName);
logger.debug("awaiting end of activities in container '" + containerName + "':" +
ctx.controller().getActivityDefs().stream().map(ActivityDef::getAlias).toList());
ctx.controller().getActivityConfigs().stream().map(ActivityConfig::getAlias).toList());
ctx.controller().shutdown();
ctx.controller().awaitCompletion(Long.MAX_VALUE);
logger.debug("completed");

1
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/metadata/MarkdownFinder.java
View File

@ -23,7 +23,6 @@ import io.nosqlbench.nb.api.config.standard.TestComponent;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.LogManager;

18
nb-engine/nb-engine-core/src/test/java/io/nosqlbench/engine/api/activityimpl/ActivityDefTest.java → nb-engine/nb-engine-core/src/test/java/io/nosqlbench/engine/api/activityimpl/ActivityConfigTest.java
View File

@ -16,26 +16,26 @@
package io.nosqlbench.engine.api.activityimpl;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class ActivityDefTest {
public class ActivityConfigTest {
@Test
public void testSimpleCycleCount() {
ActivityDef d = ActivityDef.parseActivityDef("cycles=1M");
assertThat(d.getStartCycle()).isEqualTo(0);
assertThat(d.getEndCycle()).isEqualTo(1000000);
ActivityConfig config = Activity.configFor("cycles=1M");
assertThat(config.getCyclesSpec().firstSpec()).isEqualTo(0L);
assertThat(config.getCyclesSpec().last_exclusive()).isEqualTo(1000000L);
}
@Test
public void testCycleRange() {
ActivityDef d = ActivityDef.parseActivityDef("cycles=1M..5M");
assertThat(d.getStartCycle()).isEqualTo(1000000);
assertThat(d.getEndCycle()).isEqualTo(5000000);
assertThat(d.getCycleCount()).isEqualTo(4000000);
ActivityConfig config = Activity.configFor("cycles=1M..5M");
assertThat(config.getCyclesSpec().firstSpec()).isEqualTo(1000000L);
assertThat(config.getCyclesSpec().last_exclusive()).isEqualTo(5000000L);
}
}

20
nb-engine/nb-engine-core/src/test/java/io/nosqlbench/engine/api/activityimpl/input/AtomicInputTest.java
View File

@ -16,8 +16,8 @@
package io.nosqlbench.engine.api.activityimpl.input;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.config.standard.TestComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityapi.cyclelog.buffers.results.CycleSegment;
import org.junit.jupiter.api.Test;
@ -25,16 +25,20 @@ import static org.assertj.core.api.Assertions.assertThat;
public class AtomicInputTest {
private Activity activity(String cfg) {
return new Activity(new TestComponent("testing","atomicinput"),
Activity.configFor(cfg));
}
@Test
public void testThatNoCyclesAndNoRecyclesMeansZero() {
AtomicInput input = new AtomicInput(new TestComponent("testing","atomicinput"), ActivityDef.parseActivityDef("alias=foo;cycles=0;recycles=0"));
AtomicInput input = new AtomicInput(activity("alias=foo;cycles=0;recycles=0"));
CycleSegment inputSegment = input.getInputSegment(1);
assertThat(inputSegment).isNull();
}
@Test
public void testThatNoCyclesAndDefaultRecyclesMeans1xCycles() {
AtomicInput input = new AtomicInput(new TestComponent("testing","atomicinput"), ActivityDef.parseActivityDef("alias=foo;cycles=10"));
AtomicInput input = new AtomicInput(activity("alias=foo;cycles=10"));
CycleSegment inputSegment =null;
inputSegment= input.getInputSegment(10);
@ -51,7 +55,7 @@ public class AtomicInputTest {
int intendedRecycles=4;
int stride=10;
AtomicInput input = new AtomicInput(new TestComponent("testing","atomicinput"), ActivityDef.parseActivityDef("alias=foo;cycles="+intendedCycles+";recycles="+intendedRecycles));
AtomicInput input = new AtomicInput(activity("alias=foo;cycles="+intendedCycles+";recycles="+intendedRecycles));
CycleSegment segment =null;
for (int nextRecycle = 0; nextRecycle < intendedRecycles; nextRecycle++) {
for (int nextCycle = 0; nextCycle < intendedCycles; nextCycle+=stride) {
@ -66,14 +70,14 @@ public class AtomicInputTest {
@Test
public void testThatOneCycleAndOneRecycleYieldsOneTotal() {
AtomicInput input = new AtomicInput(new TestComponent("testing","atomicinput"), ActivityDef.parseActivityDef("alias=foo;cycles=1;recycles=1"));
AtomicInput input = new AtomicInput(activity("alias=foo;cycles=1;recycles=1"));
CycleSegment segment = input.getInputSegment(1);
assertThat(segment).isNotNull();
assertThat(segment.nextCycle()).isEqualTo(0L);
}
@Test
public void testThatCycleAndRecycleOffsetsWork() {
AtomicInput input = new AtomicInput(new TestComponent("testing","atomicinput"), ActivityDef.parseActivityDef("alias=foo;cycles=310..330;recycles=37..39"));
AtomicInput input = new AtomicInput(activity("alias=foo;cycles=310..330;recycles=37..39"));
CycleSegment segment = null;
int stride=10;
segment = input.getInputSegment(stride);
@ -95,8 +99,8 @@ public class AtomicInputTest {
@Test
public void testEmptyIntervalShouldNotProvideValues() {
AtomicInput i = new AtomicInput(new TestComponent("testing","atomicinput"),ActivityDef.parseActivityDef("alias=foo;cycles=23..23"));
CycleSegment inputSegment = i.getInputSegment(1);
AtomicInput input = new AtomicInput(activity("alias=foo;cycles=23..23"));
CycleSegment inputSegment = input.getInputSegment(1);
assertThat(inputSegment).isNull();
}
}

12
nb-engine/nb-engine-core/src/test/java/io/nosqlbench/engine/api/activityimpl/input/LinkedInputTest.java
View File

@ -21,8 +21,8 @@ public class LinkedInputTest {
// TODO: Reintegrate these tests as follow the leader tests via output and input
// @Test
// public void shouldStayAtOrBehindLinkedInput() {
// Input goes2Kfast = new TargetRateInput(ActivityDef.parseActivityDef("alias=goes2k;targetrate=2000"));
// LinkedInput goesAsFast = new LinkedInput(ActivityDef.parseActivityDef("alias=asfast"),goes2Kfast);
// Input goes2Kfast = new TargetRateInput(Activity.configFor("alias=goes2k;targetrate=2000"));
// LinkedInput goesAsFast = new LinkedInput(Activity.configFor("alias=asfast"),goes2Kfast);
//
// long last2kFast = 0L;
// long lastAsFast = 0L;
@ -37,8 +37,8 @@ public class LinkedInputTest {
//
// @Test
// public void shouldBlockUntilLinkedAdvances() {
// ContiguousInput goes2Kfast = new TargetRateInput(ActivityDef.parseActivityDef("targetrate=2000"));
// LinkedInput goesAsFast = new LinkedInput(ActivityDef.parseActivityDef("alias=asfast"),goes2Kfast);
// ContiguousInput goes2Kfast = new TargetRateInput(Activity.configFor("targetrate=2000"));
// LinkedInput goesAsFast = new LinkedInput(Activity.configFor("alias=asfast"),goes2Kfast);
//
// AtomicLong asFastValue = new AtomicLong(0L);
// Runnable linked = new Runnable() {
@ -68,8 +68,8 @@ public class LinkedInputTest {
//
// @Test(enabled=false)
// public void microBenchDiffRate() {
// TargetRateInput fastInput = new TargetRateInput(ActivityDef.parseActivityDef("targetrate=10000000"));
// LinkedInput slowInput = new LinkedInput(ActivityDef.parseActivityDef("alias=asfast"),fastInput);
// TargetRateInput fastInput = new TargetRateInput(Activity.configFor("targetrate=10000000"));
// LinkedInput slowInput = new LinkedInput(Activity.configFor("alias=asfast"),fastInput);
// Timer fastInputTimer = new NicerTimer("fastinput", new DeltaHdrHistogramReservoir("fastinput",4));
// Timer slowInputTimer = new NicerTimer("slowinput", new DeltaHdrHistogramReservoir("slowinput",4));
//

318
nb-engine/nb-engine-core/src/test/java/io/nosqlbench/engine/api/util/SSLKsFactoryTest.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.util;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.engine.util.SSLKsFactory;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import org.junit.jupiter.api.Test;
@ -29,197 +29,163 @@ import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
public class SSLKsFactoryTest {
@Test
public void testJdkGetContext() {
String[] params = {
"ssl=jdk",
"tlsversion=TLSv1.2",
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg = sslCfg("ssl=jdk", "tlsversion=TLSv1.2");
assertThat(SSLKsFactory.get().getContext(sslCfg)).isNotNull();
}
private NBConfiguration sslCfg(String... params) {
return SSLKsFactory.get().getConfigModel()
.extractConfig(Activity.configFor(String.join(";",params)));
}
@Test
public void testJdkGetContextWithTruststoreAndKeystore() {
String[] params = {
"ssl=jdk",
"truststore=src/test/resources/ssl/server_truststore.p12",
"tspass=nosqlbench_server",
"keystore=src/test/resources/ssl/client.p12",
"kspass=nosqlbench_client"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg = sslCfg(
"ssl=jdk",
"truststore=src/test/resources/ssl/server_truststore.p12",
"tspass=nosqlbench_server",
"keystore=src/test/resources/ssl/client.p12",
"kspass=nosqlbench_client"
);
assertThat(SSLKsFactory.get().getContext(sslCfg)).isNotNull();
}
@Test
public void testJdkGetContextWithTruststoreAndKeystoreAndDifferentKeyPassword() {
String[] params = {
"ssl=jdk",
"truststore=src/test/resources/ssl/server_truststore.p12",
"tspass=nosqlbench_server",
"keystore=src/test/resources/ssl/client_diff_password.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg = sslCfg(
"ssl=jdk",
"truststore=src/test/resources/ssl/server_truststore.p12",
"tspass=nosqlbench_server",
"keystore=src/test/resources/ssl/client_diff_password.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench"
);
assertThat(SSLKsFactory.get().getContext(sslCfg)).isNotNull();
}
@Test
public void testJdkGetContextWithTruststore() {
String[] params = {
"ssl=jdk",
"truststore=src/test/resources/ssl/server_truststore.p12",
"tspass=nosqlbench_server"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg = sslCfg(
"ssl=jdk",
"truststore=src/test/resources/ssl/server_truststore.p12",
"tspass=nosqlbench_server"
);
assertThat(SSLKsFactory.get().getContext(sslCfg)).isNotNull();
}
@Test
public void testJdkGetContextWithKeystore() {
String[] params = {
"ssl=jdk",
"keystore=src/test/resources/ssl/client.p12",
"kspass=nosqlbench_client"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg = sslCfg(
"ssl=jdk", "keystore=src/test/resources/ssl/client.p12", "kspass=nosqlbench_client"
);
assertThat(SSLKsFactory.get().getContext(sslCfg)).isNotNull();
}
@Test
public void testJdkGetContextWithKeystoreAndDifferentKeyPassword() {
String[] params = {
"ssl=jdk",
"keystore=src/test/resources/ssl/client_diff_password.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg = sslCfg(
"ssl=jdk",
"keystore=src/test/resources/ssl/client_diff_password.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench"
);
assertThat(SSLKsFactory.get().getContext(sslCfg)).isNotNull();
}
@Test
public void testOpenSSLGetContext() {
String[] params = {
"ssl=openssl",
"tlsversion=TLSv1.2",
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg = sslCfg("ssl=openssl", "tlsversion=TLSv1.2");
assertThat(SSLKsFactory.get().getContext(sslCfg)).isNotNull();
}
@Test
public void testOpenSSLGetContextWithCaCertAndCertAndKey() {
String[] params = {
"ssl=openssl",
"caCertFilePath=src/test/resources/ssl/cacert.crt",
"certFilePath=src/test/resources/ssl/client_cert.pem",
"keyFilePath=src/test/resources/ssl/client.key"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg = sslCfg(
"ssl=openssl",
"caCertFilePath=src/test/resources/ssl/cacert.crt",
"certFilePath=src/test/resources/ssl/client_cert.pem",
"keyFilePath=src/test/resources/ssl/client.key"
);
assertThat(SSLKsFactory.get().getContext(sslCfg)).isNotNull();
}
@Test
public void testOpenSSLGetContextWithCaCert() {
String[] params = {
"ssl=openssl",
"caCertFilePath=src/test/resources/ssl/cacert.crt"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg = sslCfg(
"ssl=openssl", "caCertFilePath=src/test/resources/ssl/cacert.crt"
);
assertThat(SSLKsFactory.get().getContext(sslCfg)).isNotNull();
}
@Test
public void testOpenSSLGetContextWithCertAndKey() {
String[] params = {
"ssl=openssl",
"certFilePath=src/test/resources/ssl/client_cert.pem",
"keyFilePath=src/test/resources/ssl/client.key"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg = sslCfg(
"ssl=openssl",
"certFilePath=src/test/resources/ssl/client_cert.pem",
"keyFilePath=src/test/resources/ssl/client.key"
);
assertThat(SSLKsFactory.get().getContext(sslCfg)).isNotNull();
}
@Test
public void testLoadKeystoreError() {
String[] params = {
"ssl=jdk",
"keystore=src/test/resources/ssl/non_existing.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench_client"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageMatching("Unable to load the keystore: .*");
NBConfiguration sslCfg = sslCfg(
"ssl=jdk",
"keystore=src/test/resources/ssl/non_existing.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench_client"
);
assertThatExceptionOfType(RuntimeException.class).isThrownBy(
() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageMatching("Unable to load the keystore: .*");
}
@Test
public void testInitKeyManagerFactoryError() {
String[] params = {
"ssl=jdk",
"keystore=src/test/resources/ssl/client.p12",
"kspass=nosqlbench_client",
"keyPassword=incorrect_password"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageMatching("Unable to init KeyManagerFactory. Please check.*");
NBConfiguration sslCfg = sslCfg(
"ssl=jdk",
"keystore=src/test/resources/ssl/client.p12",
"kspass=nosqlbench_client",
"keyPassword=incorrect_password"
);
assertThatExceptionOfType(RuntimeException.class).isThrownBy(
() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageMatching("Unable to init KeyManagerFactory. Please check.*");
}
@Test
public void testLoadTruststoreError() {
String[] params = {
"ssl=jdk",
"truststore=src/test/resources/ssl/non_existing.p12",
"tspass=nosqlbench_server"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageMatching("Unable to load the truststore: .*");
NBConfiguration sslCfg = sslCfg(
"ssl=jdk",
"truststore=src/test/resources/ssl/non_existing.p12",
"tspass=nosqlbench_server"
);
assertThatExceptionOfType(RuntimeException.class).isThrownBy(
() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageMatching("Unable to load the truststore: .*");
}
@Test
void testSSLValidationActive() {
{
final String[] params1 = {
"ssl=openssl",
"certFilePath=src/test/resources/ssl/client_cert.pem",
"keyFilePath=src/test/resources/ssl/client.key",
"sslValidation=true"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params1));
NBConfiguration sslCfg1 = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg1 = sslCfg(
"ssl=openssl",
"certFilePath=src/test/resources/ssl/client_cert.pem",
"keyFilePath=src/test/resources/ssl/client.key",
"sslValidation=true"
);
assertThat(SSLKsFactory.get().getContext(sslCfg1)).isNotNull();
}
{
final String[] params2 = {
"ssl=jdk",
"keystore=src/test/resources/ssl/client_diff_password.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench",
"sslValidation=true"
};
ActivityDef activityDef2 = ActivityDef.parseActivityDef(String.join(";", params2));
NBConfiguration sslCfg2 = SSLKsFactory.get().getConfigModel().extractConfig(activityDef2.getParams());
NBConfiguration sslCfg2 = sslCfg( "ssl=jdk",
"keystore=src/test/resources/ssl/client_diff_password.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench",
"sslValidation=true"
);
assertThat(SSLKsFactory.get().getContext(sslCfg2)).isNotNull();
}
}
@ -227,86 +193,72 @@ public class SSLKsFactoryTest {
@Test
void testSSLValidationNotActive() {
{
final String[] params1 = {
"ssl=openssl",
"certFilePath=src/test/resources/ssl/client_cert.pem",
"keyFilePath=src/test/resources/ssl/client.key",
"sslValidation=false"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params1));
NBConfiguration sslCfg1 = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
NBConfiguration sslCfg1 = sslCfg(
"ssl=openssl",
"certFilePath=src/test/resources/ssl/client_cert.pem",
"keyFilePath=src/test/resources/ssl/client.key",
"sslValidation=false"
);
assertThat(SSLKsFactory.get().getContext(sslCfg1)).isNotNull();
}
{
final String[] params2 = {
"ssl=jdk",
"keystore=src/test/resources/ssl/client_diff_password.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench",
"sslValidation=false"
};
ActivityDef activityDef2 = ActivityDef.parseActivityDef(String.join(";", params2));
NBConfiguration sslCfg2 = SSLKsFactory.get().getConfigModel().extractConfig(activityDef2.getParams());
NBConfiguration sslCfg2 = sslCfg( "ssl=jdk",
"keystore=src/test/resources/ssl/client_diff_password.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench",
"sslValidation=false"
);
assertThat(SSLKsFactory.get().getContext(sslCfg2)).isNotNull();
}
}
@Test
public void testOpenSSLGetContextWithCaCertError() {
String[] params = {
"ssl=openssl",
"caCertFilePath=src/test/resources/ssl/non_existing.pem"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageContaining("Unable to load caCert from")
.withCauseInstanceOf(FileNotFoundException.class);
NBConfiguration sslCfg = sslCfg(
"ssl=openssl", "caCertFilePath=src/test/resources/ssl/non_existing.pem"
);
assertThatExceptionOfType(RuntimeException.class).isThrownBy(
() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageContaining("Unable to load caCert from")
.withCauseInstanceOf(FileNotFoundException.class);
}
@Test
public void testOpenSSLGetContextWithCertError() {
String[] params = {
"ssl=openssl",
"certFilePath=src/test/resources/ssl/non_existing.pem"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageContaining("Unable to load cert from")
.withCauseInstanceOf(FileNotFoundException.class);
NBConfiguration sslCfg = sslCfg(
"ssl=openssl", "certFilePath=src/test/resources/ssl/non_existing.pem"
);
assertThatExceptionOfType(RuntimeException.class).isThrownBy(
() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageContaining("Unable to load cert from")
.withCauseInstanceOf(FileNotFoundException.class);
}
@Test
public void testOpenSSLGetContextWithKeyError() {
String[] params = {
"ssl=openssl",
"keyFilePath=src/test/resources/ssl/non_existing.pem"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageContaining("Unable to load key from")
.withCauseInstanceOf(FileNotFoundException.class);
NBConfiguration sslCfg = sslCfg(
"ssl=openssl", "keyFilePath=src/test/resources/ssl/non_existing.pem"
);
assertThatExceptionOfType(RuntimeException.class).isThrownBy(
() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageContaining("Unable to load key from")
.withCauseInstanceOf(FileNotFoundException.class);
}
@Test
public void testOpenSSLGetContextWithMissingCertError() {
String[] params = {
"ssl=openssl",
"caCertFilePath=src/test/resources/ssl/cacert.crt",
"keyFilePath=src/test/resources/ssl/client.key"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageContaining("Unable to load key from")
.withCauseInstanceOf(IllegalArgumentException.class);
NBConfiguration sslCfg = sslCfg(
"ssl=openssl",
"caCertFilePath=src/test/resources/ssl/cacert.crt",
"keyFilePath=src/test/resources/ssl/client.key"
);
assertThatExceptionOfType(RuntimeException.class).isThrownBy(
() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageContaining("Unable to load key from")
.withCauseInstanceOf(IllegalArgumentException.class);
}
}

22
nb-engine/nb-engine-core/src/test/java/io/nosqlbench/engine/core/ActivityExecutorTest.java
View File

@ -19,8 +19,9 @@ package io.nosqlbench.engine.core;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.engine.api.activityimpl.uniform.ActivityWiring;
import io.nosqlbench.engine.api.activityimpl.uniform.StandardActivityType;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.config.standard.TestComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import io.nosqlbench.nb.api.advisor.NBAdvisorException;
import io.nosqlbench.engine.api.activityapi.core.*;
import io.nosqlbench.engine.api.activityapi.input.Input;
@ -50,7 +51,7 @@ class ActivityExecutorTest {
// TODO: Design review of this mechanism
// @Test
// synchronized void testRestart() {
// ActivityDef activityDef = ActivityDef.parseActivityDef("driver=diag;alias=test-restart;cycles=1000;cyclerate=10;op=initdelay:initdelay=5000;");
// ActivityConfig activityDef = Activity.configFor("driver=diag;alias=test-restart;cycles=1000;cyclerate=10;op=initdelay:initdelay=5000;");
// new ActivityTypeLoader().load(activityDef);
//
// final StandardActivity activity = new DelayedInitActivity(activityDef);
@ -89,7 +90,7 @@ class ActivityExecutorTest {
synchronized void testAdvisorError() {
try {
ActivityDef activityDef = ActivityDef.parseActivityDef(
ActivityConfig activityDef = Activity.configFor(
"driver=diag;alias=test-delayed-start;cycles=1000;initdelay=2000;");
new ActivityTypeLoader().load(activityDef, TestComponent.INSTANCE);
Activity activity = new DelayedInitActivity(activityDef);
@ -103,7 +104,7 @@ class ActivityExecutorTest {
@Test
synchronized void testDelayedStartSanity() {
ActivityDef activityDef = ActivityDef.parseActivityDef(
ActivityConfig activityDef = Activity.configFor(
"driver=diag;alias=test_delayed_start;cycles=1000;initdelay=2000;");
Optional<StandardActivityType> standardActivityType = new ActivityTypeLoader().load(
activityDef, TestComponent.INSTANCE);
@ -146,7 +147,7 @@ class ActivityExecutorTest {
@Test
synchronized void testNewActivityExecutor() {
final ActivityDef activityDef = ActivityDef.parseActivityDef(
final ActivityConfig activityDef = Activity.configFor(
"driver=diag;alias=test_dynamic_params;cycles=1000;initdelay=5000;");
new ActivityTypeLoader().load(activityDef, TestComponent.INSTANCE);
ActivityWiring wiring = new ActivityWiring(activityDef);
@ -202,8 +203,9 @@ class ActivityExecutorTest {
private MotorDispenser<?> getActivityMotorFactory(final SyncAction lc, Input ls) {
return new MotorDispenser<>() {
@Override
public Motor getMotor(final ActivityDef activityDef, final int slotId) {
final Activity activity = new Activity(TestComponent.INSTANCE, activityDef);
public Motor getMotor(final ActivityConfig activityConfig, final int slotId) {
final Activity activity = new Activity(TestComponent.INSTANCE,
new ActivityConfig(activityConfig));
final Motor<?> cm = new CoreMotor<>(activity, slotId, ls, lc, null);
return cm;
}
@ -229,14 +231,14 @@ class ActivityExecutorTest {
private static class DelayedInitActivity extends Activity {
private static final Logger logger = LogManager.getLogger(DelayedInitActivity.class);
public DelayedInitActivity(final ActivityDef activityDef) {
public DelayedInitActivity(final ActivityConfig activityDef) {
super(TestComponent.INSTANCE, activityDef);
}
@Override
public void initActivity() {
final Integer initDelay = this.activityDef.getParams().getOptionalInteger(
"initdelay").orElse(0);
final Integer initDelay =
this.getConfig().getOptional(Integer.class,"initdelay").orElse(0);
DelayedInitActivity.logger.info(() -> "delaying for " + initDelay);
try {
Thread.sleep(initDelay);

8
nb-engine/nb-engine-core/src/test/java/io/nosqlbench/engine/core/CoreMotorTest.java
View File

@ -18,11 +18,11 @@ package io.nosqlbench.engine.core;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.nb.api.config.standard.TestComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityapi.core.Motor;
import io.nosqlbench.engine.api.activityapi.core.SyncAction;
import io.nosqlbench.engine.api.activityimpl.motor.CoreMotor;
import io.nosqlbench.engine.core.fortesting.BlockingSegmentInput;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityConfig;
import org.junit.jupiter.api.Test;
import java.util.concurrent.atomic.AtomicLong;
@ -35,9 +35,9 @@ public class CoreMotorTest {
@Test
public void testBasicActivityMotor() {
ActivityDef activityDef = ActivityDef.parseActivityDef("alias=foo");
ActivityConfig config = Activity.configFor("alias=foo");
final Activity activity = new Activity<>(
new TestComponent("testing", "coremotor"), activityDef);
new TestComponent("testing", "coremotor"), config);
final BlockingSegmentInput lockstepper = new BlockingSegmentInput();
final AtomicLong observableAction = new AtomicLong(-3L);
SyncAction action = this.getTestConsumer(observableAction);
@ -58,7 +58,7 @@ public class CoreMotorTest {
@Test
public void testIteratorStride() {
ActivityDef activityDef = ActivityDef.parseActivityDef("stride=3");
ActivityConfig activityDef = Activity.configFor("stride=3");
Activity activity = new Activity(
TestComponent.INSTANCE, activityDef);
final BlockingSegmentInput lockstepper = new BlockingSegmentInput();

3
nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_activity_error.java
View File

@ -61,7 +61,8 @@ public class NB_activity_error extends NBBaseCommand {
stdout.write("starting activity activity_error");
controller.start(activitydef1);
controller.waitMillis(500);
controller.getActivityDef("activity_error").getParams().set("threads","unparsable"); // forced error
controller.getActivity("activity_error").getConfig().update("threads","unparsable"); //
// forced error
controller.awaitActivity("activity_error", Long.MAX_VALUE);
return null;
}

165
nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_cocycledelay_bursty_backup.java
View File

@ -16,6 +16,7 @@
package io.nosqlbench.nbr.examples.injava;
import io.nosqlbench.engine.api.activityimpl.uniform.Activity;
import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricCounter;
@ -35,71 +36,81 @@ public class NB_cocycledelay_bursty_backup extends NBBaseCommand {
}
/**
* <pre>{@code
* co_cycle_delay_bursty = {
* "alias": "co_cycle_delay_bursty",
* "driver": "diag",
* "cycles": "0..1000000",
* "threads": "10",
* "cyclerate": "1000,1.5",
* "op" : "diagrate: diagrate=500"
* };
*
* print('starting activity co_cycle_delay_bursty');
* scenario.start(co_cycle_delay_bursty);
* for (i = 0; i < 5; i++) {
* scenario.waitMillis(1000);
* if (!scenario.isRunningActivity('co_cycle_delay_bursty')) {
* print("scenario exited prematurely, aborting.");
* break;
* }
* print("backlogging, cycles=" + metrics.co_cycle_delay_bursty.cycles_servicetime.count +
* " waittime=" + metrics.co_cycle_delay_bursty.cycles_waittime.value +
* " diagrate=" + activities.co_cycle_delay_bursty.diagrate +
* " cyclerate=" + activities.co_cycle_delay_bursty.cyclerate
* );
* }
* print('step1 metrics.waittime=' + metrics.co_cycle_delay_bursty.cycles_waittime.value);
* activities.co_cycle_delay_bursty.diagrate = "10000";
*
* for (i = 0; i < 10; i++) {
* if (!scenario.isRunningActivity('co_cycle_delay_bursty')) {
* print("scenario exited prematurely, aborting.");
* break;
* }
* print("recovering, cycles=" + metrics.co_cycle_delay_bursty.cycles_servicetime.count +
* " waittime=" + metrics.co_cycle_delay_bursty.cycles_waittime.value +
* " diagrate=" + activities.co_cycle_delay_bursty.diagrate +
* " cyclerate=" + activities.co_cycle_delay_bursty.cyclerate
* );
*
* scenario.waitMillis(1000);
* if (metrics.co_cycle_delay_bursty.cycles_waittime.value < 50000000) {
* print("waittime trended back down as expected, exiting on iteration " + i);
* break;
* }
* }
* //scenario.awaitActivity("co_cycle_delay");
* print('step2 metrics.waittime=' + metrics.co_cycle_delay_bursty.cycles_waittime.value);
* scenario.stop(co_cycle_delay_bursty);
* print("stopped activity co_cycle_delay_bursty");
* }</pre>
<pre>{@code
co_cycle_delay_bursty = {
"alias": "co_cycle_delay_bursty",
"driver": "diag",
"cycles": "0..1000000",
"threads": "10",
"cyclerate": "1000,1.5",
"op" : "diagrate: diagrate=500"
};
print('starting activity co_cycle_delay_bursty');
scenario.start(co_cycle_delay_bursty);
for (i = 0; i < 5; i++) {
scenario.waitMillis(1000);
if (!scenario.isRunningActivity('co_cycle_delay_bursty')) {
print("scenario exited prematurely, aborting.");
break;
}
print("backlogging, cycles=" + metrics.co_cycle_delay_bursty.cycles_servicetime.count +
" waittime=" + metrics.co_cycle_delay_bursty.cycles_waittime.value +
" diagrate=" + activities.co_cycle_delay_bursty.diagrate +
" cyclerate=" + activities.co_cycle_delay_bursty.cyclerate
);
}
print('step1 metrics.waittime=' + metrics.co_cycle_delay_bursty.cycles_waittime.value);
activities.co_cycle_delay_bursty.diagrate = "10000";
for (i = 0; i < 10; i++) {
if (!scenario.isRunningActivity('co_cycle_delay_bursty')) {
print("scenario exited prematurely, aborting.");
break;
}
print("recovering, cycles=" + metrics.co_cycle_delay_bursty.cycles_servicetime.count +
" waittime=" + metrics.co_cycle_delay_bursty.cycles_waittime.value +
" diagrate=" + activities.co_cycle_delay_bursty.diagrate +
" cyclerate=" + activities.co_cycle_delay_bursty.cyclerate
);
scenario.waitMillis(1000);
if (metrics.co_cycle_delay_bursty.cycles_waittime.value < 50000000) {
print("waittime trended back down as expected, exiting on iteration " + i);
break;
}
}
//scenario.awaitActivity("co_cycle_delay");
print('step2 metrics.waittime=' + metrics.co_cycle_delay_bursty.cycles_waittime.value);
scenario.stop(co_cycle_delay_bursty);
print("stopped activity co_cycle_delay_bursty");
}</pre>
*/
@Override
public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) {
public Object invoke(
NBCommandParams params,
PrintWriter stdout,
PrintWriter stderr,
Reader stdin,
ContainerActivitiesController controller
)
{
Map<String, String> co_cycle_delay_bursty = Map.of(
"alias", "co_cycle_delay_bursty", "driver", "diag", "cycles", "0..1000000", "threads",
"1", "cyclerate", "1000,1.5", "op", "diagrate: diagrate=500"
);
"1", "cyclerate", "1000,1.5", "op", "diagrate: diagrate=500");
stdout.println("starting activity co_cycle_delay_bursty");
controller.start(co_cycle_delay_bursty);
NBMetricCounter service_time_counter = find().counter("activity=co_cycle_delay_bursty,name=cycles_servicetime");
NBMetricGauge wait_time_gauge = find().gauge("activity=co_cycle_delay_bursty,name=cycles_waittime");
String diagrate = controller.getActivityDef("co_cycle_delay_bursty").getParams().get("diagrate").toString();
String cyclerate = controller.getActivityDef("co_cycle_delay_bursty").getParams().get("cyclerate").toString();
NBMetricCounter service_time_counter = find().counter(
"activity=co_cycle_delay_bursty,name=cycles_servicetime");
NBMetricGauge wait_time_gauge = find().gauge(
"activity=co_cycle_delay_bursty,name=cycles_waittime");
String diagrate = controller.getActivity("co_cycle_delay_bursty").getConfig()
.get("diagrate");
String cyclerate = controller.getActivity("co_cycle_delay_bursty").getConfig()
.get("cyclerate").toString();
for (int i = 0; i < 5; i++) {
controller.waitMillis(1000);
@ -107,33 +118,41 @@ public class NB_cocycledelay_bursty_backup extends NBBaseCommand {
stdout.println("scenario exited prematurely, aborting.");
break;
}
diagrate = controller.getActivityDef("co_cycle_delay_bursty").getParams().get("diagrate").toString();
cyclerate = controller.getActivityDef("co_cycle_delay_bursty").getParams().get("cyclerate").toString();
stdout.println(
"backlogging, cycles=" + service_time_counter.getCount() +
" waittime=" + wait_time_gauge.getValue() +
" diagrate=" + diagrate +
" cyclerate=" + cyclerate
);
diagrate = controller.getActivity("co_cycle_delay_bursty").getConfig()
.get("diagrate").toString();
cyclerate = controller.getActivity("co_cycle_delay_bursty").getConfig()
.get("cyclerate").toString();
stdout.println("backlogging, cycles=" +
service_time_counter.getCount() +
" waittime=" +
wait_time_gauge.getValue() +
" diagrate=" +
diagrate +
" cyclerate=" +
cyclerate);
}
stdout.println("step1 metrics.waittime=" + wait_time_gauge.getValue());
controller.getActivityDef("co_cycle_delay_bursty").getParams().put("diagrate", "10000");
controller.getActivity("co_cycle_delay_bursty").getConfig().update("diagrate", "10000");
for (int i = 0; i < 10; i++) {
if (!controller.isRunningActivity("co_cycle_delay_bursty")) {
stdout.println("scenario exited prematurely, aborting.");
break;
}
diagrate = controller.getActivityDef("co_cycle_delay_bursty").getParams().get("diagrate").toString();
cyclerate = controller.getActivityDef("co_cycle_delay_bursty").getParams().get("cyclerate").toString();
diagrate = controller.getActivity("co_cycle_delay_bursty").getConfig()
.get("diagrate").toString();
cyclerate = controller.getActivity("co_cycle_delay_bursty").getConfig()
.get("cyclerate").toString();
stdout.println(
"recovering, cycles=" + service_time_counter.getCount() +
" waittime=" + wait_time_gauge.getValue() +
" diagrate=" + diagrate +
" cyclerate=" + cyclerate
);
stdout.println("recovering, cycles=" +
service_time_counter.getCount() +
" waittime=" +
wait_time_gauge.getValue() +
" diagrate=" +
diagrate +
" cyclerate=" +
cyclerate);
controller.waitMillis(1000);
if (wait_time_gauge.getValue() < 50000000) {

8
nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_threadchange.java
View File

@ -49,13 +49,13 @@ public class NB_threadchange extends NBBaseCommand {
Activity activity = controller.start(
"driver=diag;alias=threadchange;cycles=0..60000;threads=1;interval=2000;op='noop';rate=1000");
activity.getActivityDef().setThreads(1);
stdout.println("threads now " + activity.getActivityDef().getThreads());
activity.getConfig().setThreads(1);
stdout.println("threads now " + activity.getConfig().getThreads());
stdout.println("waiting 500 ms");
controller.waitMillis(500);
activity.getActivityDef().setThreads(5);
stdout.println("threads now " + activity.getActivityDef().getThreads());
activity.getConfig().setThreads(5);
stdout.println("threads now " + activity.getConfig().getThreads());
controller.stop("threadchange");
return null;
}

7
nbr/src/main/java/io/nosqlbench/scenarios/simframe/SimFrameUtils.java
View File

@ -45,7 +45,8 @@ public class SimFrameUtils {
}
public static Activity findFlywheelActivity(ContainerActivitiesController controller, String providedActivityName) {
Optional<Activity> optionalActivity = Optional.ofNullable(providedActivityName).flatMap(controller::getActivity);
Optional<Activity> optionalActivity =
Optional.ofNullable(providedActivityName).flatMap(controller::getOptionalActivity);
if (providedActivityName!=null && optionalActivity.isEmpty()) {
throw new RuntimeException("you specified activity '" + providedActivityName + "' but it was not found.");
}
@ -54,8 +55,8 @@ public class SimFrameUtils {
// Start the flywheel at an "idle" speed, even if the user hasn't set it
flywheel.onEvent(new ParamChange<>(new CycleRateSpec(100.0d, 1.1d, SimRateSpec.Verb.restart)));
flywheel.getActivityDef().setEndCycle(Long.MAX_VALUE);
flywheel.getActivityDef().getParams().set(SIM_CYCLES, Long.MAX_VALUE);
flywheel.getConfig().updateLastCycle(Long.MAX_VALUE);
flywheel.getConfig().update(SIM_CYCLES, Long.MAX_VALUE);
return flywheel;
}

14
nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/CMD_reset.java
View File

@ -61,7 +61,7 @@ public class CMD_reset extends NBBaseCommand {
@Override
public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) {
Optional<Activity> optionalActivity =
Optional.ofNullable(params.get("activity")).flatMap(controller::getActivity);
Optional.ofNullable(params.get("activity")).flatMap(controller::getOptionalActivity);
if (params.get("activity")!=null && optionalActivity.isEmpty()) {
throw new RuntimeException("you specified activity '" + params.get("activity") + "' but it was not found.");
}
@ -81,21 +81,23 @@ public class CMD_reset extends NBBaseCommand {
default -> {
if (!IGNORABLE.contains(key)) {
logger.debug("Resetting parameter: " + key + " to " + value);
flywheel.getActivityDef().getParams().put(key, value);
flywheel.getConfig().update(key, value);
}
}
}
});
// Get the original cycle count and re-apply it
long cycles = Long.parseLong((String) flywheel.getActivityDef().getParams().get("cycles"));
logger.debug("Resetting cycle count to " + cycles + " cycles");
flywheel.getActivityDef().setEndCycle(cycles);
long last_exclusive = flywheel.getConfig().getCyclesSpec().last_exclusive();
logger.debug("Resetting last cycle to " + last_exclusive + " cycles");
flywheel.getConfig().updateLastCycle(last_exclusive);
//TODO: This needs to be reworked, but simply calling controller.start on the flywheel results in 2
// copies of the activity running simultaneously. This is a temporary workaround.
SimFrameUtils.awaitActivity(flywheel);
// flywheel.getWiring().getMotorDispenserDelegate().getMotor(flywheel.getActivityDef(), 0).run();
// flywheel.getWiring().getMotorDispenserDelegate().getMotor(flywheel.getConfig(), 0)
// .run();
// TODO Implement this correctly around new API
}