IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

replace ActivityDef with NBConfigurable

Browse Source
This commit is contained in:
Jonathan Shook
2025-01-02 12:15:11 -06:00
parent 3ab18635bb
commit 97e7b16d77
61 changed files with 1316 additions and 1373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java
View File

@@ -144,55 +144,6 @@ public abstract class BaseDriverAdapter<RESULT
return cfg;
}
@Override
public void applyConfig(NBConfiguration cfg) {
this.cfg = cfg;
}
@Override
public void applyReconfig(NBConfiguration reconf) {
this.cfg = getReconfigModel().apply(reconf.getMap());
}
/**
* In order to be provided with config information, it is required
* that the driver adapter specify the valid configuration options,
* their types, and so on.
*/
@Override
public NBConfigModel getConfigModel() {
return ConfigModel.of(BaseDriverAdapter.class)
.add(Param.optional("alias"))
.add(Param.optional("labels", String.class, "Labels which will apply to metrics and annotations for this activity only"))
.add(Param.defaultTo("strict", true, "strict op field mode, which requires that provided op fields are recognized and used"))
.add(Param.optional(List.of("op", "stmt", "statement"), String.class, "op template in statement form"))
.add(Param.optional("tags", String.class, "tags to be used to filter operations"))
.add(Param.defaultTo("errors", "stop", "error handler configuration"))
.add(Param.optional("threads").setRegex("\\d+|\\d+x|auto").setDescription("number of concurrent operations, controlled by threadpool"))
.add(Param.optional("stride").setRegex("\\d+"))
.add(Param.optional("striderate", String.class, "rate limit for strides per second"))
.add(Param.optional("cycles").setRegex("\\d+[KMBGTPE]?|\\d+[KMBGTPE]?\\.\\.\\d+[KMBGTPE]?").setDescription("cycle interval to use"))
.add(Param.optional("recycles").setDescription("allow cycles to be re-used this many times"))
.add(Param.optional(List.of("cyclerate", "targetrate", "rate"), String.class, "rate limit for cycles per second"))
.add(Param.optional("seq", String.class, "sequencing algorithm"))
.add(Param.optional("instrument", Boolean.class))
.add(Param.optional(List.of("workload", "yaml"), String.class, "location of workload yaml file"))
.add(Param.optional("driver", String.class))
.add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|emit|none)"))
.add(Param.optional("maxtries", Integer.class))
.asReadOnly();
}
@Override
public NBConfigModel getReconfigModel() {
return ConfigModel.of(BaseDriverAdapter.class)
.add(Param.optional("threads").setRegex("\\d+|\\d+x|auto").setDescription("number of concurrent operations, controlled by threadpool"))
.add(Param.optional("striderate", String.class, "rate limit for strides per second"))
.add(Param.optional(List.of("cyclerate", "targetrate", "rate"), String.class, "rate limit for cycles per second"))
.asReadOnly();
}
@Override
public LongFunction<SPACE> getSpaceFunc(ParsedOp pop) {
@@ -233,4 +184,30 @@ public abstract class BaseDriverAdapter<RESULT
}
super.beforeDetach();
}
@Override
public void applyConfig(NBConfiguration cfg) {
this.cfg = cfg;
}
@Override
public void applyReconfig(NBConfiguration reconf) {
this.cfg = getReconfigModel().apply(reconf.getMap());
}
/// These are implemented here as _unit_ values, meaning, you shouldn't be asking
/// "Does this element have a configuration model", but instead you should be asking
/// "What is the (possibly empty?) configuration model of this element?"
@Override
public NBConfigModel getConfigModel() {
return ConfigModel.of(BaseDriverAdapter.class).asReadOnly();
}
/// These are implemented here as _unit_ values, meaning, you shouldn't be asking
/// "Does this element have a reconfiguration model", but instead you should be asking
/// "What is the (possibly empty?) reconfiguration model of this element?"
@Override
public NBConfigModel getReconfigModel() {
return ConfigModel.of(BaseDriverAdapter.class).asReadOnly();
}
}

32
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java
View File

@@ -394,28 +394,24 @@ public class ParsedOp extends NBBaseComponent implements LongFunction<Map<String
private final List<CapturePoint> captures = new ArrayList<>();
private final OpTemplate _opTemplate;
private final NBConfiguration activityCfg;
private final Map<String,Object> activityCfg;
private final ParsedTemplateMap tmap;
private final NBLabels labels;
private final List<Function<Map<String, Object>, Map<String, Object>>> preprocessors;
/**
Create a parsed command from an Op template. This version is exactly like
except that it allows
preprocessors. Preprocessors are all applied to the the op template before
it is applied to the parsed command fields, allowing you to combine or destructure
fields from more tha one representation into a single canonical representation
for processing.
@param opTemplate
The OpTemplate as provided by a user via YAML, JSON, or API (data structure)
@param activityCfg
The activity configuration, used to resolve nested config parameters
@param preprocessors
Map->Map transformers.
*/
public ParsedOp(ParsedOp pop, NBConfiguration config) {
this(pop._opTemplate,new LinkedHashMap<>(pop.activityCfg) {{ this.putAll(config.getMap());}},List.of(),pop.parent);
}
/// Create a parsed command from an Op template. Preprocessors are all applied to the the op template before
/// it is applied to the parsed command fields, allowing you to combine or destructure
/// fields from more tha one representation into a single canonical representation
/// for processing.
/// @param opTemplate The OpTemplate as provided by a user via YAML, JSON, or API (data structure)
/// @param activityCfg The activity configuration, used to resolve nested config parameters
/// @param preprocessors Map->Map transformers.
public ParsedOp(
OpTemplate opTemplate, NBConfiguration activityCfg,
OpTemplate opTemplate,
Map<String,Object> activityCfg,
List<Function<Map<String, Object>, Map<String, Object>>> preprocessors,
NBComponent parent
) {
@@ -439,7 +435,7 @@ public class ParsedOp extends NBBaseComponent implements LongFunction<Map<String
this.tmap = new ParsedTemplateMap(
getName(), map, opTemplate.getBindings(),
List.of(opTemplate.getParams(), activityCfg.getMap())
List.of(opTemplate.getParams(), activityCfg)
);
NBLabels opLabels = parent.getLabels().and(

6
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java
View File

@@ -20,7 +20,7 @@ import com.google.gson.Gson;
import io.nosqlbench.nb.api.advisor.NBAdvisorBuilder;
import io.nosqlbench.nb.api.advisor.NBAdvisorPoint;
import io.nosqlbench.nb.api.advisor.conditions.Conditions;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.errors.OpConfigError;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
@@ -53,9 +53,9 @@ public class StrInterpolator implements Function<String, String> {
private final Pattern COMMENT = Pattern.compile("^\\s*#.*");
private final Pattern INSERT = Pattern.compile("^(\\s*)INSERT:\\s+(.+)$");
public StrInterpolator(ActivityDef... activityDefs) {
public StrInterpolator(NBConfiguration... activityDefs) {
Arrays.stream(activityDefs)
.map(ad -> ad.getParams().getStringStringMap())
.map(ad -> ad.getMap())
.forEach(multimap::add);
}

140
nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/ParsedOpTest.java
View File

@@ -37,39 +37,27 @@ import static org.assertj.core.api.Assertions.assertThat;
public class ParsedOpTest {
private NBComponent getParent() {
return new TestComponent("opparent","opparent");
return new TestComponent("opparent", "opparent");
}
private ParsedOp getOp() {
ParsedOp pc = new ParsedOp(
new OpData().applyFields(
Map.of(
"op", Map.of(
"stmt", "test",
"dyna1", "{dyna1}",
"dyna2", "{{NumberNameToString()}}",
"identity", "{{Identity()}}"
),
"bindings", Map.of(
"dyna1", "NumberNameToString()"
)
)
),
ConfigModel.of(ParsedOpTest.class)
.add(Param.defaultTo("testcfg", "testval"))
.asReadOnly()
.apply(Map.of()),
List.of(),
getParent()
);
OpData opTemplate = new OpData().applyFields(Map.of(
"op", Map.of(
"stmt", "test", "dyna1", "{dyna1}", "dyna2", "{{NumberNameToString()}}", "identity",
"{{Identity()}}"), "bindings", Map.of("dyna1", "NumberNameToString()")));
NBConfiguration nbcfg = ConfigModel.of(ParsedOpTest.class)
.add(Param.defaultTo("testcfg", "testval")).asReadOnly().apply(Map.of());
ParsedOp pc = new ParsedOp(opTemplate, nbcfg.getMap(), List.of(), getParent());
return pc;
}
@Test
public void testFieldDelegationFromDynamicToStaticToConfig() {
final NBConfiguration cfg = ConfigModel.of(ParsedOpTest.class)
.add(Param.defaultTo("puppy", "dog"))
.add(Param.required("surname", String.class))
.add(Param.defaultTo("puppy", "dog")).add(Param.required("surname", String.class))
.asReadOnly().apply(Map.of("surname", "yes"));
final String opt = """
@@ -80,10 +68,11 @@ public class ParsedOpTest {
params:
ps1: "param-one"
""";
final OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null);
assertThat(stmtsDocs.getOps().matching("",true).size()).isEqualTo(1);
final OpTemplate opTemplate = stmtsDocs.getOps().matching("",true).get(0);
final ParsedOp parsedOp = new ParsedOp(opTemplate, cfg, List.of(), getParent());
final OpsDocList stmtsDocs = OpsLoader.loadString(
opt, OpTemplateFormat.yaml, cfg.getMap(), null);
assertThat(stmtsDocs.getOps().matching("", true).size()).isEqualTo(1);
final OpTemplate opTemplate = stmtsDocs.getOps().matching("", true).get(0);
final ParsedOp parsedOp = new ParsedOp(opTemplate, cfg.getMap(), List.of(), getParent());
assertThat(parsedOp.getAsFunctionOr("d1", "invalid").apply(1L)).isEqualTo("one");
assertThat(parsedOp.getAsFunctionOr("s1", "invalid").apply(1L)).isEqualTo("static-one");
@@ -104,31 +93,20 @@ public class ParsedOpTest {
final ParsedOp parsedOp = new ParsedOp(
new OpData().applyFields(Map.of(
"op", Map.of(
"field1-literal", "literalvalue1",
"field2-object", "{{NumberNameToString()}}",
"field3-template", "pre-{dyna1}-post",
"field4-map-template", Map.of(
"subfield1-object", "{{Identity(); ToString()}}"
), "field5-map-literal", Map.of(
"subfield2-literal", "LiteralValue"
)
),
"bindings", Map.of(
"dyna1", "NumberNameToString()"
))
),
ConfigModel.of(ParsedOpTest.class)
.add(Param.defaultTo("testcfg", "testval"))
.asReadOnly()
.apply(Map.of()),
List.of(),
getParent()
);
"field1-literal", "literalvalue1", "field2-object", "{{NumberNameToString()}}",
"field3-template", "pre-{dyna1}-post", "field4-map-template",
Map.of("subfield1-object", "{{Identity(); ToString()}}"), "field5-map-literal",
Map.of("subfield2-literal", "LiteralValue")), "bindings",
Map.of("dyna1", "NumberNameToString()"))),
ConfigModel.of(ParsedOpTest.class).add(Param.defaultTo("testcfg", "testval"))
.asReadOnly().apply(Map.of()).getMap(), List.of(), getParent());
final LongFunction<? extends String> f1 = parsedOp.getAsRequiredFunction("field1-literal");
final LongFunction<? extends String> f2 = parsedOp.getAsRequiredFunction("field2-object");
final LongFunction<? extends String> f3 = parsedOp.getAsRequiredFunction("field3-template");
final LongFunction<? extends Map> f4 = parsedOp.getAsRequiredFunction("field4-map-template", Map.class);
final LongFunction<? extends Map> f5 = parsedOp.getAsRequiredFunction("field5-map-literal", Map.class);
final LongFunction<? extends Map> f4 = parsedOp.getAsRequiredFunction(
"field4-map-template", Map.class);
final LongFunction<? extends Map> f5 = parsedOp.getAsRequiredFunction(
"field5-map-literal", Map.class);
assertThat(f1.apply(1)).isNotNull();
assertThat(f2.apply(2)).isNotNull();
assertThat(f3.apply(3)).isNotNull();
@@ -148,21 +126,25 @@ public class ParsedOpTest {
@Test
public void testNewListBinder() {
final LongFunction<List<Object>> lb = getOp().newListBinder("dyna1", "identity", "dyna2", "identity");
final LongFunction<List<Object>> lb = getOp().newListBinder(
"dyna1", "identity", "dyna2", "identity");
final List<Object> objects = lb.apply(1);
assertThat(objects).isEqualTo(List.of("one", 1L, "one", 1L));
}
@Test
public void testNewMapBinder() {
final LongFunction<Map<String, Object>> mb = getOp().newOrderedMapBinder("dyna1", "identity", "dyna2");
final LongFunction<Map<String, Object>> mb = getOp().newOrderedMapBinder(
"dyna1", "identity", "dyna2");
final Map<String, Object> objects = mb.apply(2);
assertThat(objects).isEqualTo(Map.<String, Object>of("dyna1", "two", "identity", 2L, "dyna2", "two"));
assertThat(objects).isEqualTo(
Map.<String, Object>of("dyna1", "two", "identity", 2L, "dyna2", "two"));
}
@Test
public void testNewAryBinder() {
final LongFunction<Object[]> ab = getOp().newArrayBinder("dyna1", "dyna1", "identity", "identity");
final LongFunction<Object[]> ab = getOp().newArrayBinder(
"dyna1", "dyna1", "identity", "identity");
final Object[] objects = ab.apply(3);
assertThat(objects).isEqualTo(new Object[]{"three", "three", 3L, 3L});
}
@@ -170,45 +152,21 @@ public class ParsedOpTest {
@Test
public void testLayeredListBinder() {
ParsedOp pc = new ParsedOp(
new OpData().applyFields(
Map.of(
"op", Map.of(
"alist", List.of(
List.of(
"item1",
"item2-{dyna1}"
),
Map.of(
"akey", "avalue",
"akey2", "a {dyna1} value2"
)
)
),
"bindings", Map.of(
"dyna1", "NumberNameToString()"
)
)
),
ConfigModel.of(ParsedOpTest.class)
.add(Param.defaultTo("testcfg", "testval"))
.asReadOnly()
.apply(Map.of()),
List.of(),
getParent()
);
new OpData().applyFields(Map.of(
"op", Map.of(
"alist",
List.of(
List.of("item1", "item2-{dyna1}"),
Map.of("akey", "avalue", "akey2", "a {dyna1} value2"))), "bindings",
Map.of("dyna1", "NumberNameToString()"))),
ConfigModel.of(ParsedOpTest.class).add(Param.defaultTo("testcfg", "testval"))
.asReadOnly().apply(Map.of()).getMap(), List.of(), getParent());
Map<String, Object> result = pc.getTemplateMap().apply(1);
assertThat(result).isEqualTo(
Map.of(
"alist", List.of(
List.of("item1", "item2-one"),
Map.of(
"akey", "avalue",
"akey2", "a one value2"
)
)
)
);
assertThat(result).isEqualTo(Map.of(
"alist", List.of(
List.of("item1", "item2-one"),
Map.of("akey", "avalue", "akey2", "a one value2"))));
}