IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

fully port template var fixes from 4.17 branch

Browse Source
This commit is contained in:
Jonathan Shook 2022-01-31 12:47:33 -06:00
parent 5a44b6d06f
commit f4ec05aef3
21 changed files with 279 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java
View File

@ -402,13 +402,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
}
break;
case "2":
doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
break;
case "unset":
try {
logger.debug("You can suffix your yaml filename or url with the " +
"format version, such as :1 or :2. Assuming version 2.");
doclist = StatementsLoader.loadPath(null, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(null, yaml_loc, activityDef.getParams(), "activities");
} catch (Exception ignored) {
try {
doclist = getVersion1StmtsDoc(interp, yaml_loc);
@ -424,7 +424,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
"for the standard format. To force loading version 1 with detailed logging, add" +
" a version qualifier to your yaml filename or url like ':1'");
// retrigger the error again, this time with logging enabled.
doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
}
}
break;

6
driver-cqld3-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java
View File

@ -403,13 +403,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
}
break;
case "2":
doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
break;
case "unset":
try {
logger.debug("You can suffix your yaml filename or url with the " +
"format version, such as :1 or :2. Assuming version 2.");
doclist = StatementsLoader.loadPath(null, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(null, yaml_loc, activityDef.getParams(), "activities");
} catch (Exception ignored) {
try {
doclist = getVersion1StmtsDoc(interp, yaml_loc);
@ -425,7 +425,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
"for the standard format. To force loading version 1 with detailed logging, add" +
" a version qualifier to your yaml filename or url like ':1'");
// retrigger the error again, this time with logging enabled.
doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
}
}
break;

3
driver-dsegraph-shaded/src/main/java/com/datastax/ebdrivers/dsegraph/GraphActivity.java
View File

@ -86,8 +86,7 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
SequencePlanner<ReadyGraphStatementTemplate> planner = new SequencePlanner<>(sequencerType);
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
StrInterpolator interp = new StrInterpolator(activityDef);
StmtsDocList unfiltered = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
StmtsDocList unfiltered = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
// log tag filtering results
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");

18
driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpOpTest.java
View File

@ -15,7 +15,8 @@ public class ReadyHttpOpTest {
public void testOnelineSpec() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: method=get uri=http://localhost/\n");
" - s1: method=get uri=http://localhost/\n",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@ -26,7 +27,8 @@ public class ReadyHttpOpTest {
public void testRFCFormMinimal() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: get http://localhost/");
" - s1: get http://localhost/",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@ -37,7 +39,8 @@ public class ReadyHttpOpTest {
public void testRFCFormVersioned() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: get http://localhost/ HTTP/1.1");
" - s1: get http://localhost/ HTTP/1.1",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@ -51,7 +54,8 @@ public class ReadyHttpOpTest {
" - s1: |\n" +
" get http://localhost/\n" +
" Content-Type: application/json" +
"");
"",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@ -65,7 +69,8 @@ public class ReadyHttpOpTest {
" - s1: |\n" +
" get http://localhost/\n" +
" \n" +
" body1");
" body1",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@ -93,7 +98,8 @@ public class ReadyHttpOpTest {
" query: StaticString('test')\n" +
" version: StaticString('test')\n" +
" header1val: StaticString('test')\n" +
" body: StaticString('test')\n");
" body: StaticString('test')\n",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
Map<String, String> parse = HttpFormatParser.parseInline(stmtDef.getStmt().orElseThrow());

4
driver-kafka/src/main/java/com/datastax/ebdrivers/kafkaproducer/KafkaProducerActivity.java
View File

@ -10,7 +10,6 @@ import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -71,8 +70,7 @@ public class KafkaProducerActivity extends SimpleActivity {
SequencePlanner<KafkaStatement> sequencer = new SequencePlanner<>(sequencerType);
String tagFilter = activityDef.getParams().getOptionalString("tags").orElse("");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, new StrInterpolator(activityDef),
"activities");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, activityDef.getParams(), "activities");
List<OpTemplate> statements = stmtsDocList.getStmts(tagFilter);
String format = getParams().getOptionalString("format").orElse(null);

4
driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java
View File

@ -17,7 +17,6 @@ import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.engine.api.util.TagFilter;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import org.apache.logging.log4j.LogManager;
@ -104,8 +103,7 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
);
SequencePlanner<ReadyMongoStatement> sequencer = new SequencePlanner<>(sequencerType);
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, new StrInterpolator(activityDef),
"activities");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, activityDef.getParams(), "activities");
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");

3
driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/ReadyMongoStatementTest.java
View File

@ -4,7 +4,6 @@ import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.virtdata.core.templates.BindPoint;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import org.apache.logging.log4j.LogManager;
@ -33,7 +32,7 @@ public class ReadyMongoStatementTest {
};
activityDef = ActivityDef.parseActivityDef(String.join(";", params));
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, new StrInterpolator(activityDef), "activities");
stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
}
@Test

2
driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
View File

@ -75,7 +75,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
this.showstmts = activityDef.getParams().getOptionalBoolean("showstatements").orElse(false);
this.fileName = activityDef.getParams().getOptionalString("filename").orElse("stdout");
this.stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
this.stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
}
@Override

2
driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java
View File

@ -171,7 +171,7 @@ public class WebDriverActivity extends SimpleActivity {
private OpSequence<CommandTemplate> initOpSequenceFromYaml() {
StrInterpolator interp = new StrInterpolator(activityDef);
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
SequencerType sequencerType = getParams()
.getOptionalString("seq")

98
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java
View File

@ -22,26 +22,48 @@ import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Map;
import java.util.function.Function;
import java.util.Optional;
public class StatementsLoader {
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
private final static Logger logger = LogManager.getLogger(StatementsLoader.class);
public static StmtsDocList loadString(String yamlContent, Map<String,?> params) {
public enum Loader {
original,
generified
}
public static StmtsDocList loadString(String yamlContent) {
RawStmtsLoader loader = new RawStmtsLoader();
StrInterpolator transformer = new StrInterpolator(params);
RawStmtsLoader loader = new RawStmtsLoader(transformer);
RawStmtsDocList rawDocList = loader.loadString(logger, yamlContent);
StmtsDocList layered = new StmtsDocList(rawDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
if (params.containsKey(k)) {
params.remove(k);
}
});
return layered;
}
public static StmtsDocList loadStmt(
Logger logger,
String statement,
Map<String,?> params
) {
StrInterpolator transformer = new StrInterpolator(params);
statement = transformer.apply(statement);
RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(statement);
StmtsDocList layered = new StmtsDocList(rawStmtsDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
params.remove(k);
});
return layered;
}
@ -50,57 +72,27 @@ public class StatementsLoader {
Content<?> content,
Map<String,String> params
) {
StrInterpolator transformer = new StrInterpolator(params);
RawStmtsLoader loader = new RawStmtsLoader(transformer);
RawStmtsDocList rawDocList = loader.loadString(logger, content.get().toString());
StmtsDocList layered = new StmtsDocList(rawDocList);
for (String varname : transformer.checkpointAccesses()) {
params.remove(varname);
}
return layered;
return loadString(content.get().toString(),params);
}
public static StmtsDocList loadContent(
public static StmtsDocList loadPath(
Logger logger,
Content<?> content
) {
RawStmtsLoader loader = new RawStmtsLoader();
RawStmtsDocList rawDocList = loader.loadString(logger, content.get().toString());
StmtsDocList layered = new StmtsDocList(rawDocList);
return layered;
String path,
Map<String,?> params,
String... searchPaths) {
RawStmtsDocList list = null;
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(content,params);
}
// }
public static StmtsDocList loadPath(
Logger logger,
String path,
String... searchPaths) {
RawStmtsDocList list = null;
StrInterpolator transformer = new StrInterpolator();
RawStmtsLoader gloaderImpl = new RawStmtsLoader(transformer);
list = gloaderImpl.loadPath(logger, path, searchPaths);
return new StmtsDocList(list);
}
public static StmtsDocList loadStmt(
Logger logger,
String statement, Function<String,String> transformer) {
String transformed = transformer.apply(statement);
RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(transformed);
return new StmtsDocList(rawStmtsDocList);
}
public static StmtsDocList loadPath(
Logger logger,
String path,
Function<String, String> transformer,
String... searchPaths) {
RawStmtsDocList list = null;
RawStmtsLoader gloaderImpl = new RawStmtsLoader(transformer);
list = gloaderImpl.loadPath(logger, path, searchPaths);
return new StmtsDocList(list);
Logger logger,
String path,
String... searchPaths) {
return loadPath(logger, path, Map.of(), searchPaths);
}
}

20
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java
View File

@ -19,6 +19,9 @@ package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.util.TagFilter;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.Param;
import java.util.*;
import java.util.stream.Collectors;
@ -26,6 +29,7 @@ import java.util.stream.Collectors;
public class StmtsDocList implements Iterable<StmtsDoc> {
private final RawStmtsDocList rawStmtsDocList;
private final Map<String,String> templateVariables = new LinkedHashMap<>();
public StmtsDocList(RawStmtsDocList rawStmtsDocList) {
this.rawStmtsDocList = rawStmtsDocList;
@ -108,4 +112,20 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
return this.getStmtDocs().get(0).getDescription();
}
public Map<String,String> getTemplateVariables() {
return templateVariables;
}
public void addTemplateVariable(String key, String defaultValue) {
this.templateVariables.put(key,defaultValue);
}
public NBConfigModel getConfigModel() {
ConfigModel cfgmodel = ConfigModel.of(StmtsDocList.class);
getTemplateVariables().forEach((k,v) -> {
cfgmodel.add(Param.defaultTo(k,v,"template parameter found in the yaml workload"));
});
return cfgmodel.asReadOnly();
}
}

4
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -475,10 +475,10 @@ public class SimpleActivity implements Activity, ProgressCapable {
Optional<String> stmt = activityDef.getParams().getOptionalString("op", "stmt", "statement");
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (stmt.isPresent()) {
stmtsDocList = StatementsLoader.loadStmt(logger, stmt.get(), interp);
stmtsDocList = StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams());
workloadSource = "commandline:" + stmt.get();
} else if (op_yaml_loc.isPresent()) {
stmtsDocList = StatementsLoader.loadPath(logger, op_yaml_loc.get(), interp, "activities");
stmtsDocList = StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities");
workloadSource = "yaml:" + op_yaml_loc.get();
}

27
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -60,7 +60,7 @@ public class NBCLIScenarioParser {
// Optional<Path> workloadPathSearch = NBPaths.findOptionalPath(workloadName, "yaml", false, "activities");
// Path workloadPath = workloadPathSearch.orElseThrow();
// Buffer in CLI word from user, but only until the next command
// Buffer in scenario names from CLI, only counting non-options non-parameters and non-reserved words
List<String> scenarioNames = new ArrayList<>();
while (arglist.size() > 0
&& !arglist.peekFirst().contains("=")
@ -73,21 +73,21 @@ public class NBCLIScenarioParser {
}
// Parse CLI command into keyed parameters, in order
LinkedHashMap<String, String> userParams = new LinkedHashMap<>();
LinkedHashMap<String, String> userProvidedParams = new LinkedHashMap<>();
while (arglist.size() > 0
&& arglist.peekFirst().contains("=")
&& !arglist.peekFirst().startsWith("-")) {
String[] arg = arglist.removeFirst().split("=", 2);
arg[0] = Synonyms.canonicalize(arg[0], logger);
if (userParams.containsKey(arg[0])) {
if (userProvidedParams.containsKey(arg[0])) {
throw new BasicError("duplicate occurrence of option on command line: " + arg[0]);
}
userParams.put(arg[0], arg[1]);
userProvidedParams.put(arg[0], arg[1]);
}
// This will buffer the new command before adding it to the main arg list
LinkedList<String> buildCmdBuffer = new LinkedList<>();
StrInterpolator userParamsInterp = new StrInterpolator(userParams);
StrInterpolator userParamsInterp = new StrInterpolator(userProvidedParams);
for (String scenarioName : scenarioNames) {
@ -99,10 +99,9 @@ public class NBCLIScenarioParser {
.name(workloadName)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.first().orElseThrow();
StmtsDocList stmts = StatementsLoader.loadContent(logger, yamlWithNamedScenarios, userParams);
Scenarios scenarios = stmts.getDocScenarios();
// TODO: The yaml needs to be parsed with arguments from each command independently to support template vars
StmtsDocList scenariosYaml = StatementsLoader.loadContent(logger, yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
Scenarios scenarios = scenariosYaml.getDocScenarios();
Map<String, String> namedSteps = scenarios.getNamedScenario(scenarioName);
@ -119,7 +118,7 @@ public class NBCLIScenarioParser {
String cmd = cmdEntry.getValue();
cmd = userParamsInterp.apply(cmd);
LinkedHashMap<String, CmdArg> parsedStep = parseStep(cmd);
LinkedHashMap<String, String> usersCopy = new LinkedHashMap<>(userParams);
LinkedHashMap<String, String> usersCopy = new LinkedHashMap<>(userProvidedParams);
LinkedHashMap<String, String> buildingCmd = new LinkedHashMap<>();
// consume each of the parameters from the steps to produce a composited command
@ -145,11 +144,6 @@ public class NBCLIScenarioParser {
undefKeys.forEach(buildingCmd::remove);
if (!buildingCmd.containsKey("workload")) {
// The logic to remove the leading slash was likely used to fix a nuisance bug before,
// although it is clearly not correct as-is. Leaving temporarily for context.
// String relativeWorkloadPathFromRoot = yamlWithNamedScenarios.asPath().toString();
// relativeWorkloadPathFromRoot = relativeWorkloadPathFromRoot.startsWith("/") ?
// relativeWorkloadPathFromRoot.substring(1) : relativeWorkloadPathFromRoot;
buildingCmd.put("workload", "workload=" + workloadName);
}
@ -177,7 +171,6 @@ public class NBCLIScenarioParser {
logger.debug("rebuilt command: " + String.join(" ", buildingCmd.values()));
buildCmdBuffer.addAll(buildingCmd.values());
}
}
buildCmdBuffer.descendingIterator().forEachRemaining(arglist::addFirst);
@ -286,7 +279,7 @@ public class NBCLIScenarioParser {
.name(referenced).extension(RawStmtsLoader.YAML_EXTENSIONS)
.one();
StmtsDocList stmts = StatementsLoader.loadContent(logger, content);
StmtsDocList stmts = StatementsLoader.loadContent(logger, content, Map.of());
if (stmts.getStmtDocs().size() == 0) {
logger.warn("Encountered yaml with no docs in '" + referenced + "'");
continue;

47
engine-api/src/main/java/io/nosqlbench/engine/api/templating/StrInterpolator.java
View File

@ -18,6 +18,7 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.errors.OpConfigError;
import org.apache.commons.text.StrLookup;
import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
@ -48,7 +49,7 @@ public class StrInterpolator implements Function<String, String> {
.forEach(multimap::add);
}
public StrInterpolator(Map<String, String> basicMap) {
public StrInterpolator(Map<String, ?> basicMap) {
multimap.add(basicMap);
}
@ -67,7 +68,7 @@ public class StrInterpolator implements Function<String, String> {
return after;
}
public Set<String> checkpointAccesses() {
public Map<String,String> checkpointAccesses() {
return multimap.checkpointAccesses();
}
@ -79,40 +80,52 @@ public class StrInterpolator implements Function<String, String> {
public static class MultiMap extends StrLookup<String> {
private final List<Map<String, String>> maps = new ArrayList<>();
private final List<Map<String, ?>> maps = new ArrayList<>();
private final String warnPrefix = "UNSET";
private final Set<String> accesses = new HashSet<>();
private final Map<String,String> accesses = new LinkedHashMap<>();
private final Map<String,String> extractedDefaults = new LinkedHashMap<>();
public void add(Map<String, String> addedMap) {
public void add(Map<String, ?> addedMap) {
maps.add(addedMap);
}
@Override
public String lookup(String key) {
String defval = null;
String value = null;
String[] parts = key.split("[:,]", 2);
if (parts.length == 2) {
key = parts[0];
defval = parts[1];
}
accesses.add(key);
for (Map<String, String> map : maps) {
String val = map.get(key);
if (val != null) {
return val;
value = parts[1];
if (!extractedDefaults.containsKey(key)) {
extractedDefaults.put(key,value);
}
}
String value = (defval != null) ? defval : warnPrefix + ":" + key;
for (Map<String, ?> map : maps) {
Object val = map.get(key);
if (val != null) {
value = val.toString();
break;
}
}
value = (value==null? extractedDefaults.get(key) : value);
value = (value != null) ? value : warnPrefix + ":" + key;
if (accesses.containsKey(key) && !accesses.get(key).equals(value)) {
throw new OpConfigError("A templated variable '" + key + "' was found with multiple default values: '" + accesses.get(key) + ", and " + value +". This is not allowed." +
" Template variables must resolve to a single value.");
}
accesses.put(key,value);
logger.debug("Template parameter '" + key + "' applied as '" + value + "'");
return value;
}
public Set<String> checkpointAccesses() {
HashSet<String> accesses = new HashSet<>(this.accesses);
public Map<String,String> checkpointAccesses() {
LinkedHashMap<String,String> accesses = new LinkedHashMap<>(this.accesses);
logger.info("removed template params after applying:" + accesses);
this.accesses.clear();
return accesses;

2
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/UniformWorkloadSpecificationTest.java
View File

@ -292,7 +292,7 @@ public class UniformWorkloadSpecificationTest {
}.getType();
List<Map<String, Object>> expectedList = gson.fromJson(json, type);
StmtsDocList stmtsDocs = StatementsLoader.loadString(yaml);
StmtsDocList stmtsDocs = StatementsLoader.loadString(yaml,Map.of());
List<OpTemplate> stmts = stmtsDocs.getStmts();
List<Map<String, Object>> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList());

7
engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java
View File

@ -7,6 +7,8 @@ import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class CommandTemplateTest {
@ -15,7 +17,7 @@ public class CommandTemplateTest {
public void testCommandTemplate() {
StmtsDocList stmtsDocs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: test1=foo test2=bar");
" - s1: test1=foo test2=bar", Map.of());
OpTemplate optpl = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(optpl);
assertThat(ct.isStatic()).isTrue();
@ -28,7 +30,8 @@ public class CommandTemplateTest {
"statements:\n" +
" - s1: test1=foo test2={bar}\n" +
" bindings:\n" +
" bar: NumberNameToString();\n");
" bar: NumberNameToString();\n",
Map.of());
OpTemplate optpl = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(optpl);
String format = gson.toJson(ct);

43
engine-api/src/test/java/io/nosqlbench/engine/api/templating/StrInterpolatorTest.java
View File

@ -27,45 +27,48 @@ public class StrInterpolatorTest {
private static final List<Map<String, String>> abcd = new ArrayList<Map<String, String>>() {{
add(
new HashMap<String,String>() {{
put("akey", "aval1");
put("bkey", "bval1");
put("ckey", "cval1");
}}
new HashMap<>() {{
put("akey", "aval1");
put("bkey", "bval1");
put("ckey", "cval1");
}}
);
add(
new HashMap<String,String>() {
{
put("akey", "aval2");
put("bkey", "bval2");
}
new HashMap<>() {
{
put("akey", "aval2");
put("bkey", "bval2");
}
}
);
add(
new HashMap<String,String>() {
{
put("json-a-b", "'a': 'b'");
}
new HashMap<>() {
{
put("json-a-b", "'a': 'b'");
}
}
);
}};
private static final StrInterpolator interp = new StrInterpolator(abcd);
@Test
public void shouldReturnIdentity() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("A");
assertThat(a).isEqualTo("A");
}
@Test
public void shouldMatchSimpleSubst() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("<<akey>>");
assertThat(a).isEqualTo("aval1");
}
@Test
public void shouldMatchAlternateSubst() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("TEMPLATE(akey)");
assertThat(a).isEqualTo("aval1");
String b = interp.apply("TEMPLATE(nokeymatches,value2)");
@ -74,44 +77,52 @@ public class StrInterpolatorTest {
@Test
public void shouldReturnWarningWhenUnmatched() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("<<nokeymatchesthis>>");
assertThat(a).isEqualTo("UNSET:nokeymatchesthis");
}
@Test
public void shouldReturnDefaultWhenNotOverridden() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("<<nokeymatchesthis:butithasadefault>>");
assertThat(a).isEqualTo("butithasadefault");
}
@Test
public void shouldOverrideDefault() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("<<bkey:bkeydefault>>");
assertThat(a).isEqualTo("bval1");
}
@Test
public void shouldWorkWithOddCharacters() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("<<unchanged:{'parm1':'val1',parm2:val2, parm3: 'val3'}>>");
assertThat(a).isEqualTo("{'parm1':'val1',parm2:val2, parm3: 'val3'}");
}
@Test
public void shouldWorkWithAllQuotes() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("<<Token:'Key': 'Value'>>");
assertThat(a).isEqualTo("'Key': 'Value'");
}
@Test
public void shouldWorkWithAllQuotesOverride() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("<<Token:'Key': 'Value'>>");
assertThat(a).isEqualTo("'Key': 'Value'");
String b = interp.apply("<<json-a-b:'Key': 'Value'>>");
StrInterpolator interp2 = new StrInterpolator(abcd);
String b = interp2.apply("<<json-a-b:'Key': 'Value'>>");
assertThat(b).isEqualTo("'a': 'b'");
}
@Test
public void shouldWorkWithMultipleGroups() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("<<Token:'Key': 'Value'>>.<<Token2:'Stuff'>>");
assertThat(a).isEqualTo("'Key': 'Value'.'Stuff'");
}

45
engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java Normal file
View File

@ -0,0 +1,45 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
public class NBCLIScenarioParserTemplateVarTest {
@Test
public void testMultipleOccurencesOfSameTemplateVar() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example-scenarios-templatevars" });
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0);
System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n");
StmtsDocList workload2 = StatementsLoader.loadPath(null, cmds.get(1).getArg("workload"),cmds.get(1).getParams());
OpTemplate optpl2 = workload2.getStmts().get(0);
System.out.println("op from cmd2:"+optpl2);
assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n");
}
@Test
public void testThatCLIOverridesWorkForTemplateVars() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example-scenarios-templatevars", "tvar1=overridden" });
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0);
System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n");
}
}

18
engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTest.java
View File

@ -55,13 +55,13 @@ public class NBCLIScenarioParserTest {
@Test
public void testThatVerboseFinalParameterThrowsError() {
assertThatExceptionOfType(BasicError.class)
.isThrownBy(() -> new NBCLIOptions(new String[]{ "scenario-test", "yaml=canttouchthis"}));
.isThrownBy(() -> new NBCLIOptions(new String[]{ "scenario-test", "yaml=canttouchthis"}));
}
@Test
public void testThatMissingScenarioNameThrowsError() {
assertThatExceptionOfType(BasicError.class)
.isThrownBy(() -> new NBCLIOptions(new String[]{ "scenario-test", "missing-scenario"}));
.isThrownBy(() -> new NBCLIOptions(new String[]{ "scenario-test", "missing-scenario"}));
}
@Test
@ -82,14 +82,17 @@ public class NBCLIScenarioParserTest {
}
@Test
public void testThatTemplateParamsAreExpandedAndRemovedOverride() {
public void testThatTemplateParamsAreExpandedAndNotRemovedOverride() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "scenario-test", "template-test", "cycles-test=20"});
List<Cmd> cmds = opts.getCommands();
assertThat(cmds.size()).isEqualTo(1);
assertThat(cmds.get(0).getArg("driver")).isEqualTo("stdout");
assertThat(cmds.get(0).getArg("cycles")).isEqualTo("20");
assertThat(cmds.get(0).getArg("cycles-test")).isNull();
assertThat(cmds.get(0).getArg("workload")).isEqualTo("scenario-test");
assertThat(cmds.get(0).getParams()).isEqualTo(Map.of(
"alias","scenariotest_templatetest_withtemplate",
"cycles","20",
"cycles-test","20",
"driver","stdout",
"workload","scenario-test"
));
}
@Test
@ -99,6 +102,7 @@ public class NBCLIScenarioParserTest {
assertThat(cmds.size()).isEqualTo(1);
assertThat(cmds.get(0).getParams()).isEqualTo(Map.of(
"alias","scenariotest_schemaonly_000",
"cycles-test","20",
"driver","stdout",
"tags","phase:schema",
"workload","scenario-test"

10
engine-cli/src/test/resources/local/example-scenarios-templatevars.yaml Normal file
View File

@ -0,0 +1,10 @@
# example-scenarios.yaml
scenarios:
default:
- run cycles=3 alias=A driver=stdout tvar1=replaced
- run cycles=5 alias=B driver=stdout
bindings:
cycle: Identity()
name: NumberNameToCycle()
statements:
- cycle: "cycle {cycle} TEMPLATE(tvar1,def1) TEMPLATE(tvar1)\n"

54
nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/Param.java
View File

@ -14,7 +14,7 @@ public class Param<T> {
private final List<String> names;
public final Class<? extends T> type;
public String description;
private String description;
private final T defaultValue;
public boolean required;
private Pattern regex;
@ -35,6 +35,7 @@ public class Param<T> {
/**
* Declare an optional String parameter with the given name.
*
* @param name the name of the parameter
*/
public static Param<String> optional(String name) {
@ -57,13 +58,28 @@ public class Param<T> {
* When users provide more than one of these in configuration data, it is considered an error.
*
* @param names one or more names that the parameter can be specified with.
* @param type The type of value that the provided configuration value must be returnable as (assignable to)
* @param <V> Generic type for inference.
* @param type The type of value that the provided configuration value must be returnable as (assignable to)
* @param <V> Generic type for inference.
*/
public static <V> Param<V> optional(List<String> names, Class<V> type) {
return new Param<V>(names, type, null, false, null);
}
/**
* Declare an optional parameter specified by any of the names which must be assignable to
* (returnable as) the specified type.
* When users provide more than one of these in configuration data, it is considered an error.
*
* @param names one or more names that the parameter can be specified with.
* @param type The type of value that the provided configuration value must be returnable as (assignable to)
* @param description A description of what this parameter is
* @param <V> Generic type for inference.
*/
public static <V> Param<V> optional(List<String> names, Class<V> type, String description) {
return new Param<V>(names, type, description, false, null);
}
/**
* Declare an optional parameter for the given name which must be assignable to
* (returnable as) the specified type.
@ -71,15 +87,30 @@ public class Param<T> {
*
* @param name the name of the parameter
* @param type The type of value that the provided configuration value must be returnable as (assignable to)
* @param <V> Generic type for inference.
* @param <V> Generic type for inference.
*/
public static <V> Param<V> optional(String name, Class<V> type) {
return new Param<V>(List.of(name), type, null, false, null);
}
/**
* Declare an optional parameter for the given name which must be assignable to
* (returnable as) the specified type.
* When users provide more than one of these in configuration data, it is considered an error.
*
* @param name the name of the parameter
* @param type The type of value that the provided configuration value must be returnable as (assignable to)
* @param description A description of what this parameter is
* @param <V> Generic type for inference.
*/
public static <V> Param<V> optional(String name, Class<V> type, String description) {
return new Param<V>(List.of(name), type, description, false, null);
}
/**
* Parameters which are given a default value are automatically marked as required, as the default
* value allows them to be accessed as such.
*
* @param name
* @param defaultValue
* @param <V>
@ -92,6 +123,20 @@ public class Param<T> {
/**
* Parameters which are given a default value are automatically marked as required, as the default
* value allows them to be accessed as such.
*
* @param name
* @param defaultValue
* @param <V>
* @return
*/
public static <V> Param<V> defaultTo(String name, V defaultValue, String description) {
return new Param<V>(List.of(name), (Class<V>) defaultValue.getClass(), description, true, defaultValue);
}
/**
* Parameters which are given a default value are automatically marked as required, as the default
* value allows them to be accessed as such.
*
* @param names
* @param defaultValue
* @param <V>
@ -110,7 +155,6 @@ public class Param<T> {
}
@Override
public String toString() {
return "Element{" +