IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

port bugfix for template variables

Browse Source
This commit is contained in:
Jonathan Shook
2022-01-21 16:04:29 -06:00
parent 3c9a9dc3c0
commit 254c54a8f6
15 changed files with 107 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java
View File

@@ -402,13 +402,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
}
break;
case "2":
doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
break;
case "unset":
try {
logger.debug("You can suffix your yaml filename or url with the " +
"format version, such as :1 or :2. Assuming version 2.");
doclist = StatementsLoader.loadPath(null, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(null, yaml_loc, activityDef.getParams(), "activities");
} catch (Exception ignored) {
try {
doclist = getVersion1StmtsDoc(interp, yaml_loc);
@@ -424,7 +424,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
"for the standard format. To force loading version 1 with detailed logging, add" +
" a version qualifier to your yaml filename or url like ':1'");
// retrigger the error again, this time with logging enabled.
doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
}
}
break;

6
driver-cqld3-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java
View File

@@ -403,13 +403,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
}
break;
case "2":
doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
break;
case "unset":
try {
logger.debug("You can suffix your yaml filename or url with the " +
"format version, such as :1 or :2. Assuming version 2.");
doclist = StatementsLoader.loadPath(null, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(null, yaml_loc, activityDef.getParams(), "activities");
} catch (Exception ignored) {
try {
doclist = getVersion1StmtsDoc(interp, yaml_loc);
@@ -425,7 +425,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
"for the standard format. To force loading version 1 with detailed logging, add" +
" a version qualifier to your yaml filename or url like ':1'");
// retrigger the error again, this time with logging enabled.
doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
}
}
break;

3
driver-dsegraph-shaded/src/main/java/com/datastax/ebdrivers/dsegraph/GraphActivity.java
View File

@@ -86,8 +86,7 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
SequencePlanner<ReadyGraphStatementTemplate> planner = new SequencePlanner<>(sequencerType);
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
StrInterpolator interp = new StrInterpolator(activityDef);
StmtsDocList unfiltered = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
StmtsDocList unfiltered = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
// log tag filtering results
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");

18
driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpOpTest.java
View File

@@ -15,7 +15,8 @@ public class ReadyHttpOpTest {
public void testOnelineSpec() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: method=get uri=http://localhost/\n");
" - s1: method=get uri=http://localhost/\n",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@@ -26,7 +27,8 @@ public class ReadyHttpOpTest {
public void testRFCFormMinimal() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: get http://localhost/");
" - s1: get http://localhost/",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@@ -37,7 +39,8 @@ public class ReadyHttpOpTest {
public void testRFCFormVersioned() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: get http://localhost/ HTTP/1.1");
" - s1: get http://localhost/ HTTP/1.1",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@@ -51,7 +54,8 @@ public class ReadyHttpOpTest {
" - s1: |\n" +
" get http://localhost/\n" +
" Content-Type: application/json" +
"");
"",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@@ -65,7 +69,8 @@ public class ReadyHttpOpTest {
" - s1: |\n" +
" get http://localhost/\n" +
" \n" +
" body1");
" body1",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@@ -93,7 +98,8 @@ public class ReadyHttpOpTest {
" query: StaticString('test')\n" +
" version: StaticString('test')\n" +
" header1val: StaticString('test')\n" +
" body: StaticString('test')\n");
" body: StaticString('test')\n",
Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
Map<String, String> parse = HttpFormatParser.parseInline(stmtDef.getStmt().orElseThrow());

9
driver-kafka/src/main/java/com/datastax/ebdrivers/kafkaproducer/KafkaProducerActivity.java
View File

@@ -10,7 +10,6 @@ import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -71,8 +70,12 @@ public class KafkaProducerActivity extends SimpleActivity {
SequencePlanner<KafkaStatement> sequencer = new SequencePlanner<>(sequencerType);
String tagFilter = activityDef.getParams().getOptionalString("tags").orElse("");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, new StrInterpolator(activityDef),
"activities");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(
logger,
yamlLoc,
activityDef.getParams(),
"activities"
);
List<OpTemplate> statements = stmtsDocList.getStmts(tagFilter);
String format = getParams().getOptionalString("format").orElse(null);

9
driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java
View File

@@ -17,7 +17,6 @@ import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.engine.api.util.TagFilter;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import org.apache.logging.log4j.LogManager;
@@ -104,8 +103,12 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
);
SequencePlanner<ReadyMongoStatement> sequencer = new SequencePlanner<>(sequencerType);
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, new StrInterpolator(activityDef),
"activities");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(
logger,
yamlLoc,
activityDef.getParams(),
"activities"
);
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");

3
driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/ReadyMongoStatementTest.java
View File

@@ -4,7 +4,6 @@ import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.virtdata.core.templates.BindPoint;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import org.apache.logging.log4j.LogManager;
@@ -33,7 +32,7 @@ public class ReadyMongoStatementTest {
};
activityDef = ActivityDef.parseActivityDef(String.join(";", params));
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, new StrInterpolator(activityDef), "activities");
stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
}
@Test

4
driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
View File

@@ -32,7 +32,6 @@ import io.nosqlbench.engine.api.activityimpl.ParameterMap;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.api.metrics.ExceptionMeterMetrics;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import io.nosqlbench.virtdata.core.templates.StringBindings;
@@ -70,12 +69,11 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
public StdoutActivity(ActivityDef activityDef) {
super(activityDef);
StrInterpolator interp = new StrInterpolator(activityDef);
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
this.showstmts = activityDef.getParams().getOptionalBoolean("showstatements").orElse(false);
this.fileName = activityDef.getParams().getOptionalString("filename").orElse("stdout");
this.stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
this.stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
}
@Override

4
driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java
View File

@@ -14,7 +14,6 @@ import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
@@ -169,9 +168,8 @@ public class WebDriverActivity extends SimpleActivity {
}
private OpSequence<CommandTemplate> initOpSequenceFromYaml() {
StrInterpolator interp = new StrInterpolator(activityDef);
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
SequencerType sequencerType = getParams()
.getOptionalString("seq")

90
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java
View File

@@ -22,26 +22,46 @@ import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Map;
import java.util.function.Function;
import java.util.Optional;
public class StatementsLoader {
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
private final static Logger logger = LogManager.getLogger(StatementsLoader.class);
public static StmtsDocList loadString(String yamlContent, Map<String,?> params) {
public enum Loader {
original,
generified
}
public static StmtsDocList loadString(String yamlContent) {
RawStmtsLoader loader = new RawStmtsLoader();
StrInterpolator transformer = new StrInterpolator(params);
RawStmtsLoader loader = new RawStmtsLoader(transformer);
RawStmtsDocList rawDocList = loader.loadString(logger, yamlContent);
StmtsDocList layered = new StmtsDocList(rawDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
params.remove(k);
});
return layered;
}
public static StmtsDocList loadStmt(
Logger logger,
String statement,
Map<String,?> params
) {
StrInterpolator transformer = new StrInterpolator(params);
statement = transformer.apply(statement);
RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(statement);
StmtsDocList layered = new StmtsDocList(rawStmtsDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
params.remove(k);
});
return layered;
}
@@ -50,57 +70,27 @@ public class StatementsLoader {
Content<?> content,
Map<String,String> params
) {
StrInterpolator transformer = new StrInterpolator(params);
RawStmtsLoader loader = new RawStmtsLoader(transformer);
RawStmtsDocList rawDocList = loader.loadString(logger, content.get().toString());
StmtsDocList layered = new StmtsDocList(rawDocList);
for (String varname : transformer.checkpointAccesses()) {
params.remove(varname);
}
return layered;
return loadString(content.get().toString(),params);
}
public static StmtsDocList loadContent(
public static StmtsDocList loadPath(
Logger logger,
Content<?> content
) {
RawStmtsLoader loader = new RawStmtsLoader();
RawStmtsDocList rawDocList = loader.loadString(logger, content.get().toString());
StmtsDocList layered = new StmtsDocList(rawDocList);
return layered;
String path,
Map<String,?> params,
String... searchPaths) {
RawStmtsDocList list = null;
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(content,params);
}
// }
public static StmtsDocList loadPath(
Logger logger,
String path,
String... searchPaths) {
RawStmtsDocList list = null;
StrInterpolator transformer = new StrInterpolator();
RawStmtsLoader gloaderImpl = new RawStmtsLoader(transformer);
list = gloaderImpl.loadPath(logger, path, searchPaths);
return new StmtsDocList(list);
}
public static StmtsDocList loadStmt(
Logger logger,
String statement, Function<String,String> transformer) {
String transformed = transformer.apply(statement);
RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(transformed);
return new StmtsDocList(rawStmtsDocList);
}
public static StmtsDocList loadPath(
Logger logger,
String path,
Function<String, String> transformer,
String... searchPaths) {
RawStmtsDocList list = null;
RawStmtsLoader gloaderImpl = new RawStmtsLoader(transformer);
list = gloaderImpl.loadPath(logger, path, searchPaths);
return new StmtsDocList(list);
return loadPath(logger, path, Map.of(), searchPaths);
}
}

20
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java
View File

@@ -19,6 +19,9 @@ package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.util.TagFilter;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.Param;
import java.util.*;
import java.util.stream.Collectors;
@@ -26,6 +29,7 @@ import java.util.stream.Collectors;
public class StmtsDocList implements Iterable<StmtsDoc> {
private final RawStmtsDocList rawStmtsDocList;
private final Map<String,String> templateVariables = new LinkedHashMap<>();
public StmtsDocList(RawStmtsDocList rawStmtsDocList) {
this.rawStmtsDocList = rawStmtsDocList;
@@ -108,4 +112,20 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
return this.getStmtDocs().get(0).getDescription();
}
public Map<String,String> getTemplateVariables() {
return templateVariables;
}
public void addTemplateVariable(String key, String defaultValue) {
this.templateVariables.put(key,defaultValue);
}
public NBConfigModel getConfigModel() {
ConfigModel cfgmodel = ConfigModel.of(StmtsDocList.class);
getTemplateVariables().forEach((k,v) -> {
cfgmodel.add(Param.defaultTo(k,"template parameter found in the yaml workload"));
});
return cfgmodel.asReadOnly();
}
}

4
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@@ -475,10 +475,10 @@ public class SimpleActivity implements Activity, ProgressCapable {
Optional<String> stmt = activityDef.getParams().getOptionalString("op", "stmt", "statement");
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (stmt.isPresent()) {
stmtsDocList = StatementsLoader.loadStmt(logger, stmt.get(), interp);
stmtsDocList = StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams());
workloadSource = "commandline:" + stmt.get();
} else if (op_yaml_loc.isPresent()) {
stmtsDocList = StatementsLoader.loadPath(logger, op_yaml_loc.get(), interp, "activities");
stmtsDocList = StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities");
workloadSource = "yaml:" + op_yaml_loc.get();
}

8
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@@ -144,11 +144,6 @@ public class NBCLIScenarioParser {
undefKeys.forEach(buildingCmd::remove);
if (!buildingCmd.containsKey("workload")) {
// The logic to remove the leading slash was likely used to fix a nuisance bug before,
// although it is clearly not correct as-is. Leaving temporarily for context.
// String relativeWorkloadPathFromRoot = yamlWithNamedScenarios.asPath().toString();
// relativeWorkloadPathFromRoot = relativeWorkloadPathFromRoot.startsWith("/") ?
// relativeWorkloadPathFromRoot.substring(1) : relativeWorkloadPathFromRoot;
buildingCmd.put("workload", "workload=" + workloadName);
}
@@ -176,7 +171,6 @@ public class NBCLIScenarioParser {
logger.debug("rebuilt command: " + String.join(" ", buildingCmd.values()));
buildCmdBuffer.addAll(buildingCmd.values());
}
}
buildCmdBuffer.descendingIterator().forEachRemaining(arglist::addFirst);
@@ -285,7 +279,7 @@ public class NBCLIScenarioParser {
.name(referenced).extension(RawStmtsLoader.YAML_EXTENSIONS)
.one();
StmtsDocList stmts = StatementsLoader.loadContent(logger, content);
StmtsDocList stmts = StatementsLoader.loadContent(logger, content, Map.of());
if (stmts.getStmtDocs().size() == 0) {
logger.warn("Encountered yaml with no docs in '" + referenced + "'");
continue;

2
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/UniformWorkloadSpecificationTest.java
View File

@@ -292,7 +292,7 @@ public class UniformWorkloadSpecificationTest {
}.getType();
List<Map<String, Object>> expectedList = gson.fromJson(json, type);
StmtsDocList stmtsDocs = StatementsLoader.loadString(yaml);
StmtsDocList stmtsDocs = StatementsLoader.loadString(yaml, Map.of());
List<OpTemplate> stmts = stmtsDocs.getStmts();
List<Map<String, Object>> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList());

12
engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java
View File

@@ -7,6 +7,8 @@ import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class CommandTemplateTest {
@@ -14,8 +16,10 @@ public class CommandTemplateTest {
@Test
public void testCommandTemplate() {
StmtsDocList stmtsDocs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: test1=foo test2=bar");
"statements:\n" +
" - s1: test1=foo test2=bar",
Map.of()
);
OpTemplate optpl = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(optpl);
assertThat(ct.isStatic()).isTrue();
@@ -28,7 +32,9 @@ public class CommandTemplateTest {
"statements:\n" +
" - s1: test1=foo test2={bar}\n" +
" bindings:\n" +
" bar: NumberNameToString();\n");
" bar: NumberNameToString();\n",
Map.of()
);
OpTemplate optpl = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(optpl);
String format = gson.toJson(ct);