IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

simplify stmts loader call paths

Browse Source
This commit is contained in:
Jonathan Shook 2022-01-05 15:33:15 -06:00
parent d54e75ce63
commit 0ca86e33f4
7 changed files with 79 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpOpTest.java
View File

@ -15,7 +15,9 @@ public class ReadyHttpOpTest {
public void testOnelineSpec() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: method=get uri=http://localhost/\n");
" - s1: method=get uri=http://localhost/\n",
Map.of()
);
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@ -26,7 +28,9 @@ public class ReadyHttpOpTest {
public void testRFCFormMinimal() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: get http://localhost/");
" - s1: get http://localhost/",
Map.of()
);
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@ -37,7 +41,9 @@ public class ReadyHttpOpTest {
public void testRFCFormVersioned() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: get http://localhost/ HTTP/1.1");
" - s1: get http://localhost/ HTTP/1.1",
Map.of()
);
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@ -51,7 +57,9 @@ public class ReadyHttpOpTest {
" - s1: |\n" +
" get http://localhost/\n" +
" Content-Type: application/json" +
"");
"",
Map.of()
);
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@ -65,7 +73,9 @@ public class ReadyHttpOpTest {
" - s1: |\n" +
" get http://localhost/\n" +
" \n" +
" body1");
" body1",
Map.of()
);
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef);
@ -93,7 +103,9 @@ public class ReadyHttpOpTest {
" query: StaticString('test')\n" +
" version: StaticString('test')\n" +
" header1val: StaticString('test')\n" +
" body: StaticString('test')\n");
" body: StaticString('test')\n",
Map.of()
);
OpTemplate stmtDef = docs.getStmts().get(0);
Map<String, String> parse = HttpFormatParser.parseInline(stmtDef.getStmt().orElseThrow());

4
driver-kafka/src/main/java/com/datastax/ebdrivers/kafkaproducer/KafkaProducerActivity.java
View File

@ -10,7 +10,6 @@ import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -71,8 +70,7 @@ public class KafkaProducerActivity extends SimpleActivity {
SequencePlanner<KafkaStatement> sequencer = new SequencePlanner<>(sequencerType);
String tagFilter = activityDef.getParams().getOptionalString("tags").orElse("");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, new StrInterpolator(activityDef),
"activities");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, activityDef.getParams(), "activities");
List<OpTemplate> statements = stmtsDocList.getStmts(tagFilter);
String format = getParams().getOptionalString("format").orElse(null);

31
driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java
View File

@ -17,7 +17,6 @@ import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.engine.api.util.TagFilter;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import org.apache.logging.log4j.LogManager;
@ -63,12 +62,12 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
// sanity check
yamlLoc = activityDef.getParams().getOptionalString("yaml", "workload")
.orElseThrow(() -> new IllegalArgumentException("yaml is not defined"));
.orElseThrow(() -> new IllegalArgumentException("yaml is not defined"));
connectionString = activityDef.getParams().getOptionalString("connection")
.orElseThrow(() -> new IllegalArgumentException("connection is not defined"));
.orElseThrow(() -> new IllegalArgumentException("connection is not defined"));
// TODO: support multiple databases
databaseName = activityDef.getParams().getOptionalString("database")
.orElseThrow(() -> new IllegalArgumentException("database is not defined"));
.orElseThrow(() -> new IllegalArgumentException("database is not defined"));
}
@Override
@ -82,7 +81,7 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
client = createMongoClient(connectionString);
mongoDatabase = client.getDatabase(databaseName);
showQuery = activityDef.getParams().getOptionalBoolean("showquery")
.orElse(false);
.orElse(false);
bindTimer = ActivityMetrics.timer(activityDef, "bind");
resultTimer = ActivityMetrics.timer(activityDef, "result");
resultSuccessTimer = ActivityMetrics.timer(activityDef, "result-success");
@ -100,12 +99,16 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
OpSequence<ReadyMongoStatement> initOpSequencer() {
SequencerType sequencerType = SequencerType.valueOf(
activityDef.getParams().getOptionalString("seq").orElse("bucket")
activityDef.getParams().getOptionalString("seq").orElse("bucket")
);
SequencePlanner<ReadyMongoStatement> sequencer = new SequencePlanner<>(sequencerType);
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, new StrInterpolator(activityDef),
"activities");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(
logger,
yamlLoc,
activityDef.getParams(),
"activities"
);
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
@ -121,7 +124,7 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
String statement = parsed.getPositionalStatement(Function.identity());
Objects.requireNonNull(statement);
sequencer.addOp(new ReadyMongoStatement(stmt), stmt.getParamOrDefault("ratio",1));
sequencer.addOp(new ReadyMongoStatement(stmt), stmt.getParamOrDefault("ratio", 1));
}
}
@ -130,12 +133,12 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
MongoClient createMongoClient(String connectionString) {
CodecRegistry codecRegistry = fromRegistries(fromCodecs(new UuidCodec(UuidRepresentation.STANDARD)),
MongoClientSettings.getDefaultCodecRegistry());
MongoClientSettings.getDefaultCodecRegistry());
MongoClientSettings settings = MongoClientSettings.builder()
.applyConnectionString(new ConnectionString(connectionString))
.codecRegistry(codecRegistry)
.uuidRepresentation(UuidRepresentation.STANDARD)
.build();
.applyConnectionString(new ConnectionString(connectionString))
.codecRegistry(codecRegistry)
.uuidRepresentation(UuidRepresentation.STANDARD)
.build();
return MongoClients.create(settings);
}

3
driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/ReadyMongoStatementTest.java
View File

@ -4,7 +4,6 @@ import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.virtdata.core.templates.BindPoint;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import org.apache.logging.log4j.LogManager;
@ -33,7 +32,7 @@ public class ReadyMongoStatementTest {
};
activityDef = ActivityDef.parseActivityDef(String.join(";", params));
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, new StrInterpolator(activityDef), "activities");
stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
}
@Test

4
driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
View File

@ -32,7 +32,6 @@ import io.nosqlbench.engine.api.activityimpl.ParameterMap;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.api.metrics.ExceptionMeterMetrics;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import io.nosqlbench.virtdata.core.templates.StringBindings;
@ -70,12 +69,11 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
public StdoutActivity(ActivityDef activityDef) {
super(activityDef);
StrInterpolator interp = new StrInterpolator(activityDef);
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
this.showstmts = activityDef.getParams().getOptionalBoolean("showstatements").orElse(false);
this.fileName = activityDef.getParams().getOptionalString("filename").orElse("stdout");
this.stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
this.stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
}
@Override

90
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java
View File

@ -22,26 +22,46 @@ import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Map;
import java.util.function.Function;
import java.util.Optional;
public class StatementsLoader {
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
private final static Logger logger = LogManager.getLogger(StatementsLoader.class);
public static StmtsDocList loadString(String yamlContent, Map<String,?> params) {
public enum Loader {
original,
generified
}
public static StmtsDocList loadString(String yamlContent) {
RawStmtsLoader loader = new RawStmtsLoader();
StrInterpolator transformer = new StrInterpolator(params);
RawStmtsLoader loader = new RawStmtsLoader(transformer);
RawStmtsDocList rawDocList = loader.loadString(logger, yamlContent);
StmtsDocList layered = new StmtsDocList(rawDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
params.remove(k);
});
return layered;
}
public static StmtsDocList loadStmt(
Logger logger,
String statement,
Map<String,?> params
) {
StrInterpolator transformer = new StrInterpolator(params);
statement = transformer.apply(statement);
RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(statement);
StmtsDocList layered = new StmtsDocList(rawStmtsDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
params.remove(k);
});
return layered;
}
@ -50,57 +70,27 @@ public class StatementsLoader {
Content<?> content,
Map<String,String> params
) {
StrInterpolator transformer = new StrInterpolator(params);
RawStmtsLoader loader = new RawStmtsLoader(transformer);
RawStmtsDocList rawDocList = loader.loadString(logger, content.get().toString());
StmtsDocList layered = new StmtsDocList(rawDocList);
for (String varname : transformer.checkpointAccesses()) {
params.remove(varname);
}
return layered;
return loadString(content.get().toString(),params);
}
public static StmtsDocList loadContent(
public static StmtsDocList loadPath(
Logger logger,
Content<?> content
) {
RawStmtsLoader loader = new RawStmtsLoader();
RawStmtsDocList rawDocList = loader.loadString(logger, content.get().toString());
StmtsDocList layered = new StmtsDocList(rawDocList);
return layered;
String path,
Map<String,?> params,
String... searchPaths) {
RawStmtsDocList list = null;
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(content,params);
}
// }
public static StmtsDocList loadPath(
Logger logger,
String path,
String... searchPaths) {
RawStmtsDocList list = null;
StrInterpolator transformer = new StrInterpolator();
RawStmtsLoader gloaderImpl = new RawStmtsLoader(transformer);
list = gloaderImpl.loadPath(logger, path, searchPaths);
return new StmtsDocList(list);
}
public static StmtsDocList loadStmt(
Logger logger,
String statement, Function<String,String> transformer) {
String transformed = transformer.apply(statement);
RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(transformed);
return new StmtsDocList(rawStmtsDocList);
}
public static StmtsDocList loadPath(
Logger logger,
String path,
Function<String, String> transformer,
String... searchPaths) {
RawStmtsDocList list = null;
RawStmtsLoader gloaderImpl = new RawStmtsLoader(transformer);
list = gloaderImpl.loadPath(logger, path, searchPaths);
return new StmtsDocList(list);
return loadPath(logger, path, Map.of(), searchPaths);
}
}

8
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -145,11 +145,6 @@ public class NBCLIScenarioParser {
undefKeys.forEach(buildingCmd::remove);
if (!buildingCmd.containsKey("workload")) {
// The logic to remove the leading slash was likely used to fix a nuisance bug before,
// although it is clearly not correct as-is. Leaving temporarily for context.
// String relativeWorkloadPathFromRoot = yamlWithNamedScenarios.asPath().toString();
// relativeWorkloadPathFromRoot = relativeWorkloadPathFromRoot.startsWith("/") ?
// relativeWorkloadPathFromRoot.substring(1) : relativeWorkloadPathFromRoot;
buildingCmd.put("workload", "workload=" + workloadName);
}
@ -177,7 +172,6 @@ public class NBCLIScenarioParser {
logger.debug("rebuilt command: " + String.join(" ", buildingCmd.values()));
buildCmdBuffer.addAll(buildingCmd.values());
}
}
buildCmdBuffer.descendingIterator().forEachRemaining(arglist::addFirst);
@ -286,7 +280,7 @@ public class NBCLIScenarioParser {
.name(referenced).extension(RawStmtsLoader.YAML_EXTENSIONS)
.one();
StmtsDocList stmts = StatementsLoader.loadContent(logger, content);
StmtsDocList stmts = StatementsLoader.loadContent(logger, content, Map.of());
if (stmts.getStmtDocs().size() == 0) {
logger.warn("Encountered yaml with no docs in '" + referenced + "'");
continue;