IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

OpTemplate interface, generalized sequencer init, first working test

Browse Source
This commit is contained in:
Jonathan Shook 2020-06-29 18:39:04 -05:00
parent fd2da9972b
commit adb49d0737
30 changed files with 460 additions and 278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java
View File

@ -30,6 +30,7 @@ import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
@ -61,7 +62,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
private final ExceptionHistoMetrics exceptionHistoMetrics;
private final ActivityDef activityDef;
private final Map<String, Writer> namedWriters = new HashMap<>();
protected List<StmtDef> stmts;
protected List<OpTemplate> stmts;
Timer retryDelayTimer;
Timer bindTimer;
Timer executeTimer;
@ -157,7 +158,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
throw new RuntimeException("There were no unfiltered statements found for this activity.");
}
for (StmtDef stmtDef : stmts) {
for (OpTemplate stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed().orError();
boolean prepared = stmtDef.getParamOrDefault("prepared", true);

6
driver-cqld4/pom.xml
View File

@ -109,6 +109,12 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>3.12.124-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>

11
driver-cqld4/src/main/java/io/nosqlbench/activitytype/cqld4/core/CqlActivity.java
View File

@ -33,6 +33,7 @@ import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
@ -62,7 +63,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
private final ExceptionHistoMetrics exceptionHistoMetrics;
private final ActivityDef activityDef;
private final Map<String, Writer> namedWriters = new HashMap<>();
protected List<StmtDef> stmts;
protected List<OpTemplate> stmts;
Timer retryDelayTimer;
Timer bindTimer;
Timer executeTimer;
@ -160,7 +161,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
throw new RuntimeException("There were no unfiltered statements found for this activity.");
}
for (StmtDef stmtDef : stmts) {
for (OpTemplate stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed().orError();
@ -314,13 +315,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
"http://docs.engineblock.io/user-guide/standard_yaml/ for more details.");
break;
case "2":
doclist = StatementsLoader.load(logger, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
break;
case "unset":
try {
logger.debug("You can suffix your yaml filename or url with the " +
"format version, such as :1 or :2. Assuming version 2.");
doclist = StatementsLoader.load(null, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(null, yaml_loc, interp, "activities");
} catch (Exception ignored) {
try {
doclist = getVersion1StmtsDoc(interp, yaml_loc);
@ -336,7 +337,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
"for the standard format. To force loading version 1 with detailed logging, add" +
" a version qualifier to your yaml filename or url like ':1'");
// retrigger the error again, this time with logging enabled.
doclist = StatementsLoader.load(logger, yaml_loc, interp, "activities");
doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
}
}
break;

61
driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpFormatParser.java Normal file
View File

@ -0,0 +1,61 @@
package io.nosqlbench.activitytype.cmds;
import io.nosqlbench.nb.api.errors.BasicError;
import java.util.HashMap;
import java.util.Map;
public class HttpFormatParser {
public static Map<String,String> parse(String command) {
if (command==null) {
return null;
}
// Only attempt to match this if it starts with a bare word
if (!command.matches("(?m)(?s)^\\{?[a-zA-Z]+}? .+")) {
return null;
}
Map<String,String> props = new HashMap<>();
String[] headAndBody = command.trim().split("\n\n",2);
if (headAndBody.length==2) {
props.put("body",headAndBody[1]);
}
String[] methodAndHeaders = headAndBody[0].split("\n",2);
if (methodAndHeaders.length>1) {
for (String header : methodAndHeaders[1].split("\n")) {
String[] headerNameAndVal = header.split(": *", 2);
if (headerNameAndVal.length!=2) {
throw new BasicError("Headers must be in 'Name: value form");
}
if (!headerNameAndVal[0].substring(0,1).toUpperCase().equals(headerNameAndVal[0].substring(0,1))) {
throw new BasicError("Headers must be capitalized to avoid ambiguity with other request parameters:'" + headerNameAndVal[0]);
}
props.put(headerNameAndVal[0],headerNameAndVal[1]);
}
}
String[] methodLine = methodAndHeaders[0].split(" ",3);
if (methodLine.length<2) {
throw new BasicError("Request template must have at least a method and a uri: " + methodAndHeaders[0]);
}
props.put("method",methodLine[0]);
props.put("uri",methodLine[1]);
if (methodLine.length==3) {
String actualVersion = methodLine[2];
String symbolicVersion = actualVersion
.replaceAll("/1.1","_1_1")
.replaceAll("/2.0","_2")
.replaceAll("/2","_2");
props.put("version", symbolicVersion);
}
return props;
}
}

30
driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpRequestSetter.java
View File

@ -1,30 +0,0 @@
package io.nosqlbench.activitytype.cmds;
import io.nosqlbench.engine.api.templating.EnumSetter;
import java.net.http.HttpRequest;
public class HttpRequestSetter implements EnumSetter<ReadyHttpRequest.FieldType,HttpRequest.Builder> {
@Override
public HttpRequest.Builder setField(
HttpRequest.Builder target,
ReadyHttpRequest.FieldType field,
Object... value) {
switch (field) {
case method:
return target.method((String)value[0], (HttpRequest.BodyPublisher) value[1]);
case host:
case path:
case query:
case header:
case version:
return target;
default:
throw new RuntimeException("field type was not set correctly:" + field);
}
}
}

112
driver-http/src/main/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequest.java
View File

@ -1,82 +1,29 @@
package io.nosqlbench.activitytype.cmds;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityimpl.motor.ParamsParser;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import io.nosqlbench.virtdata.core.templates.StringBindings;
import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.function.LongFunction;
import java.util.stream.Collectors;
public class ReadyHttpRequest implements LongFunction<HttpRequest> {
private final static HttpRequestSetter setter = new HttpRequestSetter();
public enum FieldType {
method,
port,
host,
path,
query,
header,
version
}
HttpRequest.Builder builder = HttpRequest.newBuilder();
Map<FieldType, StringBindings> unresolved = new HashMap<>();
private final CommandTemplate propertyTemplate;
// only populated if there is no value which is an actual bindings template
private final HttpRequest cachedRequest;
public ReadyHttpRequest(StmtDef stmtDef) {
CommandTemplate cmdt = new CommandTemplate(stmtDef, false);
ParsedStmt parsed = stmtDef.getParsed();
public ReadyHttpRequest(OpTemplate stmtDef) {
propertyTemplate = new CommandTemplate(stmtDef, HttpFormatParser::parse);
Set<String> namedProperties = propertyTemplate.getPropertyNames();
Map<String, String> reqParams = new HashMap<>();
String stmt = parsed.getStmt();
if (stmt != null) {
Map<String, String> parsedparams = ParamsParser.parse(stmt, false);
reqParams.putAll(parsedparams);
}
for (String paramsKey : stmtDef.getParams().keySet()) {
if (reqParams.containsKey(paramsKey)) {
throw new RuntimeException("request parameter '" + paramsKey + "' used again in params block. Choose one.");
}
}
reqParams.putAll(stmtDef.getParamsAsValueType(String.class));
for (String cfgname : reqParams.keySet()) {
FieldType cfgfield;
try {
cfgfield = FieldType.valueOf(cfgname);
} catch (IllegalArgumentException iae) {
throw new BasicError("You can't configure a request with '" + cfgname + "'." +
" Valid properties are " + Arrays.stream(FieldType.values()).map(String::valueOf).collect(Collectors.joining(",")));
}
String value = reqParams.get(cfgname);
ParsedTemplate tpl = new ParsedTemplate(value, stmtDef.getBindings());
if (tpl.getBindPoints().size() == 0) {
builder = setter.setField(builder, cfgfield, value);
} else {
BindingsTemplate bindingsTemplate = new BindingsTemplate(tpl.getBindPoints());
StringBindingsTemplate stringBindingsTemplate = new StringBindingsTemplate(value, bindingsTemplate);
StringBindings stringBindings = stringBindingsTemplate.resolve();
unresolved.put(cfgfield, stringBindings);
}
}
if (unresolved.size() == 0) {
cachedRequest = builder.build();
if (propertyTemplate.isStatic()) {
cachedRequest = apply(0);
} else {
cachedRequest = null;
}
@ -84,16 +31,45 @@ public class ReadyHttpRequest implements LongFunction<HttpRequest> {
@Override
public HttpRequest apply(long value) {
// If the request is invariant, simply return it, since it is thread-safe
if (this.cachedRequest != null) {
return this.cachedRequest;
}
HttpRequest.Builder newRq = builder.copy();
for (Map.Entry<FieldType, StringBindings> toset : unresolved.entrySet()) {
String setValue = toset.getValue().bind(value);
newRq = setter.setField(newRq, toset.getKey(), setValue);
Map<String, String> cmd = propertyTemplate.getCommand(value);
HttpRequest.Builder builder = HttpRequest.newBuilder();
HttpRequest.BodyPublisher bodyPublisher = cmd.containsKey("body") ?
HttpRequest.BodyPublishers.ofString(cmd.remove("body"))
: HttpRequest.BodyPublishers.noBody();
String method = cmd.containsKey("method") ? cmd.remove("method") : "GET";
builder.method(method, bodyPublisher);
if (cmd.containsKey("version")) {
HttpClient.Version version = HttpClient.Version.valueOf(cmd.remove("version"));
builder.version(version);
}
HttpRequest request = newRq.build();
if (cmd.containsKey("uri")) {
URI uri = URI.create(cmd.remove("uri"));
builder.uri(uri);
}
for (String header : cmd.keySet()) {
if (header.charAt(0) >= 'A' && header.charAt(0) <= 'Z') {
builder.header(header, cmd.get(header));
} else {
throw new BasicError("HTTP request parameter '" + header + "' was not recognized as a basic request parameter, and it is not capitalized to indicate that it is a header.");
}
}
HttpRequest request = builder.build();
return request;
}
}

72
driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpAction.java
View File

@ -1,6 +1,7 @@
package io.nosqlbench.activitytype.http;
import com.codahale.metrics.Timer;
import io.nosqlbench.activitytype.cmds.ReadyHttpRequest;
import io.nosqlbench.engine.api.activityapi.core.SyncAction;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
@ -32,7 +33,7 @@ public class HttpAction implements SyncAction {
private int maxTries = 1;
private boolean showstmts;
private OpSequence<CommandTemplate> sequencer;
private OpSequence<ReadyHttpRequest> sequencer;
private HttpClient client;
private HttpResponse.BodyHandler<String> bodyreader = HttpResponse.BodyHandlers.ofString();
private long timeoutMillis;
@ -70,35 +71,8 @@ public class HttpAction implements SyncAction {
String ok;
try (Timer.Context bindTime = httpActivity.bindTimer.time()) {
CommandTemplate commandTemplate = httpActivity.getOpSequence().get(cycleValue);
Map<String, String> cmdMap = commandTemplate.getCommand(cycleValue);
String host = httpActivity.getHosts()[(int) cycleValue % httpActivity.getHosts().length];
String[] command = cmdMap.get("command").split(" ", 3); // RFC 2616 Section 5.1.2
ok = cmdMap.remove("ok");
// Base request
String method = command[0].toUpperCase();
String baseuri = command[1].trim();
URI uri = URI.create(baseuri);
HttpRequest.Builder builder = HttpRequest.newBuilder(uri);
HttpRequest.BodyPublisher bodysource = bodySourceFrom(cmdMap);
builder = builder.method(method, bodysource);
if (command.length == 3) {
HttpClient.Version version = HttpClient.Version.valueOf(command[2]);
builder.version(version);
}
// All known command options must be processed by this point, so the rest are headers
for (String mustBeAHeader : cmdMap.keySet()) {
builder.header(mustBeAHeader, cmdMap.get(mustBeAHeader));
}
request = builder.build();
ReadyHttpRequest readyHttpRequest = httpActivity.getOpSequence().get(cycleValue);
request =readyHttpRequest.apply(cycleValue);
} catch (Exception e) {
throw new RuntimeException("while binding request in cycle " + cycleValue + ": " + e.getMessage(),e);
}
@ -121,26 +95,26 @@ public class HttpAction implements SyncAction {
throw new RuntimeException("while waiting for response in cycle " + cycleValue + ":" + e.getMessage(), e);
}
if (ok == null) {
if (response.statusCode() != 200) {
throw new ResponseError("Result had status code " +
response.statusCode() + ", but 'ok' was not set for this statement," +
"so it is considered an error.");
}
} else {
String[] oks = ok.split(",");
for (String ok_condition : oks) {
if (ok_condition.charAt(0)>='0' && ok_condition.charAt(0)<='9') {
int matching_status = Integer.parseInt(ok_condition);
} else {
Pattern successRegex = Pattern.compile(ok);
}
}
// Matcher matcher = successRegex.matcher(String.valueOf(response.statusCode()));
// if (!matcher.matches()) {
// throw new BasicError("status code " + response.statusCode() + " did not match " + success);
// if (ok == null) {
// if (response.statusCode() != 200) {
// throw new ResponseError("Result had status code " +
// response.statusCode() + ", but 'ok' was not set for this statement," +
// "so it is considered an error.");
// }
}
// } else {
// String[] oks = ok.split(",");
// for (String ok_condition : oks) {
// if (ok_condition.charAt(0)>='0' && ok_condition.charAt(0)<='9') {
// int matching_status = Integer.parseInt(ok_condition);
// } else {
// Pattern successRegex = Pattern.compile(ok);
// }
// }
//// Matcher matcher = successRegex.matcher(String.valueOf(response.statusCode()));
//// if (!matcher.matches()) {
//// throw new BasicError("status code " + response.statusCode() + " did not match " + success);
//// }
// }
}
return 0;
}

16
driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java
View File

@ -1,5 +1,6 @@
package io.nosqlbench.activitytype.http;
import io.nosqlbench.activitytype.cmds.ReadyHttpRequest;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
@ -52,7 +53,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
private String[] hosts;
private int port;
private OpSequence<CommandTemplate> opSequence;
private OpSequence<ReadyHttpRequest> opSequence;
public HttpActivity(ActivityDef activityDef) {
super(activityDef);
@ -65,15 +66,14 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
public void initActivity() {
super.initActivity();
stride = activityDef.getParams().getOptionalInteger("stride").orElse(1);
// stride = activityDef.getParams().getOptionalInteger("stride").orElse(1);
maxTries = activityDef.getParams().getOptionalInteger("maxTries").orElse(1);
showstmnts = activityDef.getParams().getOptionalBoolean("showstmnts").orElse(false);
// showstmnts = activityDef.getParams().getOptionalBoolean("showstmnts").orElse(false);
hosts = activityDef.getParams().getOptionalString("host").orElse("localhost").split(",");
port = activityDef.getParams().getOptionalInteger("port").orElse(80);
// hosts = activityDef.getParams().getOptionalString("host").orElse("localhost").split(",");
// port = activityDef.getParams().getOptionalInteger("port").orElse(80);
this.opSequence = createDefaultOpSequence();
this.opSequence = createOpSequence(ReadyHttpRequest::new);
setDefaultsFromOpSequence(opSequence);
bindTimer = ActivityMetrics.timer(activityDef, "bind");
@ -108,7 +108,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
return port;
}
public OpSequence<CommandTemplate> getOpSequence() {
public OpSequence<ReadyHttpRequest> getOpSequence() {
return opSequence;
}
}

97
driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequestTest.java
View File

@ -1,23 +1,114 @@
package io.nosqlbench.activitytype.cmds;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import org.junit.Test;
import java.net.http.HttpRequest;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class ReadyHttpRequestTest {
@Test
public void testStaticTemplate() {
public void testOnelineSpec() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: method=get\n");
StmtDef stmtDef = docs.getStmts().get(0);
" - s1: method=get uri=http://localhost/\n");
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpRequest readyReq = new ReadyHttpRequest(stmtDef);
HttpRequest staticReq = readyReq.apply(3);
}
@Test
public void testRFCFormMinimal() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: get http://localhost/");
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpRequest readyReq = new ReadyHttpRequest(stmtDef);
HttpRequest staticReq = readyReq.apply(3);
}
@Test
public void testRFCFormVersioned() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: get http://localhost/ HTTP/1.1");
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpRequest readyReq = new ReadyHttpRequest(stmtDef);
HttpRequest staticReq = readyReq.apply(3);
}
@Test
public void testRFCFormHeaders() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: |\n" +
" get http://localhost/\n" +
" Content-Type: application/json" +
"");
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpRequest readyReq = new ReadyHttpRequest(stmtDef);
HttpRequest staticReq = readyReq.apply(3);
}
@Test
public void testRFCFormBody() {
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: |\n" +
" get http://localhost/\n" +
" \n" +
" body1");
OpTemplate stmtDef = docs.getStmts().get(0);
ReadyHttpRequest readyReq = new ReadyHttpRequest(stmtDef);
HttpRequest staticReq = readyReq.apply(3);
}
@Test
public void testRFCAllValuesTemplated() {
// This can not be fully resolved in the unit testing context, but it could be
// in the integrated testing context. It is sufficient to verify parsing here.
StmtsDocList docs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: |\n" +
" {method} {scheme}://{host}/{path}?{query} {version}\n" +
" Header1: {header1val}\n" +
" \n" +
" {body}\n" +
"\n" +
"bindings: \n" +
" method: StaticString('test')\n" +
" scheme: StaticString('test')\n" +
" host: StaticString('test')\n" +
" path: StaticString('test')\n" +
" query: StaticString('test')\n" +
" version: StaticString('test')\n" +
" header1val: StaticString('test')\n" +
" body: StaticString('test')\n");
OpTemplate stmtDef = docs.getStmts().get(0);
Map<String, String> parse = HttpFormatParser.parse(stmtDef.getStmt());
assertThat(parse).containsAllEntriesOf(
Map.of(
"method", "{method}",
"uri", "{scheme}://{host}/{path}?{query}",
"version", "{version}",
"Header1","{header1val}",
"body","{body}\n"
)
);
}
}

2
driver-http/src/test/java/io/nosqlbench/activitytype/http/HttpActivityTypeTest.java
View File

@ -14,7 +14,7 @@ public class HttpActivityTypeTest {
String atname = httpAt.getName();
assertThat(atname.equals("http"));
ActivityDef ad = ActivityDef.parseActivityDef("driver=http; yaml=http-google.yaml; host=google.com; port=80; cycles=1;");
ActivityDef ad = ActivityDef.parseActivityDef("driver=http; yaml=http-google.yaml; port=80; cycles=1;");
HttpActivity httpActivity = httpAt.getActivity(ad);
httpActivity.initActivity();
ActionDispenser actionDispenser = httpAt.getActionDispenser(httpActivity);

7
driver-http/src/test/resources/activities/http-google.yaml
View File

@ -1,6 +1,3 @@
tags:
type: google
params:
requestType: GET
statements:
- /
- s1: |
GET http://google.com/ HTTP/1.1

5
driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java
View File

@ -4,6 +4,7 @@ import java.util.List;
import java.util.Objects;
import java.util.function.Function;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -109,8 +110,8 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
TagFilter tagFilter = new TagFilter(tagfilter);
stmtsDocList.getStmts().stream().map(tagFilter::matchesTaggedResult).forEach(r -> logger.info(r.getLog()));
List<StmtDef> stmts = stmtsDocList.getStmts(tagfilter);
for (StmtDef stmt : stmts) {
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
for (OpTemplate stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
String statement = parsed.getPositionalStatement(Function.identity());
Objects.requireNonNull(statement);

3
driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java
View File

@ -1,6 +1,7 @@
package io.nosqlbench.driver.mongodb;
import com.mongodb.ReadPreference;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
@ -14,7 +15,7 @@ public class ReadyMongoStatement {
private StringBindings bindings;
private ReadPreference readPreference;
public ReadyMongoStatement(StmtDef stmtDef) {
public ReadyMongoStatement(OpTemplate stmtDef) {
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings);

13
driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/ReadyMongoStatementTest.java
View File

@ -4,6 +4,7 @@ import java.util.List;
import java.util.Objects;
import java.util.function.Function;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@ -41,9 +42,9 @@ public class ReadyMongoStatementTest {
public void testResolvePhaseRampup() {
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("phase:rampup");
List<StmtDef> stmts = stmtsDocList.getStmts(tagfilter);
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
assertThat(stmts).hasSize(1);
for (StmtDef stmt : stmts) {
for (OpTemplate stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
assertThat(parsed.getBindPoints()).hasSize(2);
@ -64,9 +65,9 @@ public class ReadyMongoStatementTest {
public void testResolvePhaseMainRead() {
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("phase:main,name:main-find");
List<StmtDef> stmts = stmtsDocList.getStmts(tagfilter);
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
assertThat(stmts).hasSize(1);
for (StmtDef stmt : stmts) {
for (OpTemplate stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
assertThat(parsed.getBindPoints()).hasSize(1);
@ -86,9 +87,9 @@ public class ReadyMongoStatementTest {
public void testResolvePhaseMainWrite() {
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("phase:main,name:main-insert");
List<StmtDef> stmts = stmtsDocList.getStmts(tagfilter);
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
assertThat(stmts).hasSize(1);
for (StmtDef stmt : stmts) {
for (OpTemplate stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
assertThat(parsed.getBindPoints()).hasSize(2);

5
driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
View File

@ -19,6 +19,7 @@ package io.nosqlbench.activitytype.stdout;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import com.codahale.metrics.Histogram;
@ -133,7 +134,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
SequencePlanner<StringBindings> sequencer = new SequencePlanner<>(sequencerType);
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
List<StmtDef> stmts = stmtsDocList.getStmts(tagfilter);
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
String format = getParams().getOptionalString("format").orElse(null);
@ -156,7 +157,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
sequencer.addOp(sb,1L);
}
} else if (stmts.size() > 0) {
for (StmtDef stmt : stmts) {
for (OpTemplate stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
BindingsTemplate bt = new BindingsTemplate(parsed.getBindPoints());
String statement = parsed.getPositionalStatement(Function.identity());

9
driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java
View File

@ -9,6 +9,7 @@ import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityapi.planning.SequencePlanner;
import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
@ -159,7 +160,7 @@ public class WebDriverActivity extends SimpleActivity {
.orElse(SequencerType.bucket);
SequencePlanner<CommandTemplate> planner = new SequencePlanner<>(sequencerType);
commands.forEach((name,cmd) -> {
CommandTemplate commandTemplate = new CommandTemplate(cmd, Map.of(), name, false);
CommandTemplate commandTemplate = new CommandTemplate(name, cmd, Map.of(), Map.of());
planner.addOp(commandTemplate,(c) -> 1L);
});
OpSequence<CommandTemplate> sequence = planner.resolve();
@ -178,15 +179,15 @@ public class WebDriverActivity extends SimpleActivity {
SequencePlanner<CommandTemplate> planner = new SequencePlanner<>(sequencerType);
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
List<StmtDef> stmts = stmtsDocList.getStmts(tagfilter);
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
if (stmts.size() == 0) {
throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'");
}
for (StmtDef optemplate : stmts) {
for (OpTemplate optemplate : stmts) {
long ratio = optemplate.getParamOrDefault("ratio", 1);
CommandTemplate cmd = new CommandTemplate(optemplate, false);
CommandTemplate cmd = new CommandTemplate(optemplate);
planner.addOp(cmd, ratio);
}
return planner.resolve();

41
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java Normal file
View File

@ -0,0 +1,41 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.util.Tagged;
import java.util.Map;
import java.util.Optional;
public interface OpTemplate extends Tagged {
String getName();
String getStmt();
Map<String,String> getBindings();
Map<String, Object> getParams();
<T> Map<String,T> getParamsAsValueType(Class<? extends T> type);
@SuppressWarnings("unchecked")
<V> V getParamOrDefault(String name, V defaultValue);
<V> V getParam(String name, Class<? extends V> type);
@SuppressWarnings("unchecked")
<V> Optional<V> getOptionalParam(String name, Class<? extends V> type);
Optional<String> getOptionalParam(String name);
Map<String,String> getTags();
/**
* Parse the statement for anchors and return a richer view of the StmtDef which
* is simpler to use for most statement configuration needs.
* @return a new {@link ParsedStmt}
*/
ParsedStmt getParsed();
String getDesc();
}

20
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDef.java
View File

@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
public class StmtDef implements Tagged {
public class StmtDef implements OpTemplate {
private final RawStmtDef rawStmtDef;
private StmtsBlock block;
@ -37,21 +37,27 @@ public class StmtDef implements Tagged {
this.rawStmtDef = rawStmtDef;
}
@Override
public String getName() {
return block.getName() + "--" + rawStmtDef.getName();
}
@Override
public String getStmt() {
return rawStmtDef.getStmt();
}
@Override
public Map<String,String> getBindings() {
return new MultiMapLookup<>(rawStmtDef.getBindings(), block.getBindings());
}
@Override
public Map<String, Object> getParams() {
return new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
}
@Override
public <T> Map<String,T> getParamsAsValueType(Class<? extends T> type) {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
Map<String,T> map = new LinkedHashMap<>();
@ -59,6 +65,7 @@ public class StmtDef implements Tagged {
return map;
}
@Override
@SuppressWarnings("unchecked")
public <V> V getParamOrDefault(String name, V defaultValue) {
Objects.requireNonNull(defaultValue);
@ -75,6 +82,7 @@ public class StmtDef implements Tagged {
}
}
@Override
public <V> V getParam(String name, Class<? extends V> type) {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
Object object = lookup.get(name);
@ -82,6 +90,7 @@ public class StmtDef implements Tagged {
return value;
}
@Override
@SuppressWarnings("unchecked")
public <V> Optional<V> getOptionalParam(String name, Class<? extends V> type) {
if (type.isPrimitive()) {
@ -104,11 +113,13 @@ public class StmtDef implements Tagged {
return Optional.empty();
}
@Override
public Optional<String> getOptionalParam(String name) {
return getOptionalParam(name,String.class);
}
@Override
public Map<String,String> getTags() {
return new MultiMapLookup<>(rawStmtDef.getTags(), block.getTags());
}
@ -118,15 +129,12 @@ public class StmtDef implements Tagged {
return "stmt(name:" + getName() + ", stmt:" + getStmt() + ", tags:(" + getTags() + "), params:(" + getParams() +"), bindings:(" + getBindings()+"))";
}
/**
* Parse the statement for anchors and return a richer view of the StmtDef which
* is simpler to use for most statement configuration needs.
* @return a new {@link ParsedStmt}
*/
@Override
public ParsedStmt getParsed() {
return new ParsedStmt(this);
}
@Override
public String getDesc() {
return rawStmtDef.getDesc();
}

12
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsBlock.java
View File

@ -25,7 +25,7 @@ import org.jetbrains.annotations.NotNull;
import java.util.*;
public class StmtsBlock implements Tagged, Iterable<StmtDef> {
public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
private final static String NameToken = "name";
private final static String StmtToken = "stmt";
@ -40,17 +40,17 @@ public class StmtsBlock implements Tagged, Iterable<StmtDef> {
this.blockIdx = blockIdx;
}
public List<StmtDef> getStmts() {
public List<OpTemplate> getStmts() {
List<StmtDef> rawStmtDefs = new ArrayList<>();
List<OpTemplate> rawOpTemplates = new ArrayList<>();
List<RawStmtDef> statements = rawStmtsBlock.getRawStmtDefs();
for (int i = 0; i < statements.size(); i++) {
rawStmtDefs.add(
rawOpTemplates.add(
new StmtDef(this, statements.get(i))
);
}
return rawStmtDefs;
return rawOpTemplates;
}
public String getName() {
@ -105,7 +105,7 @@ public class StmtsBlock implements Tagged, Iterable<StmtDef> {
@Override
@NotNull
public Iterator<StmtDef> iterator() {
public Iterator<OpTemplate> iterator() {
return getStmts().iterator();
}
}

2
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDoc.java
View File

@ -90,7 +90,7 @@ public class StmtsDoc implements Tagged, Iterable<StmtsBlock> {
* @return The list of all included statements for all included block in this document,
* including the inherited and overridden values from the this doc and the parent block.
*/
public List<StmtDef> getStmts() {
public List<OpTemplate> getStmts() {
return getBlocks().stream().flatMap(b -> b.getStmts().stream()).collect(Collectors.toList());
}

7
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java
View File

@ -17,7 +17,6 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawScenarios;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.util.TagFilter;
@ -46,7 +45,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
.collect(Collectors.toList());
}
public List<StmtDef> getStmts() {
public List<OpTemplate> getStmts() {
return getStmts("");
}
@ -55,10 +54,10 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
* including the inherited and overridden values from the this doc and the parent block.
* @param tagFilterSpec a comma-separated tag filter spec
*/
public List<StmtDef> getStmts(String tagFilterSpec) {
public List<OpTemplate> getStmts(String tagFilterSpec) {
TagFilter ts = new TagFilter(tagFilterSpec);
List<StmtDef> stmts = getStmtDocs().stream()
List<OpTemplate> stmts = getStmtDocs().stream()
.flatMap(d -> d.getStmts().stream())
.filter(ts::matchesTagged)
.collect(Collectors.toList());

16
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -12,6 +12,7 @@ import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
@ -24,6 +25,7 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;
/**
@ -329,7 +331,7 @@ public class SimpleActivity implements Activity {
}
protected OpSequence<CommandTemplate> createDefaultOpSequence() {
protected <O> OpSequence<O> createOpSequence(Function<OpTemplate,O> opinit) {
StrInterpolator interp = new StrInterpolator(activityDef);
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
@ -338,19 +340,21 @@ public class SimpleActivity implements Activity {
.getOptionalString("seq")
.map(SequencerType::valueOf)
.orElse(SequencerType.bucket);
SequencePlanner<CommandTemplate> planner = new SequencePlanner<>(sequencerType);
SequencePlanner<O> planner = new SequencePlanner<>(sequencerType);
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
List<StmtDef> stmts = stmtsDocList.getStmts(tagfilter);
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
if (stmts.size() == 0) {
throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'");
}
for (StmtDef optemplate : stmts) {
for (OpTemplate optemplate : stmts) {
long ratio = optemplate.getParamOrDefault("ratio", 1);
CommandTemplate cmd = new CommandTemplate(optemplate, false);
planner.addOp(cmd, ratio);
// CommandTemplate cmd = new CommandTemplate(optemplate);
O driverSpecificOp = opinit.apply(optemplate);
planner.addOp(driverSpecificOp, ratio);
}
return planner.resolve();
}

138
engine-api/src/main/java/io/nosqlbench/engine/api/templating/CommandTemplate.java
View File

@ -1,5 +1,6 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityimpl.motor.ParamsParser;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
@ -9,70 +10,121 @@ import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.*;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Use the {@link StmtDef} template form as a property template for parameterized
* commands. This is a general purpose template which uses a map of named parameters.
* The {@code command} property designates the verb component of the command.
*
* To be valid for use with this template type, the template specifier (the stmt String)
* must either start with command= or have a single word at the start. In either case,
* the command will be parsed as if it started with a command=...
*
* The semantics of command are meant to be generalized. For example, with HTTP, command
* might mean the HTTP method like GET or PUT that is used. For web driver, it may be
* a webdriver command as known by the SIDE file format.
* Use the {@link StmtDef} template form as a property template for parameterized commands. This is a general purpose
* template which uses a map of named parameters. The {@code command} property designates the verb component of the
* command.
* <p>
* To be valid for use with this template type, the template specifier (the stmt String) must either start with command=
* or have a single word at the start. In either case, the command will be parsed as if it started with a command=...
* <p>
* The semantics of command are meant to be generalized. For example, with HTTP, command might mean the HTTP method like
* GET or PUT that is used. For web driver, it may be a webdriver command as known by the SIDE file format.
*/
public class CommandTemplate {
private final static Logger logger = LoggerFactory.getLogger(CommandTemplate.class);
private final String name;
private LinkedHashMap<String, StringBindings> cmdspec = new LinkedHashMap<>();
private final Map<String, String> statics = new HashMap<>();
private final Map<String, StringBindings> dynamics = new HashMap<>();
public CommandTemplate(StmtDef stmt, boolean canonicalize) {
this.name = stmt.getName();
String prefixed = stmt.getStmt();
prefixed = (prefixed.startsWith("command=") ? prefixed : "command=" + prefixed);
public CommandTemplate(OpTemplate stmt) {
this(stmt.getName(), stmt.getStmt(), stmt.getParamsAsValueType(String.class), stmt.getBindings(), null);
}
Map<String,String> cmdMap = ParamsParser.parse(prefixed, canonicalize);
Map<String, String> paramsMap = stmt.getParamsAsValueType(String.class);
paramsMap.forEach((k,v) -> {
if (cmdMap.containsKey(k)) {
public CommandTemplate(OpTemplate stmt, Function<String, Map<String, String>> parser) {
this(stmt.getName(), stmt.getStmt(), stmt.getParamsAsValueType(String.class), stmt.getBindings(), parser);
}
/**
* Create a command template from a set of optional properties.
*
* @param name The name of the command template
* @param oneline A oneline version of the parameters. Passed as 'stmt' in the yaml format.
* @param params A set of named parameters and values in name:value form.
* @param bindings A set of named bindings in name:recipe form.
*/
public CommandTemplate(String name, String oneline, Map<String, String> params, Map<String, String> bindings, Function<String, Map<String, String>> optionalParser) {
this.name = name;
Map<String, String> cmd = new HashMap<>();
// Only parse and inject the oneline form if it is defined.
// The first parser to match and return a map will be the last one tried.
// If none of the suppliemental parsers work, the default params parser is used
if (oneline != null) {
List<Function<String,Map<String,String>>> parserlist = new ArrayList<>(List.of(optionalParser));
parserlist.add(s -> ParamsParser.parse(s,false));
for (Function<String, Map<String, String>> parser : parserlist) {
Map<String, String> parsed = parser.apply(oneline);
if (parsed!=null) {
logger.debug("parsed request: " + parsed.toString());
cmd.putAll(parsed);
break;
}
}
}
// Always add the named params, but warn if they overwrite any oneline named params
params.forEach((k, v) -> {
if (cmd.containsKey(k)) {
logger.warn("command property override: '" + k + "' superseded by param form with value '" + v + "'");
}
cmdMap.put(k,v);
});
cmd.putAll(params);
cmdMap.forEach((param,value) -> {
ParsedTemplate paramTemplate = new ParsedTemplate(value, stmt.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();
cmdspec.put(param,paramStringBindings);
cmd.forEach((param, value) -> {
ParsedTemplate paramTemplate = new ParsedTemplate(value, bindings);
if (paramTemplate.getBindPoints().size() > 0) {
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();
dynamics.put(param, paramStringBindings);
statics.put(param, null);
} else {
statics.put(param, value);
}
});
}
public CommandTemplate(String command, Map<String,String> bindings, String name, boolean canonicalize) {
this.name = name;
Map<String, String> cmdMap = ParamsParser.parse(command, canonicalize);
cmdMap.forEach((param,value) -> {
ParsedTemplate paramTemplate = new ParsedTemplate(command,bindings);
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();
cmdspec.put(param,paramStringBindings);
});
}
public Map<String,String> getCommand(long cycle) {
LinkedHashMap<String, String> cmd = new LinkedHashMap<>(cmdspec.size());
cmdspec.forEach((k,v) -> {
cmd.put(k,v.bind(cycle));
public Map<String, String> getCommand(long cycle) {
HashMap<String, String> map = new HashMap<>(statics);
dynamics.forEach((k, v) -> {
map.put(k, v.bind(cycle));
});
return cmd;
return map;
}
public String getName() {
return name;
}
public boolean isStatic() {
return this.dynamics.size() == 0;
}
public Set<String> getPropertyNames() {
return this.statics.keySet();
}
// private static List<String> namedGroups(String regex) {
// List<String> namedGroups = new ArrayList<String>();
//
// Matcher m = Pattern.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>").matcher(regex);
//
// while (m.find()) {
// namedGroups.add(m.group(1));
// }
//
// return namedGroups;
// }
}

2
engine-api/src/main/java/io/nosqlbench/engine/api/util/Tagged.java
View File

@ -17,6 +17,8 @@
package io.nosqlbench.engine.api.util;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import java.util.Map;
public interface Tagged {

9
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDefDefsTest.java
View File

@ -18,10 +18,7 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsBlock;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.Test;
@ -70,9 +67,9 @@ public class RawStmtDefDefsTest {
StmtsDoc doc1 = all.getStmtDocs().get(0);
StmtsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getName()).isEqualTo("doc1--block0");
List<StmtDef> assys = block1.getStmts();
List<OpTemplate> assys = block1.getStmts();
assertThat(assys).hasSize(2);
StmtDef sdef1 = assys.get(0);
OpTemplate sdef1 = assys.get(0);
assertThat(sdef1.getName()).isEqualTo("doc1--block0--stmt1");
assertThat(assys.get(0).getStmt()).isEqualTo("s1");
}

7
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java
View File

@ -18,10 +18,7 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsBlock;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.*;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -34,7 +31,7 @@ import static org.assertj.core.api.Assertions.assertThat;
public class StmtEscapingTest {
private final static Logger logger = LoggerFactory.getLogger(StmtEscapingTest.class);
private static List<StmtDef> defs;
private static List<OpTemplate> defs;
@BeforeClass
public static void testLayering() {

12
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtTest.java
View File

@ -38,14 +38,14 @@ public class ParsedStmtTest {
@Test
public void testBasicParser() {
StmtsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0);
StmtDef stmtDef0 = block0.getStmts().get(0);
OpTemplate stmtDef0 = block0.getStmts().get(0);
ParsedStmt parsed0 = stmtDef0.getParsed();
assertThat(parsed0.getExtraBindings()).containsExactly("alpha","gamma");
assertThat(parsed0.getMissingBindings()).containsExactly("delta");
assertThat(parsed0.hasError()).isTrue();
StmtsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1);
StmtDef stmtDef1 = block1.getStmts().get(0);
OpTemplate stmtDef1 = block1.getStmts().get(0);
ParsedStmt parsed1 = stmtDef1.getParsed();
assertThat(parsed1.getExtraBindings()).containsExactly();
assertThat(parsed1.getMissingBindings()).containsExactly();
@ -56,12 +56,12 @@ public class ParsedStmtTest {
public void testMultipleBindingUsage() {
StmtsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2);
StmtDef stmtDef0 = block2.getStmts().get(0);
OpTemplate stmtDef0 = block2.getStmts().get(0);
ParsedStmt parsed0 = stmtDef0.getParsed();
assertThat(parsed0.getMissingBindings()).isEmpty();
assertThat(parsed0.hasError()).isFalse();
StmtDef stmtDef1 = block2.getStmts().get(1);
OpTemplate stmtDef1 = block2.getStmts().get(1);
ParsedStmt parsed1 = stmtDef1.getParsed();
assertThat(parsed1.getMissingBindings().isEmpty());
assertThat(parsed1.hasError()).isFalse();
@ -71,12 +71,12 @@ public class ParsedStmtTest {
public void testQuestionMarkAnchors() {
StmtsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(3);
StmtDef stmtDef0 = block2.getStmts().get(0);
OpTemplate stmtDef0 = block2.getStmts().get(0);
ParsedStmt parsed0 = stmtDef0.getParsed();
assertThat(parsed0.getMissingBindings()).isEmpty();
assertThat(parsed0.hasError()).isFalse();
StmtDef stmtDef1 = block2.getStmts().get(1);
OpTemplate stmtDef1 = block2.getStmts().get(1);
ParsedStmt parsed1 = stmtDef1.getParsed();
assertThat(parsed1.getMissingBindings().isEmpty());
assertThat(parsed1.hasError()).isFalse();

5
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java
View File

@ -18,7 +18,6 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.Test;
@ -43,7 +42,7 @@ public class StmtDetailOverrideTest {
assertThat(doc1.getBlocks()).hasSize(2);
StmtsBlock doc1block0 = doc1.getBlocks().get(0);
assertThat(doc1block0.getStmts().size()).isEqualTo(1);
StmtDef s = doc1block0.getStmts().get(0);
OpTemplate s = doc1block0.getStmts().get(0);
assertThat(s.getName()).isEqualTo("block0--stmt1");
assertThat(s.getStmt()).isEqualTo("globalstatement1");
assertThat(s.getBindings()).hasSize(1);
@ -51,7 +50,7 @@ public class StmtDetailOverrideTest {
assertThat(s.getTags()).hasSize(1);
StmtsBlock doc1block1 = doc1.getBlocks().get(1);
List<StmtDef> stmts = doc1block1.getStmts();
List<OpTemplate> stmts = doc1block1.getStmts();
assertThat(stmts).hasSize(4);
s = stmts.get(0);

6
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java
View File

@ -77,7 +77,7 @@ public class StmtsDocListTest {
@Test
public void testStmtInheritsBlockData() {
StmtsDoc doc0 = doclist.getStmtDocs().get(0);
List<StmtDef> stmts1 = doc0.getBlocks().get(0).getStmts();
List<OpTemplate> stmts1 = doc0.getBlocks().get(0).getStmts();
assertThat(stmts1).hasSize(2);
StmtsBlock block0 = doc0.getBlocks().get(0);
@ -121,13 +121,13 @@ public class StmtsDocListTest {
@Test
public void testStmtsGetter() {
StmtsDoc doc1 = doclist.getStmtDocs().get(1);
List<StmtDef> stmts = doc1.getStmts();
List<OpTemplate> stmts = doc1.getStmts();
assertThat(stmts).hasSize(4);
}
@Test
public void testFilteredStmts() {
List<StmtDef> stmts = doclist.getStmts("");
List<OpTemplate> stmts = doclist.getStmts("");
assertThat(stmts).hasSize(6);
stmts = doclist.getStmts("root1:value23");
assertThat(stmts).hasSize(2);

7
engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java
View File

@ -1,6 +1,7 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import org.junit.Test;
@ -14,9 +15,9 @@ public class CommandTemplateTest {
StmtsDocList stmtsDocs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: test1=foo test2=bar");
StmtDef stmtDef = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(stmtDef, false);
assertThat(ct.isStatic()).isTrue();
OpTemplate stmtDef = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(stmtDef);
// assertThat(ct.isStatic()).isTrue();
}