OpTemplate interface, generalized sequencer init, first working test

This commit is contained in:
Jonathan Shook
2020-06-29 18:39:04 -05:00
parent fd2da9972b
commit adb49d0737
30 changed files with 460 additions and 278 deletions

View File

@@ -0,0 +1,41 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.util.Tagged;
import java.util.Map;
import java.util.Optional;
public interface OpTemplate extends Tagged {
String getName();
String getStmt();
Map<String,String> getBindings();
Map<String, Object> getParams();
<T> Map<String,T> getParamsAsValueType(Class<? extends T> type);
@SuppressWarnings("unchecked")
<V> V getParamOrDefault(String name, V defaultValue);
<V> V getParam(String name, Class<? extends V> type);
@SuppressWarnings("unchecked")
<V> Optional<V> getOptionalParam(String name, Class<? extends V> type);
Optional<String> getOptionalParam(String name);
Map<String,String> getTags();
/**
* Parse the statement for anchors and return a richer view of the StmtDef which
* is simpler to use for most statement configuration needs.
* @return a new {@link ParsedStmt}
*/
ParsedStmt getParsed();
String getDesc();
}

View File

@@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
public class StmtDef implements Tagged {
public class StmtDef implements OpTemplate {
private final RawStmtDef rawStmtDef;
private StmtsBlock block;
@@ -37,21 +37,27 @@ public class StmtDef implements Tagged {
this.rawStmtDef = rawStmtDef;
}
@Override
public String getName() {
return block.getName() + "--" + rawStmtDef.getName();
}
@Override
public String getStmt() {
return rawStmtDef.getStmt();
}
@Override
public Map<String,String> getBindings() {
return new MultiMapLookup<>(rawStmtDef.getBindings(), block.getBindings());
}
@Override
public Map<String, Object> getParams() {
return new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
}
@Override
public <T> Map<String,T> getParamsAsValueType(Class<? extends T> type) {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
Map<String,T> map = new LinkedHashMap<>();
@@ -59,6 +65,7 @@ public class StmtDef implements Tagged {
return map;
}
@Override
@SuppressWarnings("unchecked")
public <V> V getParamOrDefault(String name, V defaultValue) {
Objects.requireNonNull(defaultValue);
@@ -75,6 +82,7 @@ public class StmtDef implements Tagged {
}
}
@Override
public <V> V getParam(String name, Class<? extends V> type) {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
Object object = lookup.get(name);
@@ -82,6 +90,7 @@ public class StmtDef implements Tagged {
return value;
}
@Override
@SuppressWarnings("unchecked")
public <V> Optional<V> getOptionalParam(String name, Class<? extends V> type) {
if (type.isPrimitive()) {
@@ -104,11 +113,13 @@ public class StmtDef implements Tagged {
return Optional.empty();
}
@Override
public Optional<String> getOptionalParam(String name) {
return getOptionalParam(name,String.class);
}
@Override
public Map<String,String> getTags() {
return new MultiMapLookup<>(rawStmtDef.getTags(), block.getTags());
}
@@ -118,15 +129,12 @@ public class StmtDef implements Tagged {
return "stmt(name:" + getName() + ", stmt:" + getStmt() + ", tags:(" + getTags() + "), params:(" + getParams() +"), bindings:(" + getBindings()+"))";
}
/**
* Parse the statement for anchors and return a richer view of the StmtDef which
* is simpler to use for most statement configuration needs.
* @return a new {@link ParsedStmt}
*/
@Override
public ParsedStmt getParsed() {
return new ParsedStmt(this);
}
@Override
public String getDesc() {
return rawStmtDef.getDesc();
}

View File

@@ -25,7 +25,7 @@ import org.jetbrains.annotations.NotNull;
import java.util.*;
public class StmtsBlock implements Tagged, Iterable<StmtDef> {
public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
private final static String NameToken = "name";
private final static String StmtToken = "stmt";
@@ -40,17 +40,17 @@ public class StmtsBlock implements Tagged, Iterable<StmtDef> {
this.blockIdx = blockIdx;
}
public List<StmtDef> getStmts() {
public List<OpTemplate> getStmts() {
List<StmtDef> rawStmtDefs = new ArrayList<>();
List<OpTemplate> rawOpTemplates = new ArrayList<>();
List<RawStmtDef> statements = rawStmtsBlock.getRawStmtDefs();
for (int i = 0; i < statements.size(); i++) {
rawStmtDefs.add(
rawOpTemplates.add(
new StmtDef(this, statements.get(i))
);
}
return rawStmtDefs;
return rawOpTemplates;
}
public String getName() {
@@ -105,7 +105,7 @@ public class StmtsBlock implements Tagged, Iterable<StmtDef> {
@Override
@NotNull
public Iterator<StmtDef> iterator() {
public Iterator<OpTemplate> iterator() {
return getStmts().iterator();
}
}

View File

@@ -90,7 +90,7 @@ public class StmtsDoc implements Tagged, Iterable<StmtsBlock> {
* @return The list of all included statements for all included block in this document,
* including the inherited and overridden values from the this doc and the parent block.
*/
public List<StmtDef> getStmts() {
public List<OpTemplate> getStmts() {
return getBlocks().stream().flatMap(b -> b.getStmts().stream()).collect(Collectors.toList());
}

View File

@@ -17,7 +17,6 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawScenarios;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.util.TagFilter;
@@ -46,7 +45,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
.collect(Collectors.toList());
}
public List<StmtDef> getStmts() {
public List<OpTemplate> getStmts() {
return getStmts("");
}
@@ -55,10 +54,10 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
* including the inherited and overridden values from the this doc and the parent block.
* @param tagFilterSpec a comma-separated tag filter spec
*/
public List<StmtDef> getStmts(String tagFilterSpec) {
public List<OpTemplate> getStmts(String tagFilterSpec) {
TagFilter ts = new TagFilter(tagFilterSpec);
List<StmtDef> stmts = getStmtDocs().stream()
List<OpTemplate> stmts = getStmtDocs().stream()
.flatMap(d -> d.getStmts().stream())
.filter(ts::matchesTagged)
.collect(Collectors.toList());

View File

@@ -12,6 +12,7 @@ import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
@@ -24,6 +25,7 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;
/**
@@ -329,7 +331,7 @@ public class SimpleActivity implements Activity {
}
protected OpSequence<CommandTemplate> createDefaultOpSequence() {
protected <O> OpSequence<O> createOpSequence(Function<OpTemplate,O> opinit) {
StrInterpolator interp = new StrInterpolator(activityDef);
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
@@ -338,19 +340,21 @@ public class SimpleActivity implements Activity {
.getOptionalString("seq")
.map(SequencerType::valueOf)
.orElse(SequencerType.bucket);
SequencePlanner<CommandTemplate> planner = new SequencePlanner<>(sequencerType);
SequencePlanner<O> planner = new SequencePlanner<>(sequencerType);
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
List<StmtDef> stmts = stmtsDocList.getStmts(tagfilter);
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
if (stmts.size() == 0) {
throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'");
}
for (StmtDef optemplate : stmts) {
for (OpTemplate optemplate : stmts) {
long ratio = optemplate.getParamOrDefault("ratio", 1);
CommandTemplate cmd = new CommandTemplate(optemplate, false);
planner.addOp(cmd, ratio);
// CommandTemplate cmd = new CommandTemplate(optemplate);
O driverSpecificOp = opinit.apply(optemplate);
planner.addOp(driverSpecificOp, ratio);
}
return planner.resolve();
}

View File

@@ -1,5 +1,6 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityimpl.motor.ParamsParser;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
@@ -9,70 +10,121 @@ import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.*;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Use the {@link StmtDef} template form as a property template for parameterized
* commands. This is a general purpose template which uses a map of named parameters.
* The {@code command} property designates the verb component of the command.
*
* To be valid for use with this template type, the template specifier (the stmt String)
* must either start with command= or have a single word at the start. In either case,
* the command will be parsed as if it started with a command=...
*
* The semantics of command are meant to be generalized. For example, with HTTP, command
* might mean the HTTP method like GET or PUT that is used. For web driver, it may be
* a webdriver command as known by the SIDE file format.
* Use the {@link StmtDef} template form as a property template for parameterized commands. This is a general purpose
* template which uses a map of named parameters. The {@code command} property designates the verb component of the
* command.
* <p>
* To be valid for use with this template type, the template specifier (the stmt String) must either start with command=
* or have a single word at the start. In either case, the command will be parsed as if it started with a command=...
* <p>
* The semantics of command are meant to be generalized. For example, with HTTP, command might mean the HTTP method like
* GET or PUT that is used. For web driver, it may be a webdriver command as known by the SIDE file format.
*/
public class CommandTemplate {
private final static Logger logger = LoggerFactory.getLogger(CommandTemplate.class);
private final String name;
private LinkedHashMap<String, StringBindings> cmdspec = new LinkedHashMap<>();
private final Map<String, String> statics = new HashMap<>();
private final Map<String, StringBindings> dynamics = new HashMap<>();
public CommandTemplate(StmtDef stmt, boolean canonicalize) {
this.name = stmt.getName();
String prefixed = stmt.getStmt();
prefixed = (prefixed.startsWith("command=") ? prefixed : "command=" + prefixed);
public CommandTemplate(OpTemplate stmt) {
this(stmt.getName(), stmt.getStmt(), stmt.getParamsAsValueType(String.class), stmt.getBindings(), null);
}
Map<String,String> cmdMap = ParamsParser.parse(prefixed, canonicalize);
Map<String, String> paramsMap = stmt.getParamsAsValueType(String.class);
paramsMap.forEach((k,v) -> {
if (cmdMap.containsKey(k)) {
public CommandTemplate(OpTemplate stmt, Function<String, Map<String, String>> parser) {
this(stmt.getName(), stmt.getStmt(), stmt.getParamsAsValueType(String.class), stmt.getBindings(), parser);
}
/**
* Create a command template from a set of optional properties.
*
* @param name The name of the command template
* @param oneline A oneline version of the parameters. Passed as 'stmt' in the yaml format.
* @param params A set of named parameters and values in name:value form.
* @param bindings A set of named bindings in name:recipe form.
*/
public CommandTemplate(String name, String oneline, Map<String, String> params, Map<String, String> bindings, Function<String, Map<String, String>> optionalParser) {
this.name = name;
Map<String, String> cmd = new HashMap<>();
// Only parse and inject the oneline form if it is defined.
// The first parser to match and return a map will be the last one tried.
// If none of the suppliemental parsers work, the default params parser is used
if (oneline != null) {
List<Function<String,Map<String,String>>> parserlist = new ArrayList<>(List.of(optionalParser));
parserlist.add(s -> ParamsParser.parse(s,false));
for (Function<String, Map<String, String>> parser : parserlist) {
Map<String, String> parsed = parser.apply(oneline);
if (parsed!=null) {
logger.debug("parsed request: " + parsed.toString());
cmd.putAll(parsed);
break;
}
}
}
// Always add the named params, but warn if they overwrite any oneline named params
params.forEach((k, v) -> {
if (cmd.containsKey(k)) {
logger.warn("command property override: '" + k + "' superseded by param form with value '" + v + "'");
}
cmdMap.put(k,v);
});
cmd.putAll(params);
cmdMap.forEach((param,value) -> {
ParsedTemplate paramTemplate = new ParsedTemplate(value, stmt.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();
cmdspec.put(param,paramStringBindings);
cmd.forEach((param, value) -> {
ParsedTemplate paramTemplate = new ParsedTemplate(value, bindings);
if (paramTemplate.getBindPoints().size() > 0) {
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();
dynamics.put(param, paramStringBindings);
statics.put(param, null);
} else {
statics.put(param, value);
}
});
}
public CommandTemplate(String command, Map<String,String> bindings, String name, boolean canonicalize) {
this.name = name;
Map<String, String> cmdMap = ParamsParser.parse(command, canonicalize);
cmdMap.forEach((param,value) -> {
ParsedTemplate paramTemplate = new ParsedTemplate(command,bindings);
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();
cmdspec.put(param,paramStringBindings);
});
}
public Map<String,String> getCommand(long cycle) {
LinkedHashMap<String, String> cmd = new LinkedHashMap<>(cmdspec.size());
cmdspec.forEach((k,v) -> {
cmd.put(k,v.bind(cycle));
public Map<String, String> getCommand(long cycle) {
HashMap<String, String> map = new HashMap<>(statics);
dynamics.forEach((k, v) -> {
map.put(k, v.bind(cycle));
});
return cmd;
return map;
}
public String getName() {
return name;
}
public boolean isStatic() {
return this.dynamics.size() == 0;
}
public Set<String> getPropertyNames() {
return this.statics.keySet();
}
// private static List<String> namedGroups(String regex) {
// List<String> namedGroups = new ArrayList<String>();
//
// Matcher m = Pattern.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>").matcher(regex);
//
// while (m.find()) {
// namedGroups.add(m.group(1));
// }
//
// return namedGroups;
// }
}

View File

@@ -17,6 +17,8 @@
package io.nosqlbench.engine.api.util;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import java.util.Map;
public interface Tagged {

View File

@@ -18,10 +18,7 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsBlock;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.Test;
@@ -70,9 +67,9 @@ public class RawStmtDefDefsTest {
StmtsDoc doc1 = all.getStmtDocs().get(0);
StmtsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getName()).isEqualTo("doc1--block0");
List<StmtDef> assys = block1.getStmts();
List<OpTemplate> assys = block1.getStmts();
assertThat(assys).hasSize(2);
StmtDef sdef1 = assys.get(0);
OpTemplate sdef1 = assys.get(0);
assertThat(sdef1.getName()).isEqualTo("doc1--block0--stmt1");
assertThat(assys.get(0).getStmt()).isEqualTo("s1");
}

View File

@@ -18,10 +18,7 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsBlock;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.*;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,7 +31,7 @@ import static org.assertj.core.api.Assertions.assertThat;
public class StmtEscapingTest {
private final static Logger logger = LoggerFactory.getLogger(StmtEscapingTest.class);
private static List<StmtDef> defs;
private static List<OpTemplate> defs;
@BeforeClass
public static void testLayering() {

View File

@@ -38,14 +38,14 @@ public class ParsedStmtTest {
@Test
public void testBasicParser() {
StmtsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0);
StmtDef stmtDef0 = block0.getStmts().get(0);
OpTemplate stmtDef0 = block0.getStmts().get(0);
ParsedStmt parsed0 = stmtDef0.getParsed();
assertThat(parsed0.getExtraBindings()).containsExactly("alpha","gamma");
assertThat(parsed0.getMissingBindings()).containsExactly("delta");
assertThat(parsed0.hasError()).isTrue();
StmtsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1);
StmtDef stmtDef1 = block1.getStmts().get(0);
OpTemplate stmtDef1 = block1.getStmts().get(0);
ParsedStmt parsed1 = stmtDef1.getParsed();
assertThat(parsed1.getExtraBindings()).containsExactly();
assertThat(parsed1.getMissingBindings()).containsExactly();
@@ -56,12 +56,12 @@ public class ParsedStmtTest {
public void testMultipleBindingUsage() {
StmtsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2);
StmtDef stmtDef0 = block2.getStmts().get(0);
OpTemplate stmtDef0 = block2.getStmts().get(0);
ParsedStmt parsed0 = stmtDef0.getParsed();
assertThat(parsed0.getMissingBindings()).isEmpty();
assertThat(parsed0.hasError()).isFalse();
StmtDef stmtDef1 = block2.getStmts().get(1);
OpTemplate stmtDef1 = block2.getStmts().get(1);
ParsedStmt parsed1 = stmtDef1.getParsed();
assertThat(parsed1.getMissingBindings().isEmpty());
assertThat(parsed1.hasError()).isFalse();
@@ -71,12 +71,12 @@ public class ParsedStmtTest {
public void testQuestionMarkAnchors() {
StmtsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(3);
StmtDef stmtDef0 = block2.getStmts().get(0);
OpTemplate stmtDef0 = block2.getStmts().get(0);
ParsedStmt parsed0 = stmtDef0.getParsed();
assertThat(parsed0.getMissingBindings()).isEmpty();
assertThat(parsed0.hasError()).isFalse();
StmtDef stmtDef1 = block2.getStmts().get(1);
OpTemplate stmtDef1 = block2.getStmts().get(1);
ParsedStmt parsed1 = stmtDef1.getParsed();
assertThat(parsed1.getMissingBindings().isEmpty());
assertThat(parsed1.hasError()).isFalse();

View File

@@ -18,7 +18,6 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.Test;
@@ -43,7 +42,7 @@ public class StmtDetailOverrideTest {
assertThat(doc1.getBlocks()).hasSize(2);
StmtsBlock doc1block0 = doc1.getBlocks().get(0);
assertThat(doc1block0.getStmts().size()).isEqualTo(1);
StmtDef s = doc1block0.getStmts().get(0);
OpTemplate s = doc1block0.getStmts().get(0);
assertThat(s.getName()).isEqualTo("block0--stmt1");
assertThat(s.getStmt()).isEqualTo("globalstatement1");
assertThat(s.getBindings()).hasSize(1);
@@ -51,7 +50,7 @@ public class StmtDetailOverrideTest {
assertThat(s.getTags()).hasSize(1);
StmtsBlock doc1block1 = doc1.getBlocks().get(1);
List<StmtDef> stmts = doc1block1.getStmts();
List<OpTemplate> stmts = doc1block1.getStmts();
assertThat(stmts).hasSize(4);
s = stmts.get(0);

View File

@@ -77,7 +77,7 @@ public class StmtsDocListTest {
@Test
public void testStmtInheritsBlockData() {
StmtsDoc doc0 = doclist.getStmtDocs().get(0);
List<StmtDef> stmts1 = doc0.getBlocks().get(0).getStmts();
List<OpTemplate> stmts1 = doc0.getBlocks().get(0).getStmts();
assertThat(stmts1).hasSize(2);
StmtsBlock block0 = doc0.getBlocks().get(0);
@@ -121,13 +121,13 @@ public class StmtsDocListTest {
@Test
public void testStmtsGetter() {
StmtsDoc doc1 = doclist.getStmtDocs().get(1);
List<StmtDef> stmts = doc1.getStmts();
List<OpTemplate> stmts = doc1.getStmts();
assertThat(stmts).hasSize(4);
}
@Test
public void testFilteredStmts() {
List<StmtDef> stmts = doclist.getStmts("");
List<OpTemplate> stmts = doclist.getStmts("");
assertThat(stmts).hasSize(6);
stmts = doclist.getStmts("root1:value23");
assertThat(stmts).hasSize(2);

View File

@@ -1,6 +1,7 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import org.junit.Test;
@@ -14,9 +15,9 @@ public class CommandTemplateTest {
StmtsDocList stmtsDocs = StatementsLoader.loadString("" +
"statements:\n" +
" - s1: test1=foo test2=bar");
StmtDef stmtDef = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(stmtDef, false);
assertThat(ct.isStatic()).isTrue();
OpTemplate stmtDef = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(stmtDef);
// assertThat(ct.isStatic()).isTrue();
}