IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-11-25 10:10:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

add ParsedCommand as a better CommandTemplate

Browse Source
This commit is contained in:
Jonathan Shook 2021-06-23 11:42:43 -05:00
parent 41ce92cf9c
commit e6e398bb6b
13 changed files with 160 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmt.java → engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmtOp.java
View File

@ -17,10 +17,9 @@
package io.nosqlbench.engine.api.activityconfig;
import io.nosqlbench.engine.api.activityconfig.yaml.OpDef;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.nb.api.config.params.Element;
import io.nosqlbench.nb.api.config.params.NBParams;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.core.templates.BindPoint;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
@ -29,33 +28,23 @@ import java.util.Map;
import java.util.Set;
import java.util.function.Function;
/**
* Allow for uniform statement anchor parsing, using the <pre>?anchor</pre>
* and <pre>{anchor}</pre> anchoring conventions. This type also includes
* all of the properties from the enclosed StmtDef, in addition to a couple of
* helpers. It should allow programmers to project this type directly from an
* existing {@link OpDef} as a substitute.
*/
public class ParsedStmt {
public class ParsedStmtOp {
private final OpDef opDef;
private final OpTemplate optpl;
private final ParsedTemplate parsed;
/**
* Construct a new ParsedStatement from the provided stmtDef and anchor token.
*
* @param opDef An existing statement def as read from the YAML API.
* @param optpl An existing statement def as read from the YAML API.
*/
public ParsedStmt(OpDef opDef, Function<String, String>... transforms) {
this.opDef = opDef;
public ParsedStmtOp(OpTemplate optpl) {
this.optpl = optpl;
String transformed = getStmt();
for (Function<String, String> transform : transforms) {
transformed = transform.apply(transformed);
}
parsed = new ParsedTemplate(transformed, opDef.getBindings());
parsed = new ParsedTemplate(transformed, optpl.getBindings());
}
public ParsedStmt orError() {
public ParsedStmtOp orError() {
if (hasError()) {
throw new RuntimeException("Unable to parse statement: " + this);
}
@ -100,42 +89,38 @@ public class ParsedStmt {
}
/**
* @return the statement name from the enclosed {@link OpDef}
* @return the statement name from the enclosed {@link OpTemplate}
*/
public String getName() {
return opDef.getName();
return optpl.getName();
}
/**
* @return the raw statement from the enclosed {@link OpDef}
* @return the raw statement from the enclosed {@link OpTemplate}
*/
public String getStmt() {
if (opDef.getOp() instanceof CharSequence) {
return opDef.getOp().toString();
} else {
throw new BasicError("Tried to access op type '" + opDef.getOp().getClass().getSimpleName() + " as a string statement");
}
return optpl.getStmt().orElseThrow();
}
/**
* @return the tags from the enclosed {@link OpDef}
* @return the tags from the enclosed {@link OpTemplate}
*/
public Map<String, String> getTags() {
return opDef.getTags();
return optpl.getTags();
}
/**
* @return the bindings from the enclosed {@link OpDef}
* @return the bindings from the enclosed {@link OpTemplate}
*/
public Map<String, String> getBindings() {
return opDef.getBindings();
return optpl.getBindings();
}
/**
* @return a params reader from the enclosed {@link OpDef} params map
* @return a params reader from the enclosed {@link OpTemplate} params map
*/
public Element getParamReader() {
return NBParams.one(opDef.getParams());
return NBParams.one(optpl.getParams());
}
public List<BindPoint> getBindPoints() {

110
engine-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedCommand.java
View File

@ -1,36 +1,40 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.nb.api.config.ParamsParser;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.core.bindings.DataMapper;
import io.nosqlbench.virtdata.core.bindings.VirtData;
import io.nosqlbench.virtdata.core.templates.CapturePoint;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import io.nosqlbench.virtdata.core.templates.StringBindings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.function.Function;
import java.util.function.LongFunction;
/**
* Parse an OpTemplate into a ParsedCommand
* Parse an OpTemplate into a ParsedCommand, which can dispense object maps
*/
public class ParsedCommand {
public class ParsedCommand implements LongFunction<Map<String, ?>> {
private final static Logger logger = LogManager.getLogger(ParsedCommand.class);
/** the name of this operation **/
/**
* the name of this operation
**/
private final String name;
/** The fields which are statically assigned **/
private final Map<String,Object> statics = new LinkedHashMap<>();
/**
* The fields which are statically assigned
**/
private final Map<String, Object> statics = new LinkedHashMap<>();
/**
* The fields which are dynamic, and must be realized via functions.
* This map contains keys which identify the field names, and values, which may be null or undefined.
*/
private final Map<String,String> dynamics = new LinkedHashMap<>();
private final Map<String, LongFunction<?>> dynamics = new LinkedHashMap<>();
/**
* The names of payload values in the result of the operation which should be saved.
@ -38,75 +42,53 @@ public class ParsedCommand {
* representation of a result. If the values are defined, then each one represents the name
* that the found value should be saved as instead of the original name.
*/
private final Map<String,String> captures = new LinkedHashMap<>();
private final List<List<CapturePoint>> captures = new ArrayList<>();
private final int mapsize;
/**
* Create a parsed command from an Op template. The op template is simply the normalized view of
* op template structure which is uniform regardless of the original format.
*
* @param ot An OpTemplate representing an operation to be performed in a native driver.
*/
ParsedCommand(OpTemplate ot) {
this(ot,List.of());
this(ot, List.of());
}
ParsedCommand(OpTemplate ot, List<Function<String, Map<String, String>>> optionalParsers) {
ParsedCommand(OpTemplate ot, List<Function<Map<String, Object>, Map<String, Object>>> preprocessors) {
this.name = ot.getName();
Map<String,Object> cmd = new LinkedHashMap<>();
if (ot.getOp() instanceof CharSequence) {
String oneline = ot.getOp().toString();
List<Function<String, Map<String, String>>> parserlist = new ArrayList<>(optionalParsers);
boolean didParse = false;
parserlist.add(s -> ParamsParser.parse(s, false));
for (Function<String, Map<String, String>> parser : parserlist) {
Map<String, String> parsed = parser.apply(oneline);
if (parsed != null) {
logger.debug("parsed request: " + parsed);
cmd.putAll(parsed);
didParse = true;
break;
}
}
} else if (ot.getOp() instanceof Map) {
Map<?,?> map = ot.getOp();
for (Map.Entry<?, ?> entry : map.entrySet()) {
cmd.put(entry.getKey().toString(),entry.getValue());
}
} else {
throw new BasicError("op template has op type of " + ot.getOp().getClass().getCanonicalName() + ", which is not supported.");
Map<String, Object> map = ot.getOp().orElseThrow();
for (Function<Map<String, Object>, Map<String, Object>> preprocessor : preprocessors) {
map = preprocessor.apply(map);
}
resolveCmdMap(cmd,ot.getBindings());
// ArrayList<Function<String, Map<String, String>>> _parsers = new ArrayList<>(parsers);
}
private void resolveCmdMap(Map<String, Object> cmd, Map<String, String> bindings) {
Map<String,Object> resolved = new LinkedHashMap<>();
cmd.forEach((k,v) -> {
map.forEach((k, v) -> {
if (v instanceof CharSequence) {
ParsedTemplate parsed = new ParsedTemplate(v.toString(), bindings);
switch (parsed.getType()) {
ParsedTemplate pt = ParsedTemplate.of(((CharSequence) v).toString(), ot.getBindings());
this.captures.add(pt.getCaptures());
switch (pt.getType()) {
case literal:
statics.put(k, ((CharSequence) v).toString());
break;
case bindref:
String spec = pt.asBinding().orElseThrow().getBindspec();
Optional<DataMapper<Object>> mapper = VirtData.getOptionalMapper(spec);
dynamics.put(k, mapper.orElseThrow());
break;
case concat:
this.dynamics.put(k,v.toString());
StringBindings sb = new StringBindings(pt);
dynamics.put(k, sb);
break;
}
} else if (v instanceof Map) {
Map<String,Object> m = (Map<String, Object>) v;
// ((Map<?, ?>) v).forEach((k,v) -> {
//
// });
resolved.put(k,Map.of("type","Map"));
} else {
statics.put(k, v);
}
});
mapsize = statics.size() + dynamics.size();
}
public String getName() {
@ -117,7 +99,17 @@ public class ParsedCommand {
return statics;
}
public Map<String, String> getDynamics() {
public Map<String, LongFunction<?>> getDynamics() {
return dynamics;
}
@Override
public Map<String, Object> apply(long value) {
HashMap<String,Object> map = new HashMap<>(mapsize);
map.putAll(statics);
dynamics.forEach((k,v) -> {
map.put(k,v.apply(value));
});
return map;
}
}

4
engine-api/src/main/java/io/nosqlbench/engine/api/util/Tagged.java
View File

@ -17,10 +17,8 @@
package io.nosqlbench.engine.api.util;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import java.util.Map;
public interface Tagged {
public Map<String,String> getTags();
Map<String,String> getTags();
}

14
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtTest.java → engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtOpTest.java
View File

@ -18,7 +18,7 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.ParsedStmtOp;
import org.junit.jupiter.api.BeforeAll;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@ -26,8 +26,8 @@ import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class ParsedStmtTest {
private static final Logger logger = LogManager.getLogger(ParsedStmtTest.class);
public class ParsedStmtOpTest {
private static final Logger logger = LogManager.getLogger(ParsedStmtOpTest.class);
private static StmtsDocList doclist;
@BeforeAll
@ -39,13 +39,13 @@ public class ParsedStmtTest {
public void testBasicParser() {
StmtsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0);
OpTemplate stmtDef0 = block0.getOps().get(0);
ParsedStmt parsed0 = stmtDef0.getParsed();
ParsedStmtOp parsed0 = stmtDef0.getParsed().orElseThrow();
assertThat(parsed0.getMissingBindings()).containsExactly("delta");
assertThat(parsed0.hasError()).isTrue();
StmtsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1);
OpTemplate stmtDef1 = block1.getOps().get(0);
ParsedStmt parsed1 = stmtDef1.getParsed();
ParsedStmtOp parsed1 = stmtDef1.getParsed().orElseThrow();
assertThat(parsed1.getMissingBindings()).containsExactly();
assertThat(parsed1.hasError()).isFalse();
}
@ -55,12 +55,12 @@ public class ParsedStmtTest {
StmtsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2);
OpTemplate stmtDef0 = block2.getOps().get(0);
ParsedStmt parsed0 = stmtDef0.getParsed();
ParsedStmtOp parsed0 = stmtDef0.getParsed().orElseThrow();
assertThat(parsed0.getMissingBindings()).isEmpty();
assertThat(parsed0.hasError()).isFalse();
OpTemplate stmtDef1 = block2.getOps().get(1);
ParsedStmt parsed1 = stmtDef1.getParsed();
ParsedStmtOp parsed1 = stmtDef1.getParsed().orElseThrow();
assertThat(parsed1.getMissingBindings()).isEmpty();
assertThat(parsed1.hasError()).isFalse();
}

15
engine-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedCommandTest.java Normal file
View File

@ -0,0 +1,15 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityconfig.yaml.OpData;
import org.junit.jupiter.api.Test;
import java.util.Map;
public class ParsedCommandTest {
@Test
public void testParsedCommand() {
ParsedCommand pc = new ParsedCommand(new OpData().applyFields(Map.of("op", Map.of("stmt", "test"))));
}
}

30
nb-api/src/main/java/io/nosqlbench/nb/api/config/ParamsParser.java
View File

@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
/**
@ -74,6 +75,16 @@ import java.util.Map;
* spaces and partial word are included in the last value assigment found. Leading spaces on literal
* values are skipped unless escaped.</p>
*
* <H3>Detection</H3>
* When a caller wants to parse this format optionally when the format is recognizable as having parameters,
* the {@link #hasValues(String)} method can be called. To be recognized as having parameters, a more strict
* definition is used: The patter must start with a simple assignment having a varname which starts with an
* alphabetic character or an underscore, followed by any alpha-numeric, dot {@code .}, dash {@code -}, or
* underscore {@code _} with an assignment character {@code =} following. The following regex can be used
* as an example for documentation purposes when explaining op mapping conventions to users:
* <pre>{@code
* [_a-zA-Z][-_.\\w]*
* }</pre>
*/
public class ParamsParser {
public final static String ASSIGN_CHARS = "=:";
@ -89,7 +100,8 @@ public class ParamsParser {
public static boolean hasValues(String input, String assignChars) {
for (int i = 0; i < assignChars.length(); i++) {
if (input.contains(assignChars.substring(i, i + 1))) {
Pattern assignPattern = Pattern.compile("[A-Za-z_][-_\\w\\d.]*\\s*" + assignChars.charAt(i) + ".*");
if (assignPattern.matcher(input).matches()) {
return true;
}
}
@ -137,7 +149,7 @@ public class ParamsParser {
switch (s) {
case expectingName:
if (c =='\'' || c=='"') {
if (c == '\'' || c == '"') {
throw new RuntimeException("Unable to parse a name starting with character '" + c + "'. Names" +
" must be literal values.");
} else if (c != ' ' && c != ';') {
@ -152,7 +164,7 @@ public class ParamsParser {
String partial = parms.get(lastVarname);
if (partial == null) {
throw new RuntimeException("space continuation while reading name or value, but no prior " +
"for " + lastVarname + " exists");
"for " + lastVarname + " exists");
}
parms.put(lastVarname, partial + " " + varname);
varname.setLength(0);
@ -186,7 +198,7 @@ public class ParamsParser {
value.append(c);
} else {
parms.put(varname.toString(), value.toString());
lastVarname=varname.toString();
lastVarname = varname.toString();
varname.setLength(0);
value.setLength(0);
s = ParseState.expectingName;
@ -237,22 +249,22 @@ public class ParamsParser {
s = ParseState.expectingName;
break;
case readingName:
parms.put(lastVarname,parms.get(lastVarname)+' '+ varname);
parms.put(lastVarname, parms.get(lastVarname) + ' ' + varname);
varname.setLength(0);
break;
default:
}
if (input.length()>0 && parms.size()==0) {
if (input.length() > 0 && parms.size() == 0) {
throw new RuntimeException("Unable to parse input:" + input);
}
if (canonicalize) {
List<String> keys= new ArrayList<>(parms.keySet());
List<String> keys = new ArrayList<>(parms.keySet());
for (String key : keys) {
String properkey= Synonyms.canonicalize(key,logger);
String properkey = Synonyms.canonicalize(key, logger);
if (!key.equals(properkey)) {
parms.put(properkey,parms.get(key));
parms.put(properkey, parms.get(key));
parms.remove(key);
}
}

13
nb-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/ParamsParserTest.java
View File

@ -165,4 +165,17 @@ public class ParamsParserTest {
assertThat(p.get("b")).isEqualTo(" sixer");
}
@Test
public void testHasValues() {
assertThat(ParamsParser.hasValues("has=values")).isTrue();
assertThat(ParamsParser.hasValues("has = values")).isTrue();
assertThat(ParamsParser.hasValues("has =values")).isTrue();
assertThat(ParamsParser.hasValues("has= values")).isTrue();
assertThat(ParamsParser.hasValues("3has= values")).isFalse();
assertThat(ParamsParser.hasValues("_has= values")).isTrue();
assertThat(ParamsParser.hasValues("h-as= values")).isTrue();
assertThat(ParamsParser.hasValues("h.as= values")).isTrue();
assertThat(ParamsParser.hasValues("h_as= values")).isTrue();
}
}

9
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/Binder.java
View File

@ -1,15 +1,22 @@
package io.nosqlbench.virtdata.core.bindings;
import java.util.function.LongFunction;
/***
* A Binder is a type that knows how to return a result object given a long value
* to bind mapped values with.
* @param <R> The resulting object type
*/
public interface Binder<R> {
public interface Binder<R> extends LongFunction<R> {
/**
* Bind values derived from a long to some object, returning an object type R
* @param value a long input value
* @return an R
*/
R bind(long value);
@Override
default R apply(long value) {
return bind(value);
}
}

9
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/DataMapper.java
View File

@ -1,5 +1,12 @@
package io.nosqlbench.virtdata.core.bindings;
public interface DataMapper<R> {
import java.util.function.LongFunction;
public interface DataMapper<R> extends LongFunction<R> {
R get(long input);
@Override
default R apply(long value) {
return get(value);
}
}

4
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplate.java
View File

@ -85,6 +85,10 @@ public class ParsedTemplate {
private final List<CapturePoint> captures;
private final String rawtemplate;
public static ParsedTemplate of(String rawtemplate, Map<String,String> bindings) {
return new ParsedTemplate(rawtemplate,bindings);
}
/**
* The type of a parsed template depends on the structure of the bindings provided.
*/

6
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindings.java
View File

@ -2,6 +2,7 @@ package io.nosqlbench.virtdata.core.templates;
import io.nosqlbench.virtdata.core.bindings.Binder;
import io.nosqlbench.virtdata.core.bindings.Bindings;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
/**
* Allows the generation of strings from a string template and bindings template.
@ -16,6 +17,11 @@ public class StringBindings implements Binder<String> {
this.bindings = bindings;
}
public StringBindings(ParsedTemplate pt) {
this.compositor = new StringCompositor(pt);
this.bindings = new BindingsTemplate(pt.getBindPoints()).resolveBindings();
}
/**
* Call the data mapper bindings, assigning the returned values positionally to the anchors in the string binding.
*

10
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindingsTemplate.java
View File

@ -6,16 +6,14 @@ import io.nosqlbench.virtdata.core.bindings.Bindings;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import java.util.HashSet;
import java.util.List;
import java.util.function.Function;
/**
* Uses a string template and a bindings template to create instances of {@link StringBindings}.
*/
public class StringBindingsTemplate {
private String stringTemplate;
private BindingsTemplate bindingsTemplate;
private final String stringTemplate;
private final BindingsTemplate bindingsTemplate;
public StringBindingsTemplate(String stringTemplate, BindingsTemplate bindingsTemplate) {
this.stringTemplate = stringTemplate;
@ -56,7 +54,7 @@ public class StringBindingsTemplate {
HashSet<String> unqualifiedNames = new HashSet<>(compositor.getBindPointNames());
unqualifiedNames.removeAll(new HashSet<>(bindingsTemplate.getBindPointNames()));
if (unqualifiedNames.size()>0) {
throw new RuntimeException("Named anchors were specified in the template which were not provided in the bindings: " + unqualifiedNames.toString());
throw new RuntimeException("Named anchors were specified in the template which were not provided in the bindings: " + unqualifiedNames);
}
Bindings bindings = bindingsTemplate.resolveBindings();
@ -68,7 +66,7 @@ public class StringBindingsTemplate {
HashSet<String> unqualifiedNames = new HashSet<>(compositor.getBindPointNames());
unqualifiedNames.removeAll(new HashSet<>(bindingsTemplate.getBindPointNames()));
if (unqualifiedNames.size()>0) {
throw new RuntimeException("Named anchors were specified in the template which were not provided in the bindings: " + unqualifiedNames.toString());
throw new RuntimeException("Named anchors were specified in the template which were not provided in the bindings: " + unqualifiedNames);
}
return bindingsTemplate.getDiagnostics();
}

4
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringCompositor.java
View File

@ -35,6 +35,10 @@ public class StringCompositor implements ValuesBinder<StringCompositor, String>
this.stringfunc = stringfunc;
}
public StringCompositor(ParsedTemplate pt) {
templateSegments = pt.getSpans();
}
// for testing
protected String[] parseTemplate(String template) {
ParsedTemplate parsed = new ParsedTemplate(template, Collections.emptyMap());