IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-11-29 20:14:04 -06:00
Code Issues Packages Projects Releases Wiki Activity

improvements to command structure for debugging and scripting

Browse Source
This commit is contained in:
Jonathan Shook 2020-04-20 10:03:24 -05:00
parent 760861230a
commit 5c80a283a3
8 changed files with 176 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
activitytype-cql/src/main/resources/examples/cql-rowops.yaml
View File

@ -1,7 +1,7 @@
scenarios:
default:
schema: run driver=cql tags==phase:schema cycles==UNDEF threads==1
rampup: run driver=cql tags==phase:rampup cycles=TEMPLATE(rampup-cycles100K) threads=auto
rampup: run driver=cql tags==phase:rampup cycles=TEMPLATE(rampup-cycles,100K) threads=auto
bindings:
userid: Template('user-{}',ToString()); SaveString('userid');

2
activitytype-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
View File

@ -138,7 +138,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
String format = getParams().getOptionalString("format").orElse(null);
if ((stmts.size()==0 && stmtsDocList.getDocBindings().size() > 0) || format!=null) {
logger.info("Creating stdout statement template from bindings, since none is otherwise defined.");
logger.info("Creating stdout statement template from bindings...");
String generatedStmt = genStatementTemplate(stmtsDocList.getDocBindings().keySet());
BindingsTemplate bt = new BindingsTemplate();
stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);

44
engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java
View File

@ -1,10 +1,6 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -18,16 +14,14 @@ public class BasicScriptBuffer implements ScriptBuffer {
public ScriptBuffer add(Cmd cmd) {
Map<String, String> params = cmd.getParams();
if (!cmd.getParams().isEmpty()) {
sb.append(toJSONParams("params", cmd.getParams()));
}
switch (cmd.getCmdType()) {
case script:
sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
String scriptData = NBCLIScriptAssembly.loadScript(cmd);
sb.append(scriptData);
break;
case fragment:
sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
sb.append(cmd.getArg("script_fragment"));
if (!cmd.getArg("script_fragment").endsWith("\n")) {
sb.append("\n");
@ -35,20 +29,22 @@ public class BasicScriptBuffer implements ScriptBuffer {
break;
case start: // start activity
case run: // run activity
// Sanity check that this can parse before using it
sb.append("scenario.").append(cmd.toString()).append("(")
.append(toJSONBlock(cmd.getParams()))
.append(");\n");
break;
case await: // await activity
sb.append("scenario.awaitActivity(\"").append(cmd.getArg("alias_name")).append("\");\n");
break;
case stop: // stop activity
sb.append("scenario.stop(\"").append(cmd.getArg("alias_name")).append("\");\n");
break;
case waitmillis:
long millis_to_wait = Long.parseLong(cmd.getArg("millis_to_wait"));
sb.append("scenario.waitMillis(").append(millis_to_wait).append(");\n");
sb.append("scenario.").append(cmd).append("\n");
//// // Sanity check that this can parse before using it
//// sb.append("scenario.").append(cmd.toString()).append("(")
//// .append(Cmd.toJSONBlock(cmd.getParams(), false))
//// .append(");\n");
//// break;
// sb.append("scenario.awaitActivity(\"").append(cmd.getArg("alias_name")).append("\");\n");
// break;
// sb.append("scenario.stop(\"").append(cmd.getArg("alias_name")).append("\");\n");
// break;
// long millis_to_wait = Long.parseLong(cmd.getArg("millis_to_wait"));
// sb.append("scenario.waitMillis(").append(millis_to_wait).append(");\n");
break;
}
return this;
@ -60,15 +56,5 @@ public class BasicScriptBuffer implements ScriptBuffer {
return sb.toString();
}
public static String toJSONBlock(Map<?,?> map) {
StringBuilder sb = new StringBuilder();
List<String> l = new ArrayList<>();
map.forEach((k, v) -> l.add("'" + k + "': '" + v + "'"));
return "{" + String.join(",\n ", l) + "};\n";
}
public static String toJSONParams(String varname, Map<?, ?> map) {
return "// params.size==" + map.size() + "\n" + varname + "="+toJSONBlock(map);
}
}

112
engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java
View File

@ -2,13 +2,12 @@ package io.nosqlbench.engine.cli;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.security.InvalidParameterException;
import java.util.*;
import java.util.stream.Collectors;
import java.util.function.Function;
/**
* Encapsulate Command parsing and structure for the NoSQLBench command line.
@ -17,29 +16,54 @@ import java.util.stream.Collectors;
* An example of a command tha thas both would look like {@code script test.js p1=v1}
*/
public class Cmd {
private final static Logger logger = LoggerFactory.getLogger(Cmd.class);
public enum CmdType {
script("script_path"),
fragment("script_fragment"),
script(Arg.of("script_path", s -> s)),
fragment(Arg.of("script_fragment")),
start(),
run(),
await("alias_name"),
stop("alias_name"),
waitmillis("millis_to_wait");
await(Arg.of("alias_name")),
stop(Arg.of("alias_name")),
waitmillis(Arg.of("millis_to_wait", Long::parseLong));
private final String[] positional;
private final Arg<?>[] positional;
CmdType(String... positional) {
CmdType(Arg<?>... positional) {
this.positional = positional;
}
public String[] getPositionalArgs() {
public String[] getPositionalArgNames() {
String[] names = new String[positional.length];
for (int i = 0; i < names.length; i++) {
names[i] = positional[i].name;
}
return names;
}
public Arg<?>[] getPositionalArgs() {
return positional;
}
}
private static final class Arg<T> {
public final String name;
public final Function<String, T> converter;
public Arg(String name, Function<String, T> converter) {
this.name = name;
this.converter = converter;
}
public static <T> Arg<T> of(String name, Function<String, T> converter) {
return new Arg<>(name, converter);
}
public static Arg<String> of(String name) {
return new Arg<>(name, s -> s);
}
}
private Map<String, String> cmdArgs;
@ -49,7 +73,7 @@ public class Cmd {
private final CmdType cmdType;
public Cmd(CmdType cmdType, Map<String,String> cmdArgs) {
public Cmd(CmdType cmdType, Map<String, String> cmdArgs) {
this.cmdArgs = cmdArgs;
this.cmdType = cmdType;
}
@ -63,19 +87,31 @@ public class Cmd {
}
public String toString() {
return "type:" + cmdType + ((cmdArgs != null) ? ";cmdArgs=" + cmdArgs.toString() : "");
StringBuilder sb = new StringBuilder();
sb.append(cmdType.toString());
sb.append("(");
if (getParams().size() > cmdType.positional.length) {
sb.append(toJSONBlock(getParams(), false));
} else {
for (String value : getParams().values()) {
sb.append("'").append(value).append("'").append(",");
}
sb.setLength(sb.length() - 1);
}
sb.append(");");
return sb.toString();
}
public static Cmd parseArg(LinkedList<String> arglist, NBCLIOptions options) {
public static Cmd parseArg(LinkedList<String> arglist, PathCanonicalizer fixer) {
String cmdName = arglist.removeFirst();
CmdType cmdType = CmdType.valueOf(cmdName);
Map<String,String> params = new LinkedHashMap<>();
Map<String, String> params = new LinkedHashMap<>();
for (String paramName : cmdType.getPositionalArgs()) {
for (Arg<?> paramName : cmdType.getPositionalArgs()) {
String arg = arglist.peekFirst();
if (arg==null) {
if (arg == null) {
throw new InvalidParameterException("command '" + cmdName + " requires a value for " + paramName
+ ", but there were no remaining arguments after it.");
}
@ -88,8 +124,8 @@ public class Cmd {
+ ", but a reserved word was found instead: " + arg);
}
logger.debug("cmd name:" + cmdName +", positional " + paramName + ": " + arg);
params.put(paramName,arglist.removeFirst());
logger.debug("cmd name:" + cmdName + ", positional " + paramName + ": " + arg);
params.put(paramName.name, paramName.converter.apply(arglist.removeFirst()).toString());
}
while (arglist.size() > 0 &&
@ -100,30 +136,34 @@ public class Cmd {
String pname = assigned[0];
String pval = assigned[1];
if (pname.equals("yaml")||pname.equals("workload")) {
String yaml = pval;
Optional<Content<?>> found = NBIO.local().prefix("activities")
.prefix(options.wantsIncludes())
.name(yaml)
.first();
if (found.isPresent()) {
if (!found.get().asPath().toString().equals(yaml)) {
logger.info("rewrote path for " + yaml + " as " + found.get().asPath().toString());
pval=found.get().asPath().toString();
} else {
logger.debug("kept path for " + yaml + " as " + found.get().asPath().toString());
}
} else {
logger.debug("unable to find " + yaml + " for path qualification");
}
if (pname.equals("yaml") || pname.equals("workload")) {
pval = fixer.canonicalizePath(pval);
}
if (params.containsKey(pname)) {
throw new InvalidParameterException("parameter '" + pname + "' is already set for " + cmdType);
}
params.put(pname,pval);
params.put(pname, pval);
}
return new Cmd(cmdType, params);
}
public static String toJSONBlock(Map<String, String> map, boolean oneline) {
int klen = map.keySet().stream().mapToInt(String::length).max().orElse(1);
StringBuilder sb = new StringBuilder();
List<String> l = new ArrayList<>();
map.forEach((k, v) -> l.add(
(oneline ? "" : " ") + "'" + k + "'"
+": " + (oneline ? "" : " ".repeat(klen - k.length())) +
"'" + v + "'"
));
return "{" + (oneline ? "" : "\n") + String.join(",\n", l) + (oneline ? "}" : "\n}");
}
public static String toJSONParams(String varname, Map<String, String> map, boolean oneline) {
return "// params.size==" + map.size() + "\n" + varname + "=" + toJSONBlock(map, oneline);
}
}

8
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@ -161,6 +161,8 @@ public class NBCLIOptions {
arglist = nonincludes;
nonincludes = new LinkedList<>();
PathCanonicalizer canonicalizer = new PathCanonicalizer(wantsIncludes());
while (arglist.peekFirst() != null) {
String word = arglist.peekFirst();
if (word.startsWith("--") && word.contains("=")) {
@ -179,7 +181,7 @@ public class NBCLIOptions {
case LIST_METRICS:
arglist.removeFirst();
arglist.addFirst("start");
Cmd cmd = Cmd.parseArg(arglist,this);
Cmd cmd = Cmd.parseArg(arglist,canonicalizer);
wantsMetricsForActivity = cmd.getArg("driver");
break;
case SESSION_NAME:
@ -334,7 +336,7 @@ public class NBCLIOptions {
case AWAIT:
case STOP:
case WAIT_MILLIS:
cmd = Cmd.parseArg(arglist,this);
cmd = Cmd.parseArg(arglist,canonicalizer);
cmdList.add(cmd);
break;
// cmd = Cmd.parseArg(arglist, this, "alias_to_await");
@ -369,7 +371,7 @@ public class NBCLIOptions {
arglist.removeFirst();
arglist.addFirst("scripts/auto/" + word);
arglist.addFirst("script");
cmd = Cmd.parseArg(arglist,this);
cmd = Cmd.parseArg(arglist,canonicalizer);
cmdList.add(cmd);
} else if (
NBCLIScenarioParser.isFoundWorkload(word, wantsIncludes())

38
engine-cli/src/main/java/io/nosqlbench/engine/cli/PathCanonicalizer.java Normal file
View File

@ -0,0 +1,38 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Optional;
public class PathCanonicalizer {
private final static Logger logger = LoggerFactory.getLogger(Cmd.class);
private final String[] includes;
public PathCanonicalizer(String... includes) {
this.includes = includes;
}
public String canonicalizePath(String path) {
Optional<Content<?>> found = NBIO.local().prefix("activities")
.prefix(includes)
.name(path)
.first();
if (found.isPresent()) {
if (!found.get().asPath().toString().equals(path)) {
logger.info("rewrote path for " + path + " as " + found.get().asPath().toString());
return found.get().asPath().toString();
} else {
logger.trace("kept path for " + path + " as " + found.get().asPath().toString());
}
} else {
logger.trace("unable to find " + path + " for path qualification");
}
return path;
}
}

31
engine-cli/src/test/java/io/nosqlbench/engine/cli/CmdTest.java Normal file
View File

@ -0,0 +1,31 @@
package io.nosqlbench.engine.cli;
import org.junit.Test;
import java.util.LinkedList;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
public class CmdTest {
private final static PathCanonicalizer p = new PathCanonicalizer();
@Test
public void testCmdForWaitMillis() {
Cmd cmd = Cmd.parseArg(new LinkedList<String>(List.of("waitmillis", "234")), p);
assertThat(cmd.getArg("millis_to_wait")).isEqualTo("234");
assertThat(cmd.toString()).isEqualTo("waitmillis('234');");
}
@Test
public void testCmdForStart() {
Cmd cmd = Cmd.parseArg(new LinkedList<>(List.of("start","type=stdout","otherparam=foo")),p);
assertThat(cmd.toString()).isEqualTo("start({\n" +
" 'type': 'stdout',\n" +
" 'otherparam': 'foo'\n" +
"});");
}
}

9
engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioController.java
View File

@ -349,16 +349,25 @@ public class ScenarioController {
}
}
public boolean await(Map<String,String> activityDefMap) {
return this.awaitActivity(activityDefMap);
}
public boolean awaitActivity(Map<String, String> activityDefMap) {
ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap));
return awaitActivity(ad);
}
public boolean await(String alias) {
return this.awaitActivity(alias);
}
public boolean awaitActivity(String alias) {
ActivityDef toAwait = aliasToDef(alias);
return awaitActivity(toAwait);
}
public boolean await(ActivityDef activityDef) {
return this.awaitActivity(activityDef);
}
public boolean awaitActivity(ActivityDef activityDef) {
ActivityExecutor activityExecutor = getActivityExecutor(activityDef, false);
if (activityExecutor == null) {