diff --git a/activitytype-cql/src/main/resources/examples/cql-rowops.yaml b/activitytype-cql/src/main/resources/examples/cql-rowops.yaml index b93eff9d2..5874b937d 100644 --- a/activitytype-cql/src/main/resources/examples/cql-rowops.yaml +++ b/activitytype-cql/src/main/resources/examples/cql-rowops.yaml @@ -1,7 +1,7 @@ scenarios: default: schema: run driver=cql tags==phase:schema cycles==UNDEF threads==1 - rampup: run driver=cql tags==phase:rampup cycles=TEMPLATE(rampup-cycles100K) threads=auto + rampup: run driver=cql tags==phase:rampup cycles=TEMPLATE(rampup-cycles,100K) threads=auto bindings: userid: Template('user-{}',ToString()); SaveString('userid'); diff --git a/activitytype-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java b/activitytype-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java index 088a2aee4..58015502d 100644 --- a/activitytype-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java +++ b/activitytype-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java @@ -138,7 +138,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve String format = getParams().getOptionalString("format").orElse(null); if ((stmts.size()==0 && stmtsDocList.getDocBindings().size() > 0) || format!=null) { - logger.info("Creating stdout statement template from bindings, since none is otherwise defined."); + logger.info("Creating stdout statement template from bindings..."); String generatedStmt = genStatementTemplate(stmtsDocList.getDocBindings().keySet()); BindingsTemplate bt = new BindingsTemplate(); stmtsDocList.getDocBindings().forEach(bt::addFieldBinding); diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java index 2046c7c61..56bf76029 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java @@ -1,10 +1,6 @@ package io.nosqlbench.engine.cli; -import io.nosqlbench.engine.api.activityimpl.ActivityDef; - -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; @@ -18,16 +14,14 @@ public class BasicScriptBuffer implements ScriptBuffer { public ScriptBuffer add(Cmd cmd) { Map params = cmd.getParams(); - if (!cmd.getParams().isEmpty()) { - sb.append(toJSONParams("params", cmd.getParams())); - } - switch (cmd.getCmdType()) { case script: + sb.append(Cmd.toJSONParams("params", cmd.getParams(), false)); String scriptData = NBCLIScriptAssembly.loadScript(cmd); sb.append(scriptData); break; case fragment: + sb.append(Cmd.toJSONParams("params", cmd.getParams(), false)); sb.append(cmd.getArg("script_fragment")); if (!cmd.getArg("script_fragment").endsWith("\n")) { sb.append("\n"); @@ -35,20 +29,22 @@ public class BasicScriptBuffer implements ScriptBuffer { break; case start: // start activity case run: // run activity - // Sanity check that this can parse before using it - sb.append("scenario.").append(cmd.toString()).append("(") - .append(toJSONBlock(cmd.getParams())) - .append(");\n"); - break; case await: // await activity - sb.append("scenario.awaitActivity(\"").append(cmd.getArg("alias_name")).append("\");\n"); - break; case stop: // stop activity - sb.append("scenario.stop(\"").append(cmd.getArg("alias_name")).append("\");\n"); - break; case waitmillis: - long millis_to_wait = Long.parseLong(cmd.getArg("millis_to_wait")); - sb.append("scenario.waitMillis(").append(millis_to_wait).append(");\n"); + + sb.append("scenario.").append(cmd).append("\n"); +//// // Sanity check that this can parse before using it +//// sb.append("scenario.").append(cmd.toString()).append("(") +//// .append(Cmd.toJSONBlock(cmd.getParams(), false)) +//// .append(");\n"); +//// break; +// sb.append("scenario.awaitActivity(\"").append(cmd.getArg("alias_name")).append("\");\n"); +// break; +// sb.append("scenario.stop(\"").append(cmd.getArg("alias_name")).append("\");\n"); +// break; +// long millis_to_wait = Long.parseLong(cmd.getArg("millis_to_wait")); +// sb.append("scenario.waitMillis(").append(millis_to_wait).append(");\n"); break; } return this; @@ -60,15 +56,5 @@ public class BasicScriptBuffer implements ScriptBuffer { return sb.toString(); } - public static String toJSONBlock(Map map) { - StringBuilder sb = new StringBuilder(); - List l = new ArrayList<>(); - map.forEach((k, v) -> l.add("'" + k + "': '" + v + "'")); - return "{" + String.join(",\n ", l) + "};\n"; - } - - public static String toJSONParams(String varname, Map map) { - return "// params.size==" + map.size() + "\n" + varname + "="+toJSONBlock(map); - } } diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java index 5f237ab3e..1a4e10194 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java @@ -2,13 +2,12 @@ package io.nosqlbench.engine.cli; import io.nosqlbench.nb.api.content.Content; import io.nosqlbench.nb.api.content.NBIO; -import io.nosqlbench.nb.api.errors.BasicError; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.security.InvalidParameterException; import java.util.*; -import java.util.stream.Collectors; +import java.util.function.Function; /** * Encapsulate Command parsing and structure for the NoSQLBench command line. @@ -17,29 +16,54 @@ import java.util.stream.Collectors; * An example of a command tha thas both would look like {@code script test.js p1=v1} */ public class Cmd { - private final static Logger logger = LoggerFactory.getLogger(Cmd.class); public enum CmdType { - script("script_path"), - fragment("script_fragment"), + script(Arg.of("script_path", s -> s)), + fragment(Arg.of("script_fragment")), start(), run(), - await("alias_name"), - stop("alias_name"), - waitmillis("millis_to_wait"); + await(Arg.of("alias_name")), + stop(Arg.of("alias_name")), + waitmillis(Arg.of("millis_to_wait", Long::parseLong)); - private final String[] positional; + private final Arg[] positional; - CmdType(String... positional) { + CmdType(Arg... positional) { this.positional = positional; } - public String[] getPositionalArgs() { + public String[] getPositionalArgNames() { + String[] names = new String[positional.length]; + for (int i = 0; i < names.length; i++) { + names[i] = positional[i].name; + } + return names; + } + + public Arg[] getPositionalArgs() { return positional; } } + private static final class Arg { + public final String name; + public final Function converter; + + public Arg(String name, Function converter) { + this.name = name; + this.converter = converter; + } + + public static Arg of(String name, Function converter) { + return new Arg<>(name, converter); + } + + public static Arg of(String name) { + return new Arg<>(name, s -> s); + } + } + private Map cmdArgs; @@ -49,7 +73,7 @@ public class Cmd { private final CmdType cmdType; - public Cmd(CmdType cmdType, Map cmdArgs) { + public Cmd(CmdType cmdType, Map cmdArgs) { this.cmdArgs = cmdArgs; this.cmdType = cmdType; } @@ -63,19 +87,31 @@ public class Cmd { } public String toString() { - return "type:" + cmdType + ((cmdArgs != null) ? ";cmdArgs=" + cmdArgs.toString() : ""); + StringBuilder sb = new StringBuilder(); + sb.append(cmdType.toString()); + sb.append("("); + if (getParams().size() > cmdType.positional.length) { + sb.append(toJSONBlock(getParams(), false)); + } else { + for (String value : getParams().values()) { + sb.append("'").append(value).append("'").append(","); + } + sb.setLength(sb.length() - 1); + } + sb.append(");"); + return sb.toString(); } - public static Cmd parseArg(LinkedList arglist, NBCLIOptions options) { + public static Cmd parseArg(LinkedList arglist, PathCanonicalizer fixer) { String cmdName = arglist.removeFirst(); CmdType cmdType = CmdType.valueOf(cmdName); - Map params = new LinkedHashMap<>(); + Map params = new LinkedHashMap<>(); - for (String paramName : cmdType.getPositionalArgs()) { + for (Arg paramName : cmdType.getPositionalArgs()) { String arg = arglist.peekFirst(); - if (arg==null) { + if (arg == null) { throw new InvalidParameterException("command '" + cmdName + " requires a value for " + paramName + ", but there were no remaining arguments after it."); } @@ -88,8 +124,8 @@ public class Cmd { + ", but a reserved word was found instead: " + arg); } - logger.debug("cmd name:" + cmdName +", positional " + paramName + ": " + arg); - params.put(paramName,arglist.removeFirst()); + logger.debug("cmd name:" + cmdName + ", positional " + paramName + ": " + arg); + params.put(paramName.name, paramName.converter.apply(arglist.removeFirst()).toString()); } while (arglist.size() > 0 && @@ -100,30 +136,34 @@ public class Cmd { String pname = assigned[0]; String pval = assigned[1]; - if (pname.equals("yaml")||pname.equals("workload")) { - String yaml = pval; - Optional> found = NBIO.local().prefix("activities") - .prefix(options.wantsIncludes()) - .name(yaml) - .first(); - if (found.isPresent()) { - if (!found.get().asPath().toString().equals(yaml)) { - logger.info("rewrote path for " + yaml + " as " + found.get().asPath().toString()); - pval=found.get().asPath().toString(); - } else { - logger.debug("kept path for " + yaml + " as " + found.get().asPath().toString()); - } - } else { - logger.debug("unable to find " + yaml + " for path qualification"); - } + + if (pname.equals("yaml") || pname.equals("workload")) { + pval = fixer.canonicalizePath(pval); } if (params.containsKey(pname)) { throw new InvalidParameterException("parameter '" + pname + "' is already set for " + cmdType); } - params.put(pname,pval); + params.put(pname, pval); } return new Cmd(cmdType, params); } + public static String toJSONBlock(Map map, boolean oneline) { + + int klen = map.keySet().stream().mapToInt(String::length).max().orElse(1); + StringBuilder sb = new StringBuilder(); + List l = new ArrayList<>(); + map.forEach((k, v) -> l.add( + (oneline ? "" : " ") + "'" + k + "'" + +": " + (oneline ? "" : " ".repeat(klen - k.length())) + + "'" + v + "'" + )); + return "{" + (oneline ? "" : "\n") + String.join(",\n", l) + (oneline ? "}" : "\n}"); + } + + public static String toJSONParams(String varname, Map map, boolean oneline) { + return "// params.size==" + map.size() + "\n" + varname + "=" + toJSONBlock(map, oneline); + } + } diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java index 420df2401..24f34e79c 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java @@ -161,6 +161,8 @@ public class NBCLIOptions { arglist = nonincludes; nonincludes = new LinkedList<>(); + PathCanonicalizer canonicalizer = new PathCanonicalizer(wantsIncludes()); + while (arglist.peekFirst() != null) { String word = arglist.peekFirst(); if (word.startsWith("--") && word.contains("=")) { @@ -179,7 +181,7 @@ public class NBCLIOptions { case LIST_METRICS: arglist.removeFirst(); arglist.addFirst("start"); - Cmd cmd = Cmd.parseArg(arglist,this); + Cmd cmd = Cmd.parseArg(arglist,canonicalizer); wantsMetricsForActivity = cmd.getArg("driver"); break; case SESSION_NAME: @@ -334,7 +336,7 @@ public class NBCLIOptions { case AWAIT: case STOP: case WAIT_MILLIS: - cmd = Cmd.parseArg(arglist,this); + cmd = Cmd.parseArg(arglist,canonicalizer); cmdList.add(cmd); break; // cmd = Cmd.parseArg(arglist, this, "alias_to_await"); @@ -369,7 +371,7 @@ public class NBCLIOptions { arglist.removeFirst(); arglist.addFirst("scripts/auto/" + word); arglist.addFirst("script"); - cmd = Cmd.parseArg(arglist,this); + cmd = Cmd.parseArg(arglist,canonicalizer); cmdList.add(cmd); } else if ( NBCLIScenarioParser.isFoundWorkload(word, wantsIncludes()) diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/PathCanonicalizer.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/PathCanonicalizer.java new file mode 100644 index 000000000..55dd184ac --- /dev/null +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/PathCanonicalizer.java @@ -0,0 +1,38 @@ +package io.nosqlbench.engine.cli; + +import io.nosqlbench.nb.api.content.Content; +import io.nosqlbench.nb.api.content.NBIO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Optional; + +public class PathCanonicalizer { + private final static Logger logger = LoggerFactory.getLogger(Cmd.class); + + private final String[] includes; + + public PathCanonicalizer(String... includes) { + this.includes = includes; + } + + public String canonicalizePath(String path) { + + Optional> found = NBIO.local().prefix("activities") + .prefix(includes) + .name(path) + .first(); + + if (found.isPresent()) { + if (!found.get().asPath().toString().equals(path)) { + logger.info("rewrote path for " + path + " as " + found.get().asPath().toString()); + return found.get().asPath().toString(); + } else { + logger.trace("kept path for " + path + " as " + found.get().asPath().toString()); + } + } else { + logger.trace("unable to find " + path + " for path qualification"); + } + return path; + } +} diff --git a/engine-cli/src/test/java/io/nosqlbench/engine/cli/CmdTest.java b/engine-cli/src/test/java/io/nosqlbench/engine/cli/CmdTest.java new file mode 100644 index 000000000..63a96af0c --- /dev/null +++ b/engine-cli/src/test/java/io/nosqlbench/engine/cli/CmdTest.java @@ -0,0 +1,31 @@ +package io.nosqlbench.engine.cli; + +import org.junit.Test; + +import java.util.LinkedList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CmdTest { + + private final static PathCanonicalizer p = new PathCanonicalizer(); + + @Test + public void testCmdForWaitMillis() { + + Cmd cmd = Cmd.parseArg(new LinkedList(List.of("waitmillis", "234")), p); + assertThat(cmd.getArg("millis_to_wait")).isEqualTo("234"); + assertThat(cmd.toString()).isEqualTo("waitmillis('234');"); + } + + @Test + public void testCmdForStart() { + Cmd cmd = Cmd.parseArg(new LinkedList<>(List.of("start","type=stdout","otherparam=foo")),p); + assertThat(cmd.toString()).isEqualTo("start({\n" + + " 'type': 'stdout',\n" + + " 'otherparam': 'foo'\n" + + "});"); + } + +} diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioController.java b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioController.java index f8514a482..175a42976 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioController.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioController.java @@ -349,16 +349,25 @@ public class ScenarioController { } } + public boolean await(Map activityDefMap) { + return this.awaitActivity(activityDefMap); + } public boolean awaitActivity(Map activityDefMap) { ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap)); return awaitActivity(ad); } + public boolean await(String alias) { + return this.awaitActivity(alias); + } public boolean awaitActivity(String alias) { ActivityDef toAwait = aliasToDef(alias); return awaitActivity(toAwait); } + public boolean await(ActivityDef activityDef) { + return this.awaitActivity(activityDef); + } public boolean awaitActivity(ActivityDef activityDef) { ActivityExecutor activityExecutor = getActivityExecutor(activityDef, false); if (activityExecutor == null) {