diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/scripting/ScriptEnvBuffer.java b/engine-api/src/main/java/io/nosqlbench/engine/api/scripting/ScriptEnvBuffer.java index be420b956..3906c14bd 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/scripting/ScriptEnvBuffer.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/scripting/ScriptEnvBuffer.java @@ -140,6 +140,7 @@ public class ScriptEnvBuffer extends SimpleScriptContext { private final String prefix; CharArrayWriter buffer = new CharArrayWriter(); private List timedLog = new ArrayList(); + private final StringBuilder sb = new StringBuilder(); public DiagWriter(Writer wrapped, String prefix) { this.wrapped = wrapped; @@ -152,9 +153,22 @@ public class ScriptEnvBuffer extends SimpleScriptContext { buffer.write(cbuf, off, len); String text = new String(cbuf, off, len); - if (!text.equals("\n")) { - String tslogEntry = tsprefix + prefix + new String(cbuf, off, len); - timedLog.add(tslogEntry); + + sb.append(text); + + if (text.contains("\n")) { + String msgs = sb.toString(); + String extra = msgs.substring(msgs.lastIndexOf("\n")+1); + sb.setLength(0); + sb.append(extra); + String[] parts = msgs.substring(0,msgs.length()-extra.length()).split("\n"); + for (String part : parts) { + if (!part.isBlank()) { + String tslogEntry = tsprefix + prefix + part + "\n"; + timedLog.add(tslogEntry); + } + } + } wrapped.write(cbuf, off, len); diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java index cc6ae212a..1ae658157 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java @@ -1,31 +1,40 @@ package io.nosqlbench.engine.cli; +import io.nosqlbench.engine.api.templating.StrInterpolator; +import io.nosqlbench.nb.api.content.Content; +import io.nosqlbench.nb.api.content.NBIO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.HashMap; import java.util.Map; public class BasicScriptBuffer implements ScriptBuffer { + private final static Logger logger = LoggerFactory.getLogger(Cmd.class); + private final StringBuilder sb = new StringBuilder(); private final Map scriptParams = new HashMap<>(); - @Override public ScriptBuffer add(Cmd cmd) { Map params = cmd.getParams(); switch (cmd.getCmdType()) { case script: - sb.append(Cmd.toJSONParams("params", cmd.getParams(), false)); - sb.append(";\n"); - String scriptData = NBCLIScriptAssembly.loadScript(cmd); +// sb.append(Cmd.toJSONParams("params", cmd.getParams(), false)); +// sb.append(";\n"); + combineGlobalParams(scriptParams, cmd); + String scriptData = loadScript(cmd); sb.append(scriptData); break; case fragment: - sb.append(Cmd.toJSONParams("params", cmd.getParams(), false)); - sb.append(";\n"); +// sb.append(Cmd.toJSONParams("params", cmd.getParams(), false)); +// sb.append(";\n"); + combineGlobalParams(scriptParams, cmd); sb.append(cmd.getArg("script_fragment")); - if (!cmd.getArg("script_fragment").endsWith("\n")) { + if (cmd.getArg("script_fragment").endsWith(";")) { sb.append("\n"); } break; @@ -53,10 +62,63 @@ public class BasicScriptBuffer implements ScriptBuffer { } + /** + * Merge the params from the command into the global params map, but ensure that users know + * if they are overwriting values, which could cause difficult to find bugs in their scripts. + * + * @param scriptParams The existing global params map + * @param cmd The command containing the new params to merge in + */ + private void combineGlobalParams(Map scriptParams, Cmd cmd) { + for (String newkey : cmd.getParams().keySet()) { + String newvalue = cmd.getParams().get(newkey); + + if (scriptParams.containsKey(newkey)) { + logger.warn("command '" + cmd.getCmdType() + "' overwrote param '" + newkey + " as " + newvalue); + } + scriptParams.put(newkey,newvalue); + } + } + + @Override + public ScriptBuffer add(Cmd... cmds) { + for (Cmd cmd : cmds) { + add(cmd); + } + return this; + } + @Override public String getParsedScript() { return sb.toString(); } + @Override + public Map getCombinedParams() { + return scriptParams; + } + + public static String assemble(NBCLIOptions options) { + ScriptBuffer script = new BasicScriptBuffer(); + for (Cmd command : options.getCommands()) { + script.add(command); + } + return script.getParsedScript(); + } + + public static String loadScript(Cmd cmd) { + String scriptData; + String script_path = cmd.getArg("script_path"); + + logger.debug("Looking for " + script_path); + + Content one = NBIO.all().prefix("scripts").name(script_path).extension("js").one(); + scriptData = one.asString(); + + StrInterpolator interpolator = new StrInterpolator(cmd.getParams()); + scriptData = interpolator.apply(scriptData); + return scriptData; + } + } diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java index 1a4e10194..d6286b051 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java @@ -1,7 +1,5 @@ package io.nosqlbench.engine.cli; -import io.nosqlbench.nb.api.content.Content; -import io.nosqlbench.nb.api.content.NBIO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,11 +14,12 @@ import java.util.function.Function; * An example of a command tha thas both would look like {@code script test.js p1=v1} */ public class Cmd { + private final static Logger logger = LoggerFactory.getLogger(Cmd.class); public enum CmdType { script(Arg.of("script_path", s -> s)), - fragment(Arg.of("script_fragment")), + fragment(Arg.ofFreeform("script_fragment")), start(), run(), await(Arg.of("alias_name")), @@ -49,18 +48,23 @@ public class Cmd { private static final class Arg { public final String name; public final Function converter; + public final boolean freeform; - public Arg(String name, Function converter) { + public Arg(String name, Function converter, boolean freeform) { this.name = name; this.converter = converter; + this.freeform = freeform; } public static Arg of(String name, Function converter) { - return new Arg<>(name, converter); + return new Arg<>(name, converter, false); } public static Arg of(String name) { - return new Arg<>(name, s -> s); + return new Arg<>(name, s -> s, false); + } + public static Arg ofFreeform(String name) { + return new Arg<>(name, s->s, true); } } @@ -109,23 +113,28 @@ public class Cmd { Map params = new LinkedHashMap<>(); - for (Arg paramName : cmdType.getPositionalArgs()) { - String arg = arglist.peekFirst(); - if (arg == null) { - throw new InvalidParameterException("command '" + cmdName + " requires a value for " + paramName - + ", but there were no remaining arguments after it."); - } - if (arg.contains("=")) { - throw new InvalidParameterException("command '" + cmdName + "' requires a value for " + paramName + "" + - ", but a named parameter was found instead: " + arg); - } - if (NBCLIOptions.RESERVED_WORDS.contains(arg)) { - throw new InvalidParameterException("command '" + cmdName + "' requires a value for " + paramName - + ", but a reserved word was found instead: " + arg); + for (Arg arg : cmdType.getPositionalArgs()) { + + String nextarg = arglist.peekFirst(); + + if (nextarg == null) { + throw new InvalidParameterException( + "command '" + cmdName + " requires a value for " + arg.name + + ", but there were no remaining arguments after it."); + } else if (arg.freeform) { + logger.debug("freeform parameter:" + nextarg); + } else if (nextarg.contains("=")) { + throw new InvalidParameterException( + "command '" + cmdName + "' requires a value for " + arg.name + "" + + ", but a named parameter was found instead: " + nextarg); + } else if (NBCLIOptions.RESERVED_WORDS.contains(nextarg)) { + throw new InvalidParameterException( + "command '" + cmdName + "' requires a value for " + arg.name + + ", but a reserved word was found instead: " + nextarg); } - logger.debug("cmd name:" + cmdName + ", positional " + paramName + ": " + arg); - params.put(paramName.name, paramName.converter.apply(arglist.removeFirst()).toString()); + logger.debug("cmd name:" + cmdName + ", positional " + arg.name + ": " + nextarg); + params.put(arg.name, arg.converter.apply(arglist.removeFirst()).toString()); } while (arglist.size() > 0 && @@ -156,7 +165,7 @@ public class Cmd { List l = new ArrayList<>(); map.forEach((k, v) -> l.add( (oneline ? "" : " ") + "'" + k + "'" - +": " + (oneline ? "" : " ".repeat(klen - k.length())) + + + ": " + (oneline ? "" : " ".repeat(klen - k.length())) + "'" + v + "'" )); return "{" + (oneline ? "" : "\n") + String.join(",\n", l) + (oneline ? "}" : "\n}"); diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index eb8da69cd..b6134334d 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -7,6 +7,7 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogIm import io.nosqlbench.engine.api.activityapi.input.InputType; import io.nosqlbench.engine.api.activityapi.output.OutputType; import io.nosqlbench.engine.core.*; +import io.nosqlbench.engine.core.script.ScriptParams; import io.nosqlbench.nb.api.content.Content; import io.nosqlbench.nb.api.content.NBIO; import io.nosqlbench.nb.api.errors.BasicError; @@ -243,7 +244,10 @@ public class NBCLI { ScenariosExecutor executor = new ScenariosExecutor("executor-" + sessionName, 1); Scenario scenario = new Scenario(sessionName, options.getProgressSpec()); - String scriptData = NBCLIScriptAssembly.assemble(options); + ScriptBuffer buffer = new BasicScriptBuffer().add(options.getCommands().toArray(new Cmd[0])); + String scriptData = buffer.getParsedScript(); + Map globalParams=buffer.getCombinedParams(); + if (options.wantsShowScript()) { System.out.println("// Rendered Script"); System.out.println(scriptData); @@ -268,6 +272,9 @@ public class NBCLI { Level maxLevel = Level.toLevel(Math.min(clevel.toInt(), llevel.toInt())); scenario.addScriptText(scriptData); + ScriptParams scriptParams = new ScriptParams(); + scriptParams.putAll(buffer.getCombinedParams()); + scenario.addScenarioScriptParams(scriptParams); ScenarioLogger sl = new ScenarioLogger(scenario) .setLogDir(options.getLogsDirectory()) .setMaxLogs(options.getLogsMax()) diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/ScriptBuffer.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/ScriptBuffer.java index f377a7766..eca4c7ef7 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/ScriptBuffer.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/ScriptBuffer.java @@ -1,9 +1,32 @@ package io.nosqlbench.engine.cli; +import java.util.List; +import java.util.Map; + /** * Add cmd */ public interface ScriptBuffer { - ScriptBuffer add(Cmd cmd); + /** + * Add parsed commands to the script buffer + * @param cmd A parsed command + * @return This ScriptBuffer + */ + ScriptBuffer add(Cmd... cmd); + + /** + * Get the text image of the combined script with + * all previously added commands included + * @return The script text + */ String getParsedScript(); + + /** + * Get a map which contains all of the params which came from + * commands of global scope, like {@code script} and {@code fragment} commands. + * If one of these commands overwrites a named parameter from another, + * an error should be logged at warning or higher level. + * @return A globa params map. + */ + Map getCombinedParams(); } diff --git a/engine-cli/src/test/java/io/nosqlbench/engine/cli/BasicScriptBufferTest.java b/engine-cli/src/test/java/io/nosqlbench/engine/cli/BasicScriptBufferTest.java new file mode 100644 index 000000000..cca5bfab0 --- /dev/null +++ b/engine-cli/src/test/java/io/nosqlbench/engine/cli/BasicScriptBufferTest.java @@ -0,0 +1,75 @@ +package io.nosqlbench.engine.cli; + +import org.junit.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class BasicScriptBufferTest { + + @Test + public void testScriptInterpolation() { + NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "script_to_interpolate", "parameter1=replaced"}); + + BasicScriptBuffer b = new BasicScriptBuffer(); + b.add(opts.getCommands().toArray(new Cmd[0])); + String s = b.getParsedScript(); + + assertThat(s).contains("let foo=replaced;"); + assertThat(s).contains("let bar=UNSET:parameter2"); + } + + @Test + public void testAutoScriptCommand() { + NBCLIOptions opts = new NBCLIOptions(new String[]{ "acommand" }); + BasicScriptBuffer b = new BasicScriptBuffer(); + b.add(opts.getCommands().toArray(new Cmd[0])); + String s = b.getParsedScript(); + + assertThat(s).contains("acommand script text"); + } + + @Test + public void testScriptParamsSingle() { + NBCLIOptions opts = new NBCLIOptions(new String[] { + "script", + "testscripts/printscript.js", + "param1=value1" + }); + BasicScriptBuffer b = new BasicScriptBuffer(); + b.add(opts.getCommands().toArray(new Cmd[0])); + String script = b.getParsedScript(); + + assertThat(script).matches("(?s).*a single line.*"); + } + + @Test + public void testScriptParamsMulti() { + NBCLIOptions opts = new NBCLIOptions(new String[] { + "script", + "testscripts/printscript.js", + "param1=value1", + "script", + "testscripts/printparam.js", + "paramname=another", + "param2=andanother" + }); + BasicScriptBuffer b = new BasicScriptBuffer(); + b.add(opts.getCommands().toArray(new Cmd[0])); + String script = b.getParsedScript(); + + assertThat(script).matches("(?s).*a single line.*"); + } + + @Test(expected = NumberFormatException.class) + public void shouldThrowErrorForInvalidWaitMillisOperand() { + NBCLIOptions opts = new NBCLIOptions(new String[]{ "waitmillis", "noway" }); + BasicScriptBuffer b = new BasicScriptBuffer(); + b.add(opts.getCommands().toArray(new Cmd[0])); + String s = b.getParsedScript(); + } + + + +} diff --git a/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScriptAssemblyTest.java b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScriptAssemblyTest.java index 25e7a5114..1abe26cbe 100644 --- a/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScriptAssemblyTest.java +++ b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScriptAssemblyTest.java @@ -24,30 +24,5 @@ import static org.assertj.core.api.Assertions.assertThat; public class NBCLIScriptAssemblyTest { - @Test - public void testScriptParamsSingle() { - NBCLIOptions opts = new NBCLIOptions(new String[] { - "script", - "testscripts/printscript.js", - "param1=value1" - }); - String script = NBCLIScriptAssembly.assemble(opts); - assertThat(script).matches("(?s).*a single line.*"); - } - - @Test - public void testScriptParamsMulti() { - NBCLIOptions opts = new NBCLIOptions(new String[] { - "script", - "testscripts/printscript.js", - "param1=value1", - "script", - "testscripts/printparam.js", - "paramname=another", - "param2=andanother" - }); - String script = NBCLIScriptAssembly.assemble(opts); - assertThat(script).matches("(?s).*a single line.*"); - } } diff --git a/engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java b/engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java index 5ffe0380e..ce8f0226a 100644 --- a/engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java +++ b/engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java @@ -112,21 +112,6 @@ public class TestNBCLIOptions { NBCLIOptions opts = new NBCLIOptions(new String[]{"script"}); } - @Test - public void testScriptInterpolation() { - NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "script_to_interpolate", "parameter1=replaced"}); - String s = NBCLIScriptAssembly.assemble(opts); - assertThat(s).contains("let foo=replaced;"); - assertThat(s).contains("let bar=UNSET:parameter2"); - } - - @Test - public void testAutoScriptCommand() { - NBCLIOptions opts = new NBCLIOptions(new String[]{ "acommand" }); - String s = NBCLIScriptAssembly.assemble(opts); - assertThat(s).contains("acommand script text"); - } - @Test public void shouldRecognizeStartActivityCmd() { NBCLIOptions opts = new NBCLIOptions(new String[]{ "start", "driver=woot" }); @@ -186,14 +171,6 @@ public class TestNBCLIOptions { } - @Test(expected = NumberFormatException.class) - public void shouldThrowErrorForInvalidWaitMillisOperand() { - NBCLIOptions opts = new NBCLIOptions(new String[]{ "waitmillis", "noway" }); - List cmds = opts.getCommands(); - NBCLIScriptAssembly.assemble(opts); - - } - @Test public void listWorkloads() { NBCLIOptions opts = new NBCLIOptions(new String[]{ "--list-workloads"}); diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScriptParams.java b/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScriptParams.java index 1d16e4ca7..3523687d5 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScriptParams.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScriptParams.java @@ -21,12 +21,14 @@ import java.util.HashMap; import java.util.Map; public class ScriptParams extends HashMap { + public Map withOverrides(Map overrides) { HashMap result = new HashMap<>(); result.putAll(this); result.putAll(overrides); return result; } + public Map withDefaults(Map defaults) { HashMap result = new HashMap<>(); result.putAll(defaults);