more improvements to command processing

engine-api/src/main/java/io/nosqlbench/engine/api/scripting/ScriptEnvBuffer.java

@@ -140,6 +140,7 @@ public class ScriptEnvBuffer extends SimpleScriptContext {
         private final String prefix;
         CharArrayWriter buffer = new CharArrayWriter();
         private List<String> timedLog = new ArrayList<String>();
+        private final StringBuilder sb = new StringBuilder();
 
         public DiagWriter(Writer wrapped, String prefix) {
             this.wrapped = wrapped;
@@ -152,10 +153,23 @@ public class ScriptEnvBuffer extends SimpleScriptContext {
 
             buffer.write(cbuf, off, len);
             String text = new String(cbuf, off, len);
-            if (!text.equals("\n")) {
+
-                String tslogEntry = tsprefix + prefix + new String(cbuf, off, len);
+            sb.append(text);
+
+            if (text.contains("\n")) {
+                String msgs = sb.toString();
+                String extra = msgs.substring(msgs.lastIndexOf("\n")+1);
+                sb.setLength(0);
+                sb.append(extra);
+                String[] parts = msgs.substring(0,msgs.length()-extra.length()).split("\n");
+                for (String part : parts) {
+                    if (!part.isBlank()) {
+                        String tslogEntry = tsprefix + prefix + part + "\n";
                         timedLog.add(tslogEntry);
                     }
+                }
+
+            }
 
             wrapped.write(cbuf, off, len);
         }

engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java

@@ -1,31 +1,40 @@
 package io.nosqlbench.engine.cli;
 
+import io.nosqlbench.engine.api.templating.StrInterpolator;
+import io.nosqlbench.nb.api.content.Content;
+import io.nosqlbench.nb.api.content.NBIO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.util.HashMap;
 import java.util.Map;
 
 
 public class BasicScriptBuffer implements ScriptBuffer {
 
+    private final static Logger logger = LoggerFactory.getLogger(Cmd.class);
+
     private final StringBuilder sb = new StringBuilder();
     private final Map<String, String> scriptParams = new HashMap<>();
 
 
-    @Override
     public ScriptBuffer add(Cmd cmd) {
         Map<String, String> params = cmd.getParams();
 
         switch (cmd.getCmdType()) {
             case script:
-                sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
+//                sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
-                sb.append(";\n");
+//                sb.append(";\n");
-                String scriptData = NBCLIScriptAssembly.loadScript(cmd);
+                combineGlobalParams(scriptParams, cmd);
+                String scriptData = loadScript(cmd);
                 sb.append(scriptData);
                 break;
             case fragment:
-                sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
+//                sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
-                sb.append(";\n");
+//                sb.append(";\n");
+                combineGlobalParams(scriptParams, cmd);
                 sb.append(cmd.getArg("script_fragment"));
-                if (!cmd.getArg("script_fragment").endsWith("\n")) {
+                if (cmd.getArg("script_fragment").endsWith(";")) {
                     sb.append("\n");
                 }
                 break;
@@ -53,10 +62,63 @@ public class BasicScriptBuffer implements ScriptBuffer {
 
     }
 
+    /**
+     * Merge the params from the command into the global params map, but ensure that users know
+     * if they are overwriting values, which could cause difficult to find bugs in their scripts.
+     *
+     * @param scriptParams The existing global params map
+     * @param cmd The command containing the new params to merge in
+     */
+    private void combineGlobalParams(Map<String, String> scriptParams, Cmd cmd) {
+        for (String newkey : cmd.getParams().keySet()) {
+            String newvalue = cmd.getParams().get(newkey);
+
+            if (scriptParams.containsKey(newkey)) {
+                logger.warn("command '" + cmd.getCmdType() + "' overwrote param '" + newkey + " as " + newvalue);
+            }
+            scriptParams.put(newkey,newvalue);
+        }
+    }
+
+    @Override
+    public ScriptBuffer add(Cmd... cmds) {
+        for (Cmd cmd : cmds) {
+            add(cmd);
+        }
+        return this;
+    }
+
     @Override
     public String getParsedScript() {
         return sb.toString();
     }
 
+    @Override
+    public Map<String, String> getCombinedParams() {
+        return scriptParams;
+    }
+
+    public static String assemble(NBCLIOptions options) {
+        ScriptBuffer script = new BasicScriptBuffer();
+        for (Cmd command : options.getCommands()) {
+            script.add(command);
+        }
+        return script.getParsedScript();
+    }
+
+    public static String loadScript(Cmd cmd) {
+        String scriptData;
+        String script_path = cmd.getArg("script_path");
+
+        logger.debug("Looking for " + script_path);
+
+        Content<?> one = NBIO.all().prefix("scripts").name(script_path).extension("js").one();
+        scriptData = one.asString();
+
+        StrInterpolator interpolator = new StrInterpolator(cmd.getParams());
+        scriptData = interpolator.apply(scriptData);
+        return scriptData;
+    }
+
 
 }

engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java

@@ -1,7 +1,5 @@
 package io.nosqlbench.engine.cli;
 
-import io.nosqlbench.nb.api.content.Content;
-import io.nosqlbench.nb.api.content.NBIO;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -16,11 +14,12 @@ import java.util.function.Function;
  * An example of a command tha thas both would look like {@code script test.js p1=v1}
  */
 public class Cmd {
+
     private final static Logger logger = LoggerFactory.getLogger(Cmd.class);
 
     public enum CmdType {
         script(Arg.of("script_path", s -> s)),
-        fragment(Arg.of("script_fragment")),
+        fragment(Arg.ofFreeform("script_fragment")),
         start(),
         run(),
         await(Arg.of("alias_name")),
@@ -49,18 +48,23 @@ public class Cmd {
     private static final class Arg<T> {
         public final String name;
         public final Function<String, T> converter;
+        public final boolean freeform;
 
-        public Arg(String name, Function<String, T> converter) {
+        public Arg(String name, Function<String, T> converter, boolean freeform) {
             this.name = name;
             this.converter = converter;
+            this.freeform = freeform;
         }
 
         public static <T> Arg<T> of(String name, Function<String, T> converter) {
-            return new Arg<>(name, converter);
+            return new Arg<>(name, converter, false);
         }
 
         public static Arg<String> of(String name) {
-            return new Arg<>(name, s -> s);
+            return new Arg<>(name, s -> s, false);
+        }
+        public static Arg<String> ofFreeform(String name) {
+            return new Arg<>(name, s->s, true);
         }
     }
 
@@ -109,23 +113,28 @@ public class Cmd {
 
         Map<String, String> params = new LinkedHashMap<>();
 
-        for (Arg<?> paramName : cmdType.getPositionalArgs()) {
+        for (Arg<?> arg : cmdType.getPositionalArgs()) {
-            String arg = arglist.peekFirst();
+
-            if (arg == null) {
+            String nextarg = arglist.peekFirst();
-                throw new InvalidParameterException("command '" + cmdName + " requires a value for " + paramName
+
+            if (nextarg == null) {
+                throw new InvalidParameterException(
+                    "command '" + cmdName + " requires a value for " + arg.name
                         + ", but there were no remaining arguments after it.");
-            }
+            } else if (arg.freeform) {
-            if (arg.contains("=")) {
+                logger.debug("freeform parameter:" + nextarg);
-                throw new InvalidParameterException("command '" + cmdName + "' requires a value for " + paramName + "" +
+            } else if (nextarg.contains("=")) {
-                    ", but a named parameter was found instead: " + arg);
+                throw new InvalidParameterException(
-            }
+                    "command '" + cmdName + "' requires a value for " + arg.name + "" +
-            if (NBCLIOptions.RESERVED_WORDS.contains(arg)) {
+                        ", but a named parameter was found instead: " + nextarg);
-                throw new InvalidParameterException("command '" + cmdName + "' requires a value for " + paramName
+            } else if (NBCLIOptions.RESERVED_WORDS.contains(nextarg)) {
-                    + ", but a reserved word was found instead: " + arg);
+                throw new InvalidParameterException(
+                    "command '" + cmdName + "' requires a value for " + arg.name
+                        + ", but a reserved word was found instead: " + nextarg);
             }
 
-            logger.debug("cmd name:" + cmdName + ", positional " + paramName + ": " + arg);
+            logger.debug("cmd name:" + cmdName + ", positional " + arg.name + ": " + nextarg);
-            params.put(paramName.name, paramName.converter.apply(arglist.removeFirst()).toString());
+            params.put(arg.name, arg.converter.apply(arglist.removeFirst()).toString());
         }
 
         while (arglist.size() > 0 &&
@@ -156,7 +165,7 @@ public class Cmd {
         List<String> l = new ArrayList<>();
         map.forEach((k, v) -> l.add(
             (oneline ? "" : "    ") + "'" + k + "'"
-                +": " + (oneline ? "" : " ".repeat(klen - k.length())) +
+                + ": " + (oneline ? "" : " ".repeat(klen - k.length())) +
                 "'" + v + "'"
         ));
         return "{" + (oneline ? "" : "\n") + String.join(",\n", l) + (oneline ? "}" : "\n}");

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -7,6 +7,7 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogIm
 import io.nosqlbench.engine.api.activityapi.input.InputType;
 import io.nosqlbench.engine.api.activityapi.output.OutputType;
 import io.nosqlbench.engine.core.*;
+import io.nosqlbench.engine.core.script.ScriptParams;
 import io.nosqlbench.nb.api.content.Content;
 import io.nosqlbench.nb.api.content.NBIO;
 import io.nosqlbench.nb.api.errors.BasicError;
@@ -243,7 +244,10 @@ public class NBCLI {
         ScenariosExecutor executor = new ScenariosExecutor("executor-" + sessionName, 1);
 
         Scenario scenario = new Scenario(sessionName, options.getProgressSpec());
-        String scriptData = NBCLIScriptAssembly.assemble(options);
+        ScriptBuffer buffer = new BasicScriptBuffer().add(options.getCommands().toArray(new Cmd[0]));
+        String scriptData = buffer.getParsedScript();
+        Map<String,String> globalParams=buffer.getCombinedParams();
+
         if (options.wantsShowScript()) {
             System.out.println("// Rendered Script");
             System.out.println(scriptData);
@@ -268,6 +272,9 @@ public class NBCLI {
         Level maxLevel = Level.toLevel(Math.min(clevel.toInt(), llevel.toInt()));
 
         scenario.addScriptText(scriptData);
+        ScriptParams scriptParams = new ScriptParams();
+        scriptParams.putAll(buffer.getCombinedParams());
+        scenario.addScenarioScriptParams(scriptParams);
         ScenarioLogger sl = new ScenarioLogger(scenario)
             .setLogDir(options.getLogsDirectory())
             .setMaxLogs(options.getLogsMax())

engine-cli/src/main/java/io/nosqlbench/engine/cli/ScriptBuffer.java

@@ -1,9 +1,32 @@
 package io.nosqlbench.engine.cli;
 
+import java.util.List;
+import java.util.Map;
+
 /**
  * Add cmd
  */
 public interface ScriptBuffer {
-    ScriptBuffer add(Cmd cmd);
+    /**
+     * Add parsed commands to the script buffer
+     * @param cmd A parsed command
+     * @return This ScriptBuffer
+     */
+    ScriptBuffer add(Cmd... cmd);
+
+    /**
+     * Get the text image of the combined script with
+     * all previously added commands included
+     * @return The script text
+     */
     String getParsedScript();
+
+    /**
+     * Get a map which contains all of the params which came from
+     * commands of global scope, like {@code script} and {@code fragment} commands.
+     * If one of these commands overwrites a named parameter from another,
+     * an error should be logged at warning or higher level.
+     * @return A globa params map.
+     */
+    Map<String, String> getCombinedParams();
 }

engine-cli/src/test/java/io/nosqlbench/engine/cli/BasicScriptBufferTest.java

@@ -0,0 +1,75 @@
+package io.nosqlbench.engine.cli;
+
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class BasicScriptBufferTest {
+
+    @Test
+    public void testScriptInterpolation() {
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "script_to_interpolate", "parameter1=replaced"});
+
+        BasicScriptBuffer b = new BasicScriptBuffer();
+        b.add(opts.getCommands().toArray(new Cmd[0]));
+        String s = b.getParsedScript();
+
+        assertThat(s).contains("let foo=replaced;");
+        assertThat(s).contains("let bar=UNSET:parameter2");
+    }
+
+    @Test
+    public void testAutoScriptCommand() {
+        NBCLIOptions opts = new NBCLIOptions(new String[]{ "acommand" });
+        BasicScriptBuffer b = new BasicScriptBuffer();
+        b.add(opts.getCommands().toArray(new Cmd[0]));
+        String s = b.getParsedScript();
+
+        assertThat(s).contains("acommand script text");
+    }
+
+    @Test
+    public void testScriptParamsSingle() {
+        NBCLIOptions opts = new NBCLIOptions(new String[] {
+            "script",
+            "testscripts/printscript.js",
+            "param1=value1"
+        });
+        BasicScriptBuffer b = new BasicScriptBuffer();
+        b.add(opts.getCommands().toArray(new Cmd[0]));
+        String script = b.getParsedScript();
+
+        assertThat(script).matches("(?s).*a single line.*");
+    }
+
+    @Test
+    public void testScriptParamsMulti() {
+        NBCLIOptions opts = new NBCLIOptions(new String[] {
+            "script",
+            "testscripts/printscript.js",
+            "param1=value1",
+            "script",
+            "testscripts/printparam.js",
+            "paramname=another",
+            "param2=andanother"
+        });
+        BasicScriptBuffer b = new BasicScriptBuffer();
+        b.add(opts.getCommands().toArray(new Cmd[0]));
+        String script = b.getParsedScript();
+
+        assertThat(script).matches("(?s).*a single line.*");
+    }
+
+    @Test(expected = NumberFormatException.class)
+    public void shouldThrowErrorForInvalidWaitMillisOperand() {
+        NBCLIOptions opts = new NBCLIOptions(new String[]{ "waitmillis", "noway" });
+        BasicScriptBuffer b = new BasicScriptBuffer();
+        b.add(opts.getCommands().toArray(new Cmd[0]));
+        String s = b.getParsedScript();
+    }
+
+
+
+}

engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScriptAssemblyTest.java

@@ -24,30 +24,5 @@ import static org.assertj.core.api.Assertions.assertThat;
 
 public class NBCLIScriptAssemblyTest {
 
-    @Test
-    public void testScriptParamsSingle() {
-        NBCLIOptions opts = new NBCLIOptions(new String[] {
-                "script",
-                "testscripts/printscript.js",
-                "param1=value1"
-        });
-        String script = NBCLIScriptAssembly.assemble(opts);
-        assertThat(script).matches("(?s).*a single line.*");
-    }
-
-    @Test
-    public void testScriptParamsMulti() {
-        NBCLIOptions opts = new NBCLIOptions(new String[] {
-                "script",
-                "testscripts/printscript.js",
-                "param1=value1",
-                "script",
-                "testscripts/printparam.js",
-                "paramname=another",
-                "param2=andanother"
-        });
-        String script = NBCLIScriptAssembly.assemble(opts);
-        assertThat(script).matches("(?s).*a single line.*");
-    }
 
 }

engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java

@@ -112,21 +112,6 @@ public class TestNBCLIOptions {
         NBCLIOptions opts = new NBCLIOptions(new String[]{"script"});
     }
 
-    @Test
-    public void testScriptInterpolation() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "script_to_interpolate", "parameter1=replaced"});
-        String s = NBCLIScriptAssembly.assemble(opts);
-        assertThat(s).contains("let foo=replaced;");
-        assertThat(s).contains("let bar=UNSET:parameter2");
-    }
-
-    @Test
-    public void testAutoScriptCommand() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "acommand" });
-        String s = NBCLIScriptAssembly.assemble(opts);
-        assertThat(s).contains("acommand script text");
-    }
-
     @Test
     public void shouldRecognizeStartActivityCmd() {
         NBCLIOptions opts = new NBCLIOptions(new String[]{ "start", "driver=woot" });
@@ -186,14 +171,6 @@ public class TestNBCLIOptions {
 
     }
 
-    @Test(expected = NumberFormatException.class)
-    public void shouldThrowErrorForInvalidWaitMillisOperand() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "waitmillis", "noway" });
-        List<Cmd> cmds = opts.getCommands();
-        NBCLIScriptAssembly.assemble(opts);
-
-    }
-
     @Test
     public void listWorkloads() {
         NBCLIOptions opts = new NBCLIOptions(new String[]{ "--list-workloads"});

engine-core/src/main/java/io/nosqlbench/engine/core/script/ScriptParams.java

@@ -21,12 +21,14 @@ import java.util.HashMap;
 import java.util.Map;
 
 public class ScriptParams extends HashMap<String,String> {
+
     public Map<String,String> withOverrides(Map<String,String> overrides) {
         HashMap<String,String> result = new HashMap<>();
         result.putAll(this);
         result.putAll(overrides);
         return result;
     }
+
     public Map<String,String> withDefaults(Map<String,String> defaults) {
         HashMap<String,String> result = new HashMap<>();
         result.putAll(defaults);