IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

more improvements to command processing

Browse Source
This commit is contained in:
Jonathan Shook 2020-04-22 07:24:34 -05:00
parent 908ab7c1a3
commit 113932aed5
9 changed files with 226 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
engine-api/src/main/java/io/nosqlbench/engine/api/scripting/ScriptEnvBuffer.java
View File

@ -140,6 +140,7 @@ public class ScriptEnvBuffer extends SimpleScriptContext {
private final String prefix;
CharArrayWriter buffer = new CharArrayWriter();
private List<String> timedLog = new ArrayList<String>();
private final StringBuilder sb = new StringBuilder();
public DiagWriter(Writer wrapped, String prefix) {
this.wrapped = wrapped;
@ -152,9 +153,22 @@ public class ScriptEnvBuffer extends SimpleScriptContext {
buffer.write(cbuf, off, len);
String text = new String(cbuf, off, len);
if (!text.equals("\n")) {
String tslogEntry = tsprefix + prefix + new String(cbuf, off, len);
timedLog.add(tslogEntry);
sb.append(text);
if (text.contains("\n")) {
String msgs = sb.toString();
String extra = msgs.substring(msgs.lastIndexOf("\n")+1);
sb.setLength(0);
sb.append(extra);
String[] parts = msgs.substring(0,msgs.length()-extra.length()).split("\n");
for (String part : parts) {
if (!part.isBlank()) {
String tslogEntry = tsprefix + prefix + part + "\n";
timedLog.add(tslogEntry);
}
}
}
wrapped.write(cbuf, off, len);

76
engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java
View File

@ -1,31 +1,40 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Map;
public class BasicScriptBuffer implements ScriptBuffer {
private final static Logger logger = LoggerFactory.getLogger(Cmd.class);
private final StringBuilder sb = new StringBuilder();
private final Map<String, String> scriptParams = new HashMap<>();
@Override
public ScriptBuffer add(Cmd cmd) {
Map<String, String> params = cmd.getParams();
switch (cmd.getCmdType()) {
case script:
sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
sb.append(";\n");
String scriptData = NBCLIScriptAssembly.loadScript(cmd);
// sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
// sb.append(";\n");
combineGlobalParams(scriptParams, cmd);
String scriptData = loadScript(cmd);
sb.append(scriptData);
break;
case fragment:
sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
sb.append(";\n");
// sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
// sb.append(";\n");
combineGlobalParams(scriptParams, cmd);
sb.append(cmd.getArg("script_fragment"));
if (!cmd.getArg("script_fragment").endsWith("\n")) {
if (cmd.getArg("script_fragment").endsWith(";")) {
sb.append("\n");
}
break;
@ -53,10 +62,63 @@ public class BasicScriptBuffer implements ScriptBuffer {
}
/**
* Merge the params from the command into the global params map, but ensure that users know
* if they are overwriting values, which could cause difficult to find bugs in their scripts.
*
* @param scriptParams The existing global params map
* @param cmd The command containing the new params to merge in
*/
private void combineGlobalParams(Map<String, String> scriptParams, Cmd cmd) {
for (String newkey : cmd.getParams().keySet()) {
String newvalue = cmd.getParams().get(newkey);
if (scriptParams.containsKey(newkey)) {
logger.warn("command '" + cmd.getCmdType() + "' overwrote param '" + newkey + " as " + newvalue);
}
scriptParams.put(newkey,newvalue);
}
}
@Override
public ScriptBuffer add(Cmd... cmds) {
for (Cmd cmd : cmds) {
add(cmd);
}
return this;
}
@Override
public String getParsedScript() {
return sb.toString();
}
@Override
public Map<String, String> getCombinedParams() {
return scriptParams;
}
public static String assemble(NBCLIOptions options) {
ScriptBuffer script = new BasicScriptBuffer();
for (Cmd command : options.getCommands()) {
script.add(command);
}
return script.getParsedScript();
}
public static String loadScript(Cmd cmd) {
String scriptData;
String script_path = cmd.getArg("script_path");
logger.debug("Looking for " + script_path);
Content<?> one = NBIO.all().prefix("scripts").name(script_path).extension("js").one();
scriptData = one.asString();
StrInterpolator interpolator = new StrInterpolator(cmd.getParams());
scriptData = interpolator.apply(scriptData);
return scriptData;
}
}

53
engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java
View File

@ -1,7 +1,5 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -16,11 +14,12 @@ import java.util.function.Function;
* An example of a command tha thas both would look like {@code script test.js p1=v1}
*/
public class Cmd {
private final static Logger logger = LoggerFactory.getLogger(Cmd.class);
public enum CmdType {
script(Arg.of("script_path", s -> s)),
fragment(Arg.of("script_fragment")),
fragment(Arg.ofFreeform("script_fragment")),
start(),
run(),
await(Arg.of("alias_name")),
@ -49,18 +48,23 @@ public class Cmd {
private static final class Arg<T> {
public final String name;
public final Function<String, T> converter;
public final boolean freeform;
public Arg(String name, Function<String, T> converter) {
public Arg(String name, Function<String, T> converter, boolean freeform) {
this.name = name;
this.converter = converter;
this.freeform = freeform;
}
public static <T> Arg<T> of(String name, Function<String, T> converter) {
return new Arg<>(name, converter);
return new Arg<>(name, converter, false);
}
public static Arg<String> of(String name) {
return new Arg<>(name, s -> s);
return new Arg<>(name, s -> s, false);
}
public static Arg<String> ofFreeform(String name) {
return new Arg<>(name, s->s, true);
}
}
@ -109,23 +113,28 @@ public class Cmd {
Map<String, String> params = new LinkedHashMap<>();
for (Arg<?> paramName : cmdType.getPositionalArgs()) {
String arg = arglist.peekFirst();
if (arg == null) {
throw new InvalidParameterException("command '" + cmdName + " requires a value for " + paramName
+ ", but there were no remaining arguments after it.");
}
if (arg.contains("=")) {
throw new InvalidParameterException("command '" + cmdName + "' requires a value for " + paramName + "" +
", but a named parameter was found instead: " + arg);
}
if (NBCLIOptions.RESERVED_WORDS.contains(arg)) {
throw new InvalidParameterException("command '" + cmdName + "' requires a value for " + paramName
+ ", but a reserved word was found instead: " + arg);
for (Arg<?> arg : cmdType.getPositionalArgs()) {
String nextarg = arglist.peekFirst();
if (nextarg == null) {
throw new InvalidParameterException(
"command '" + cmdName + " requires a value for " + arg.name
+ ", but there were no remaining arguments after it.");
} else if (arg.freeform) {
logger.debug("freeform parameter:" + nextarg);
} else if (nextarg.contains("=")) {
throw new InvalidParameterException(
"command '" + cmdName + "' requires a value for " + arg.name + "" +
", but a named parameter was found instead: " + nextarg);
} else if (NBCLIOptions.RESERVED_WORDS.contains(nextarg)) {
throw new InvalidParameterException(
"command '" + cmdName + "' requires a value for " + arg.name
+ ", but a reserved word was found instead: " + nextarg);
}
logger.debug("cmd name:" + cmdName + ", positional " + paramName + ": " + arg);
params.put(paramName.name, paramName.converter.apply(arglist.removeFirst()).toString());
logger.debug("cmd name:" + cmdName + ", positional " + arg.name + ": " + nextarg);
params.put(arg.name, arg.converter.apply(arglist.removeFirst()).toString());
}
while (arglist.size() > 0 &&
@ -156,7 +165,7 @@ public class Cmd {
List<String> l = new ArrayList<>();
map.forEach((k, v) -> l.add(
(oneline ? "" : " ") + "'" + k + "'"
+": " + (oneline ? "" : " ".repeat(klen - k.length())) +
+ ": " + (oneline ? "" : " ".repeat(klen - k.length())) +
"'" + v + "'"
));
return "{" + (oneline ? "" : "\n") + String.join(",\n", l) + (oneline ? "}" : "\n}");

9
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -7,6 +7,7 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogIm
import io.nosqlbench.engine.api.activityapi.input.InputType;
import io.nosqlbench.engine.api.activityapi.output.OutputType;
import io.nosqlbench.engine.core.*;
import io.nosqlbench.engine.core.script.ScriptParams;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
@ -243,7 +244,10 @@ public class NBCLI {
ScenariosExecutor executor = new ScenariosExecutor("executor-" + sessionName, 1);
Scenario scenario = new Scenario(sessionName, options.getProgressSpec());
String scriptData = NBCLIScriptAssembly.assemble(options);
ScriptBuffer buffer = new BasicScriptBuffer().add(options.getCommands().toArray(new Cmd[0]));
String scriptData = buffer.getParsedScript();
Map<String,String> globalParams=buffer.getCombinedParams();
if (options.wantsShowScript()) {
System.out.println("// Rendered Script");
System.out.println(scriptData);
@ -268,6 +272,9 @@ public class NBCLI {
Level maxLevel = Level.toLevel(Math.min(clevel.toInt(), llevel.toInt()));
scenario.addScriptText(scriptData);
ScriptParams scriptParams = new ScriptParams();
scriptParams.putAll(buffer.getCombinedParams());
scenario.addScenarioScriptParams(scriptParams);
ScenarioLogger sl = new ScenarioLogger(scenario)
.setLogDir(options.getLogsDirectory())
.setMaxLogs(options.getLogsMax())

25
engine-cli/src/main/java/io/nosqlbench/engine/cli/ScriptBuffer.java
View File

@ -1,9 +1,32 @@
package io.nosqlbench.engine.cli;
import java.util.List;
import java.util.Map;
/**
* Add cmd
*/
public interface ScriptBuffer {
ScriptBuffer add(Cmd cmd);
/**
* Add parsed commands to the script buffer
* @param cmd A parsed command
* @return This ScriptBuffer
*/
ScriptBuffer add(Cmd... cmd);
/**
* Get the text image of the combined script with
* all previously added commands included
* @return The script text
*/
String getParsedScript();
/**
* Get a map which contains all of the params which came from
* commands of global scope, like {@code script} and {@code fragment} commands.
* If one of these commands overwrites a named parameter from another,
* an error should be logged at warning or higher level.
* @return A globa params map.
*/
Map<String, String> getCombinedParams();
}

75
engine-cli/src/test/java/io/nosqlbench/engine/cli/BasicScriptBufferTest.java Normal file
View File

@ -0,0 +1,75 @@
package io.nosqlbench.engine.cli;
import org.junit.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
public class BasicScriptBufferTest {
@Test
public void testScriptInterpolation() {
NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "script_to_interpolate", "parameter1=replaced"});
BasicScriptBuffer b = new BasicScriptBuffer();
b.add(opts.getCommands().toArray(new Cmd[0]));
String s = b.getParsedScript();
assertThat(s).contains("let foo=replaced;");
assertThat(s).contains("let bar=UNSET:parameter2");
}
@Test
public void testAutoScriptCommand() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "acommand" });
BasicScriptBuffer b = new BasicScriptBuffer();
b.add(opts.getCommands().toArray(new Cmd[0]));
String s = b.getParsedScript();
assertThat(s).contains("acommand script text");
}
@Test
public void testScriptParamsSingle() {
NBCLIOptions opts = new NBCLIOptions(new String[] {
"script",
"testscripts/printscript.js",
"param1=value1"
});
BasicScriptBuffer b = new BasicScriptBuffer();
b.add(opts.getCommands().toArray(new Cmd[0]));
String script = b.getParsedScript();
assertThat(script).matches("(?s).*a single line.*");
}
@Test
public void testScriptParamsMulti() {
NBCLIOptions opts = new NBCLIOptions(new String[] {
"script",
"testscripts/printscript.js",
"param1=value1",
"script",
"testscripts/printparam.js",
"paramname=another",
"param2=andanother"
});
BasicScriptBuffer b = new BasicScriptBuffer();
b.add(opts.getCommands().toArray(new Cmd[0]));
String script = b.getParsedScript();
assertThat(script).matches("(?s).*a single line.*");
}
@Test(expected = NumberFormatException.class)
public void shouldThrowErrorForInvalidWaitMillisOperand() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "waitmillis", "noway" });
BasicScriptBuffer b = new BasicScriptBuffer();
b.add(opts.getCommands().toArray(new Cmd[0]));
String s = b.getParsedScript();
}
}

25
engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScriptAssemblyTest.java
View File

@ -24,30 +24,5 @@ import static org.assertj.core.api.Assertions.assertThat;
public class NBCLIScriptAssemblyTest {
@Test
public void testScriptParamsSingle() {
NBCLIOptions opts = new NBCLIOptions(new String[] {
"script",
"testscripts/printscript.js",
"param1=value1"
});
String script = NBCLIScriptAssembly.assemble(opts);
assertThat(script).matches("(?s).*a single line.*");
}
@Test
public void testScriptParamsMulti() {
NBCLIOptions opts = new NBCLIOptions(new String[] {
"script",
"testscripts/printscript.js",
"param1=value1",
"script",
"testscripts/printparam.js",
"paramname=another",
"param2=andanother"
});
String script = NBCLIScriptAssembly.assemble(opts);
assertThat(script).matches("(?s).*a single line.*");
}
}

23
engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java
View File

@ -112,21 +112,6 @@ public class TestNBCLIOptions {
NBCLIOptions opts = new NBCLIOptions(new String[]{"script"});
}
@Test
public void testScriptInterpolation() {
NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "script_to_interpolate", "parameter1=replaced"});
String s = NBCLIScriptAssembly.assemble(opts);
assertThat(s).contains("let foo=replaced;");
assertThat(s).contains("let bar=UNSET:parameter2");
}
@Test
public void testAutoScriptCommand() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "acommand" });
String s = NBCLIScriptAssembly.assemble(opts);
assertThat(s).contains("acommand script text");
}
@Test
public void shouldRecognizeStartActivityCmd() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "start", "driver=woot" });
@ -186,14 +171,6 @@ public class TestNBCLIOptions {
}
@Test(expected = NumberFormatException.class)
public void shouldThrowErrorForInvalidWaitMillisOperand() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "waitmillis", "noway" });
List<Cmd> cmds = opts.getCommands();
NBCLIScriptAssembly.assemble(opts);
}
@Test
public void listWorkloads() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "--list-workloads"});

2
engine-core/src/main/java/io/nosqlbench/engine/core/script/ScriptParams.java
View File

@ -21,12 +21,14 @@ import java.util.HashMap;
import java.util.Map;
public class ScriptParams extends HashMap<String,String> {
public Map<String,String> withOverrides(Map<String,String> overrides) {
HashMap<String,String> result = new HashMap<>();
result.putAll(this);
result.putAll(overrides);
return result;
}
public Map<String,String> withDefaults(Map<String,String> defaults) {
HashMap<String,String> result = new HashMap<>();
result.putAll(defaults);