From 0e97ba33b48cfab414b972c9382a168e121c6698 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Thu, 16 Apr 2020 11:14:39 -0500 Subject: [PATCH] fixed canonicalization bug improved output for --list-workloads and --list-scenarios --- .../driver/webdriver/CommandTemplate.java | 9 +++--- .../driver/webdriver/WebDriverActivity.java | 7 +++-- ...{WindowSetSize.java => SetWindowSize.java} | 8 ++--- .../webdriver/verbs/WebDriverVerbs.java | 3 +- .../engine/api/activityimpl/ParameterMap.java | 4 +-- .../api/activityimpl/motor/ParamsParser.java | 23 +++++++++++--- .../activityimpl/motor/ParamsParserTest.java | 30 +++++++++---------- .../java/io/nosqlbench/engine/cli/NBCLI.java | 26 ++++++++-------- nb/pom.xml | 6 ++++ 9 files changed, 70 insertions(+), 46 deletions(-) rename driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/{WindowSetSize.java => SetWindowSize.java} (72%) diff --git a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/CommandTemplate.java b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/CommandTemplate.java index fa56d9500..e29063623 100644 --- a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/CommandTemplate.java +++ b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/CommandTemplate.java @@ -9,6 +9,7 @@ import io.nosqlbench.virtdata.core.templates.StringBindings; import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.nodes.ScalarNode; import java.security.InvalidParameterException; import java.util.LinkedHashMap; @@ -25,10 +26,10 @@ public class CommandTemplate { private final String name; private LinkedHashMap cmdspec = new LinkedHashMap<>(); - public CommandTemplate(StmtDef stmt) { + public CommandTemplate(StmtDef stmt, boolean canonicalize) { this.name = stmt.getName(); String prefixed = "command=" + stmt.getStmt(); - Map cmdMap = ParamsParser.parse(prefixed); + Map cmdMap = ParamsParser.parse(prefixed, canonicalize); Map paramsMap = stmt.getParams(); paramsMap.forEach((k,v) -> { if (cmdMap.containsKey(k)) { @@ -46,9 +47,9 @@ public class CommandTemplate { } - public CommandTemplate(String command, Map bindings, String name) { + public CommandTemplate(String command, Map bindings, String name, boolean canonicalize) { this.name = name; - Map cmdMap = ParamsParser.parse(command); + Map cmdMap = ParamsParser.parse(command, canonicalize); cmdMap.forEach((param,value) -> { ParsedTemplate paramTemplate = new ParsedTemplate(command,bindings); BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints()); diff --git a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java index 000de0526..df4c32f63 100644 --- a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java +++ b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java @@ -14,6 +14,7 @@ import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; import io.nosqlbench.engine.api.activityimpl.ActivityDef; import io.nosqlbench.engine.api.activityimpl.SimpleActivity; import io.nosqlbench.engine.api.templating.StrInterpolator; +import io.nosqlbench.engine.api.util.Synonyms; import io.nosqlbench.nb.api.content.NBIO; import io.nosqlbench.nb.api.errors.BasicError; import joptsimple.internal.Strings; @@ -62,7 +63,7 @@ public class WebDriverActivity extends SimpleActivity { private OpSequence initOpSequence() { OpSequence sequence; - String yaml_loc = getParams().getOptionalString("yaml").orElse(null); + String yaml_loc = getParams().getOptionalString("yaml","workload").orElse(null); String side_loc = getParams().getOptionalString("side").orElse(null); if (yaml_loc == null && side_loc == null) { @@ -161,7 +162,7 @@ public class WebDriverActivity extends SimpleActivity { .orElse(SequencerType.bucket); SequencePlanner planner = new SequencePlanner<>(sequencerType); commands.forEach((name,cmd) -> { - CommandTemplate commandTemplate = new CommandTemplate(cmd, Map.of(), name); + CommandTemplate commandTemplate = new CommandTemplate(cmd, Map.of(), name, false); planner.addOp(commandTemplate,(c) -> 1L); }); OpSequence sequence = planner.resolve(); @@ -188,7 +189,7 @@ public class WebDriverActivity extends SimpleActivity { for (StmtDef optemplate : stmts) { long ratio = Long.parseLong(optemplate.getParams().getOrDefault("ratio", "1")); - CommandTemplate cmd = new CommandTemplate(optemplate); + CommandTemplate cmd = new CommandTemplate(optemplate, false); planner.addOp(cmd, ratio); } return planner.resolve(); diff --git a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/WindowSetSize.java b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/SetWindowSize.java similarity index 72% rename from driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/WindowSetSize.java rename to driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/SetWindowSize.java index 1d30daa06..a02ac0c3e 100644 --- a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/WindowSetSize.java +++ b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/SetWindowSize.java @@ -3,17 +3,17 @@ package io.nosqlbench.driver.webdriver.verbs; import io.nosqlbench.driver.webdriver.WebContext; import org.openqa.selenium.Dimension; -public class WindowSetSize implements WebDriverVerb { +public class SetWindowSize implements WebDriverVerb { private final int width; private final int height; - public WindowSetSize(int width, int height) { + public SetWindowSize(int width, int height) { this.width = width; this.height = height; } - public WindowSetSize(String windowSize) { - String[] xes = windowSize.split("x"); + public SetWindowSize(String size) { + String[] xes = size.split("x"); this.width = Integer.parseInt(xes[0]); this.height = Integer.parseInt(xes[1]); } diff --git a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/WebDriverVerbs.java b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/WebDriverVerbs.java index bce71997e..b16af2083 100644 --- a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/WebDriverVerbs.java +++ b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/WebDriverVerbs.java @@ -20,7 +20,8 @@ public class WebDriverVerbs { open(Open.class, (m) -> new Open(m.get("url"),m.get("target"))), get(Get.class, (m) -> new Get(m.get("target"))), mouseOver(MouseOver.class, (m) -> new MouseOver(Bys.get(m.get("target")))), - mouseOut(MouseOut.class,(m) -> new MouseOut()) + mouseOut(MouseOut.class,(m) -> new MouseOut()), + setWindowSize(SetWindowSize.class,(m) -> new SetWindowSize(m.get("target"))), // find_element(FindElement.class, (m) -> new FindElement(m.get("by"))), // find_elements(FindElements.class, (m) -> new FindElements(m.get("by"))), diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/ParameterMap.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/ParameterMap.java index fda05f759..a82991585 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/ParameterMap.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/ParameterMap.java @@ -253,7 +253,7 @@ public class ParameterMap extends ConcurrentHashMap implements Bi if (encodedParams == null) { throw new RuntimeException("Must provide a non-null String to parse parameters."); } - Map parsedMap = ParamsParser.parse(encodedParams); + Map parsedMap = ParamsParser.parse(encodedParams,true); return new ParameterMap(parsedMap); } @@ -346,4 +346,4 @@ public class ParameterMap extends ConcurrentHashMap implements Bi } } -} \ No newline at end of file +} diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/ParamsParser.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/ParamsParser.java index 73b3d2c19..e26d175f5 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/ParamsParser.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/ParamsParser.java @@ -17,8 +17,12 @@ package io.nosqlbench.engine.api.activityimpl.motor; -import java.util.LinkedHashMap; -import java.util.Map; +import io.nosqlbench.engine.api.activityimpl.ParameterMap; +import io.nosqlbench.engine.api.util.Synonyms; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; /** @@ -71,8 +75,10 @@ import java.util.Map; * */ public class ParamsParser { + private final static Logger logger = LoggerFactory.getLogger(ParamsParser.class); - public static Map parse(String input) { + + public static Map parse(String input, boolean canonicalize) { ParseState s = ParseState.expectingName; @@ -131,7 +137,6 @@ public class ParamsParser { break; case expectingVal: if (c == ' ') { - } else if (c == '\\') { isEscaped = true; } else if (c == '\'') { @@ -215,6 +220,16 @@ public class ParamsParser { throw new RuntimeException("Unable to parse input:" + input); } + if (canonicalize) { + List keys= new ArrayList<>(parms.keySet()); + for (String key : keys) { + String properkey= Synonyms.canonicalize(key,logger); + if (!key.equals(properkey)) { + parms.put(properkey,parms.get(key)); + parms.remove(key); + } + } + } return parms; } diff --git a/engine-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/ParamsParserTest.java b/engine-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/ParamsParserTest.java index a7743ef29..6f1803a93 100644 --- a/engine-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/ParamsParserTest.java +++ b/engine-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/ParamsParserTest.java @@ -28,7 +28,7 @@ public class ParamsParserTest { @Test public void testSimpleParams() { Map p; - p = ParamsParser.parse("a=1;"); + p = ParamsParser.parse("a=1;",true); assertThat(p).hasSize(1); assertThat(p).containsKey("a"); assertThat(p.get("a")).isEqualTo("1"); @@ -37,7 +37,7 @@ public class ParamsParserTest { @Test public void testNullValueParam() { Map p; - p = ParamsParser.parse("a=1;b=;"); + p = ParamsParser.parse("a=1;b=;",true); assertThat(p).hasSize(2); assertThat(p).containsKey("b"); assertThat(p.get("b")).isNull(); @@ -46,7 +46,7 @@ public class ParamsParserTest { @Test public void testSingleQuote() { Map p; - p = ParamsParser.parse("a=1;b='fourfive';"); + p = ParamsParser.parse("a=1;b='fourfive';",true); assertThat(p).hasSize(2); assertThat(p).containsKey("b"); assertThat(p.get("b")).isEqualTo("fourfive"); @@ -55,7 +55,7 @@ public class ParamsParserTest { @Test public void testSingleQuotedEscape() { Map p; - p = ParamsParser.parse("a=1;b='fo\\'urfive';"); + p = ParamsParser.parse("a=1;b='fo\\'urfive';",true); assertThat(p).hasSize(2); assertThat(p).containsKey("b"); assertThat(p.get("b")).isEqualTo("fo'urfive"); @@ -65,7 +65,7 @@ public class ParamsParserTest { @Test public void testDoubleQuote() { Map p; - p = ParamsParser.parse("a=1;b=\"six\";"); + p = ParamsParser.parse("a=1;b=\"six\";",true); assertThat(p).hasSize(2); assertThat(p).containsKey("b"); assertThat(p.get("b")).isEqualTo("six"); @@ -74,7 +74,7 @@ public class ParamsParserTest { @Test public void testDoubleQuotedEscape() { Map p; - p = ParamsParser.parse("a=1;b=\"si\\'x\";"); + p = ParamsParser.parse("a=1;b=\"si\\'x\";",true); assertThat(p).hasSize(2); assertThat(p).containsKey("b"); assertThat(p.get("b")).isEqualTo("si'x"); @@ -83,7 +83,7 @@ public class ParamsParserTest { @Test public void testSQuotesInDQuotes() { Map p; - p = ParamsParser.parse("a=1;b=\"si'x\";"); + p = ParamsParser.parse("a=1;b=\"si'x\";",true); assertThat(p).hasSize(2); assertThat(p).containsKey("b"); assertThat(p.get("b")).isEqualTo("si'x"); @@ -92,7 +92,7 @@ public class ParamsParserTest { @Test public void testDQuotesInSquotes() { Map p; - p = ParamsParser.parse("a=1;b='Sev\"en';"); + p = ParamsParser.parse("a=1;b='Sev\"en';",true); assertThat(p).hasSize(2); assertThat(p).containsKey("b"); assertThat(p.get("b")).isEqualTo("Sev\"en"); @@ -101,7 +101,7 @@ public class ParamsParserTest { @Test public void testSpaces() { Map p; - p = ParamsParser.parse("a=1; b=2;"); + p = ParamsParser.parse("a=1; b=2;",true); assertThat(p).hasSize(2); assertThat(p).containsKey("a"); assertThat(p.get("a")).isEqualTo("1"); @@ -112,7 +112,7 @@ public class ParamsParserTest { @Test public void testMissingSemi() { Map p; - p = ParamsParser.parse("a=1; b=2"); + p = ParamsParser.parse("a=1; b=2",true); assertThat(p).hasSize(2); assertThat(p).containsKey("b"); assertThat(p.get("b")).isEqualTo("2"); @@ -120,7 +120,7 @@ public class ParamsParserTest { @Test public void testSpaceDelimiter() { - Map p = ParamsParser.parse("a=1 b=2"); + Map p = ParamsParser.parse("a=1 b=2",true); assertThat(p).hasSize(2); assertThat(p).containsKey("a"); assertThat(p).containsKey("b"); @@ -131,7 +131,7 @@ public class ParamsParserTest { @Test public void testSpaceDelimiterGappedFirst() { - Map p = ParamsParser.parse("a=1 2 3 b=2"); + Map p = ParamsParser.parse("a=1 2 3 b=2",true); assertThat(p).hasSize(2); assertThat(p).containsKey("a"); assertThat(p).containsKey("b"); @@ -142,7 +142,7 @@ public class ParamsParserTest { @Test public void testSpaceDelimiterGappedLast() { - Map p = ParamsParser.parse("a=1 b=2 3 4"); + Map p = ParamsParser.parse("a=1 b=2 3 4",true); assertThat(p).hasSize(2); assertThat(p).containsKey("a"); assertThat(p).containsKey("b"); @@ -153,7 +153,7 @@ public class ParamsParserTest { @Test(expected = RuntimeException.class) public void testValidNameException() { - Map p = ParamsParser.parse("a=1\\\\;'\";b=2 3 4"); + Map p = ParamsParser.parse("a=1\\\\;'\";b=2 3 4",true); assertThat(p).hasSize(2); assertThat(p).containsKey("a"); assertThat(p).containsKey("b"); @@ -163,7 +163,7 @@ public class ParamsParserTest { @Test public void testSkippingLiteralLeadingSpaces() { - Map p = ParamsParser.parse("a= fiver b=\\ sixer"); + Map p = ParamsParser.parse("a= fiver b=\\ sixer",true); assertThat(p).hasSize(2); assertThat(p).containsKey("a"); assertThat(p).containsKey("b"); diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index 62812ec00..77b0d0e82 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -307,22 +307,22 @@ public class NBCLI { System.out.println(" " + templateEntry.getKey() + " = " + templateEntry.getValue()); } } - System.out.println(); - System.out.println( - "# To see examples to learn from, use\n" + - "# --list-workloads --include examples" - ); - System.out.println( - "# To see included scenarios, use\n" + - "# --list-scenarios shows details." - ); - System.out.println( - "# To copy an example to your local directory, use\n" + - " --copy " - ); + System.out.println("\n"); } } + + if (!includeScenarios) { + System.out.println("\n# To see scenarios scenarios, use --list-scenarios"); + } + + System.out.println( + "\n" + + "# To include examples, add --include=examples\n" + + "# To copy any of these to your local directory, use\n" + + "# --include=examples --copy=examplename\n" + ); + } private String loadHelpFile(String filename) { diff --git a/nb/pom.xml b/nb/pom.xml index 884a62b23..c5b8abbae 100644 --- a/nb/pom.xml +++ b/nb/pom.xml @@ -21,6 +21,12 @@ + + io.nosqlbench + driver-web + 3.12.86-SNAPSHOT + + io.nosqlbench engine-cli