IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

support for param=UNDEF in named scenario templates

Browse Source
This commit is contained in:
Jonathan Shook 2020-04-06 12:06:10 -05:00
parent 034c8a9417
commit becab83c2c
9 changed files with 87 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
activitytype-cql/src/main/resources/activities/cql-iot-dse.yaml
View File

@ -1,7 +1,7 @@
# nb -v run driver=cql yaml=cql-iot tags=phase:schema host=dsehost
scenarios:
default:
- run driver=cql tags==phase:schema threads==1
- run driver=cql tags==phase:schema threads==1 cycles==UNDEF
- run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
- run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
bindings:

2
activitytype-cql/src/main/resources/activities/cql-iot.yaml
View File

@ -1,7 +1,7 @@
# nb -v run driver=cql yaml=cql-iot tags=phase:schema host=dsehost
scenarios:
default:
- run driver=cql tags==phase:schema threads==1
- run driver=cql tags==phase:schema threads==1 cycles==UNDEF
- run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
- run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
params:

2
activitytype-cql/src/main/resources/activities/cql-keyvalue.yaml
View File

@ -1,7 +1,7 @@
# nb -v run driver=cql yaml=cql-keyvalue tags=phase:schema host=dsehost
scenarios:
default:
- run driver=cql tags==phase:schema threads==1
- run driver=cql tags==phase:schema threads==1 cycles==UNDEF
- run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
- run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
bindings:

2
activitytype-cql/src/main/resources/activities/cql-tabular.yaml
View File

@ -1,7 +1,7 @@
# nb -v cql-tabular rampup-cycles=1E6 main-cycles=1E9
scenarios:
default:
- run driver=cql tags==phase:schema threads==1
- run driver=cql tags==phase:schema threads==1 cycles==UNDEF
- run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
- run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
bindings:

64
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -61,12 +61,12 @@ public class NBCLI {
}
public void run(String[] args) {
if (args.length>0 && args[0].toLowerCase().equals("virtdata")) {
VirtDataMainApp.main(Arrays.copyOfRange(args,1,args.length));
if (args.length > 0 && args[0].toLowerCase().equals("virtdata")) {
VirtDataMainApp.main(Arrays.copyOfRange(args, 1, args.length));
System.exit(0);
}
if (args.length>0 && args[0].toLowerCase().equals("docserver")) {
DocServerApp.main(Arrays.copyOfRange(args,1,args.length));
if (args.length > 0 && args[0].toLowerCase().equals("docserver")) {
DocServerApp.main(Arrays.copyOfRange(args, 1, args.length));
System.exit(0);
}
@ -93,8 +93,13 @@ public class NBCLI {
System.exit(0);
}
if (options.wantsWorkloadsList()) {
printWorkloads(false);
System.exit(0);
}
if (options.wantsScenariosList()) {
printWorkloads();
printWorkloads(true);
System.exit(0);
}
@ -111,11 +116,11 @@ public class NBCLI {
throw new BasicError("A file named " + writeTo.toString() + " exists. Remove it first.");
}
try {
Files.writeString(writeTo,data.getCharBuffer(), StandardCharsets.UTF_8);
Files.writeString(writeTo, data.getCharBuffer(), StandardCharsets.UTF_8);
} catch (IOException e) {
throw new BasicError("Unable to write to " + writeTo.toString() + ": " + e.getMessage());
}
logger.info("Copied internal resource '" + data.asPath() + "' to '" + writeTo.toString() +"'");
logger.info("Copied internal resource '" + data.asPath() + "' to '" + writeTo.toString() + "'");
System.exit(0);
}
@ -156,19 +161,19 @@ public class NBCLI {
}
String reportGraphiteTo = options.wantsReportGraphiteTo();
if (options.wantsDockerMetrics()){
if (options.wantsDockerMetrics()) {
logger.info("Docker metrics is enabled. Docker must be installed for this to work");
DockerMetricsManager dmh= new DockerMetricsManager();
DockerMetricsManager dmh = new DockerMetricsManager();
dmh.startMetrics();
logger.info("Docker Containers are started, for grafana and prometheus, hit" +
"these urls in your browser: http://<host>:3000 and http://<host>:9090" +
"the default grafana creds are admin/admin");
if (reportGraphiteTo != null){
if (reportGraphiteTo != null) {
logger.warn(String.format("Docker metrics are enabled (--docker-metrics)" +
" but graphite reporting (--report-graphite-to) is set to %s \n" +
"usually only one of the two is configured.",
reportGraphiteTo));
}else{
} else {
//TODO: is this right?
logger.info("Setting graphite reporting to localhost");
reportGraphiteTo = "localhost:9109";
@ -190,7 +195,7 @@ public class NBCLI {
String sessionName = new SessionNamer().format(options.getSessionName());
if (options.wantsEnableChart()){
if (options.wantsEnableChart()) {
logger.info("Charting enabled");
if (options.getHistoLoggerConfigs().size() == 0) {
logger.info("Adding default histologger configs");
@ -229,10 +234,10 @@ public class NBCLI {
System.exit(0);
}
if (options.wantsEnableChart()){
if (options.wantsEnableChart()) {
logger.info("Charting enabled");
scenario.enableCharting();
} else{
} else {
logger.info("Charting disabled");
}
@ -258,28 +263,33 @@ public class NBCLI {
}
}
public void printWorkloads() {
public void printWorkloads(boolean includeScenarios) {
List<WorkloadDesc> workloads = NBCLIScenarioParser.getWorkloadsWithScenarioScripts();
for (WorkloadDesc workload : workloads) {
System.out.println("\n# from: "+ workload.getYamlPath());
System.out.println(workload.getYamlPath());
if (includeScenarios) {
System.out.println(" # scenarios:");
List<String> scenarioList = workload.getScenarioNames();
String workloadName = workload.getYamlPath().replaceAll("\\.yaml", "") ;
Set<String> templates = workload.getTemplates();
String workloadName = workload.getYamlPath().replaceAll("\\.yaml", "");
for (String scenario : scenarioList) {
if (scenario.equals("default")) {
scenario = scenario + "\n # same as running ./nb " + workloadName ;
System.out.println(" nb " + workloadName + " " + scenario);
}
System.out.println(" ./nb " + workloadName + " " + scenario);
}
if (templates.size()>0){
System.out.println("# with the following optional parameters and defaults: ");
Set<String> templates = workload.getTemplates();
if (templates.size() > 0) {
System.out.println(" # defaults");
templates.stream()
.map(x -> x.replaceAll(",","="))
.map(x -> x.replaceAll(":","="))
.map(x -> " # "+x)
.map(x -> x.replaceAll(",", "="))
.map(x -> x.replaceAll(":", "="))
.map(x -> " " + x)
.forEach(System.out::println);
}
System.out.println();
}
}
}

10
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@ -112,6 +112,7 @@ public class NBCLIOptions {
private boolean dockerMetrics = false;
private boolean wantsScenariosList = false;
private String wantsToCopyWorkload = null;
private boolean wantsWorkloadsList = false;
public NBCLIOptions(String[] args) {
parse(args);
@ -308,10 +309,13 @@ public class NBCLIOptions {
consoleLoggingPattern = readWordOrThrow(arglist, "logging pattern");
break;
case LIST_SCENARIOS:
case LIST_WORKLOADS:
arglist.removeFirst();
wantsScenariosList = true;
break;
case LIST_WORKLOADS:
arglist.removeFirst();
wantsWorkloadsList =true;
break;
case COPY_WORKLOAD:
arglist.removeFirst();
wantsToCopyWorkload = readWordOrThrow(arglist, "workload to copy");
@ -578,6 +582,10 @@ public class NBCLIOptions {
return wantsToCopyWorkload;
}
public boolean wantsWorkloadsList() {
return wantsWorkloadsList;
}
public static enum CmdType {
start,
run,

8
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIScenarioParser.java
View File

@ -111,6 +111,14 @@ public class NBCLIScenarioParser {
builtcmd.put("workload", "workload=" + workloadPath.toString());
}
// Undefine any keys with a value of 'undef'
List<String> undefKeys = builtcmd.entrySet()
.stream()
.filter(e -> e.getValue().toLowerCase().endsWith("=undef"))
.map(Map.Entry::getKey)
.collect(Collectors.toList());
undefKeys.forEach(builtcmd::remove);
logger.debug("Named scenario built command: " + String.join(" ", builtcmd.values()));
buildCmdBuffer.addAll(builtcmd.values());
}

13
engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTest.java
View File

@ -84,5 +84,18 @@ public class NBCLIScenarioParserTest {
assertThat(cmds.get(0).getCmdSpec()).isEqualTo("driver=stdout;cycles=20;cycles-test=20;workload=activities/scenario-test.yaml;");
}
@Test
public void testThatUndefValuesAreUndefined() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "scenario-test", "schema-only", "cycles-test=20"});
List<NBCLIOptions.Cmd> cmds = opts.getCommands();
assertThat(cmds.size()).isEqualTo(1);
assertThat(cmds.get(0).getCmdSpec()).isEqualTo("driver=stdout;workload=scenario-test;tags=phase:schema;cycles-test=20;");
NBCLIOptions opts1 = new NBCLIOptions(new String[]{ "scenario-test", "schema-only", "doundef=20"});
List<NBCLIOptions.Cmd> cmds1 = opts1.getCommands();
assertThat(cmds1.size()).isEqualTo(1);
assertThat(cmds1.get(0).getCmdSpec()).isEqualTo("driver=stdout;workload=scenario-test;tags=phase:schema;");
}
}

2
engine-cli/src/test/resources/activities/scenario-test.yaml
View File

@ -5,7 +5,7 @@ scenarios:
- run driver=stdout yaml===scenario-test tags=phase:rampup cycles=TEMPLATE(cycles1,10)
- run driver=stdout yaml===scenario-test tags=phase:main cycles=TEMPLATE(cycles2,10)
schema-only:
- "run driver=stdout yaml=scenario-test tags=phase:schema"
- "run driver=stdout yaml=scenario-test tags=phase:schema doundef==undef"
template-test:
with-template: run driver=stdout cycles=TEMPLATE(cycles-test,10)
blocks: