IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-11-25 18:20:33 -06:00
Code Issues Packages Projects Releases Wiki Activity

support --include

Browse Source
This commit is contained in:
Jonathan Shook 2020-04-16 07:58:23 -05:00
parent b0300da068
commit 3bdf674562
4 changed files with 158 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -28,20 +28,25 @@ public class NBCLIScenarioParser {
private final static Logger logger = LoggerFactory.getLogger(NBCLIScenarioParser.class);
private static final String SEARCH_IN = "activities";
public static boolean isFoundWorkload(String workload) {
public static boolean isFoundWorkload(String workload,
String... includes) {
Optional<Content<?>> found = NBIO.all()
.prefix("activities")
.prefix(includes)
.name(workload)
.extension("yaml")
.first();
return found.isPresent();
}
public static void parseScenarioCommand(LinkedList<String> arglist, Set<String> RESERVED_WORDS) {
public static void parseScenarioCommand(LinkedList<String> arglist,
Set<String> RESERVED_WORDS,
String... includes) {
String workloadName = arglist.removeFirst();
Optional<Content<?>> found = NBIO.all()
.prefix("activities")
.prefix(includes)
.name(workloadName)
.extension("yaml")
.first();
@ -84,7 +89,9 @@ public class NBCLIScenarioParser {
for (String scenarioName : scenarioNames) {
// Load in named scenario
Content<?> yamlWithNamedScenarios = NBIO.all().prefix(SEARCH_IN)
Content<?> yamlWithNamedScenarios = NBIO.all()
.prefix(SEARCH_IN)
.prefix(includes)
.name(workloadName)
.extension("yaml")
.one();
@ -269,10 +276,11 @@ public class NBCLIScenarioParser {
private static Pattern templatePattern2 = Pattern.compile("<<(.+?)>>");
public static List<WorkloadDesc> getWorkloadsWithScenarioScripts() {
public static List<WorkloadDesc> getWorkloadsWithScenarioScripts(String... includes) {
List<Content<?>> activities = NBIO.all()
.prefix(SEARCH_IN)
.prefix(includes)
.extension("yaml")
.list();

54
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -96,23 +96,38 @@ public class NBCLI {
}
if (options.wantsWorkloadsList()) {
printWorkloads(false);
printWorkloads(false, options.wantsIncludes());
System.exit(0);
}
if (options.wantsScenariosList()) {
printWorkloads(true);
printWorkloads(true, options.wantsIncludes());
System.exit(0);
}
if (options.wantsToCopyWorkload()) {
String workloadToCopy = options.wantsToCopyWorkloadNamed();
logger.debug("user requests to copy out " + workloadToCopy);
if (options.wantsToCopyResource()) {
String resourceToCopy = options.wantsToCopyResourceNamed();
logger.debug("user requests to copy out " + resourceToCopy);
Optional<Content<?>> tocopy = NBIO.classpath()
.prefix("activities")
.prefix(options.wantsIncludes())
.name(resourceToCopy).extension("yaml").first();
if (tocopy.isEmpty()) {
tocopy = NBIO.classpath()
.prefix().prefix(options.wantsIncludes())
.prefix(options.wantsIncludes())
.name(resourceToCopy).first();
}
Content<?> data = tocopy.orElseThrow(
() -> new BasicError(
"Unable to find " + resourceToCopy +
" in classpath to copy out")
);
Optional<Content<?>> tocopy = NBIO.classpath().prefix("activities")
.name(workloadToCopy).extension("yaml").first();
Content<?> data = tocopy.orElseThrow(() -> new BasicError("Unable to find " + workloadToCopy + " in " +
"classpath to copy out"));
Path writeTo = Path.of(data.asPath().getFileName().toString());
if (Files.exists(writeTo)) {
throw new BasicError("A file named " + writeTo.toString() + " exists. Remove it first.");
@ -267,8 +282,11 @@ public class NBCLI {
}
}
public void printWorkloads(boolean includeScenarios) {
List<WorkloadDesc> workloads = NBCLIScenarioParser.getWorkloadsWithScenarioScripts();
public void printWorkloads(boolean includeScenarios,
String... includes) {
List<WorkloadDesc> workloads =
NBCLIScenarioParser.getWorkloadsWithScenarioScripts(includes);
for (WorkloadDesc workload : workloads) {
System.out.println(workload.getYamlPath());
@ -285,11 +303,23 @@ public class NBCLI {
Map<String, String> templates = workload.getTemplates();
if (templates.size() > 0) {
System.out.println(" # defaults");
for (Map.Entry<String, String> templateEntry: templates.entrySet()) {
for (Map.Entry<String, String> templateEntry : templates.entrySet()) {
System.out.println(" " + templateEntry.getKey() + " = " + templateEntry.getValue());
}
}
System.out.println();
System.out.println(
"# To see examples to learn from, use\n" +
"# --list-workloads --include examples"
);
System.out.println(
"# To see included scenarios, use\n" +
"# --list-scenarios shows details."
);
System.out.println(
"# To copy an example to your local directory, use\n" +
" --copy <path>"
);
}
}

148
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@ -9,7 +9,6 @@ import io.nosqlbench.nb.api.content.NBIO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.InputStream;
import java.security.InvalidParameterException;
import java.util.*;
import java.util.stream.Collectors;
@ -116,6 +115,7 @@ public class NBCLIOptions {
private boolean wantsScenariosList = false;
private String wantsToCopyWorkload = null;
private boolean wantsWorkloadsList = false;
private List<String> wantsToIncludePaths = new ArrayList<String>();
public NBCLIOptions(String[] args) {
parse(args);
@ -132,6 +132,8 @@ public class NBCLIOptions {
return;
}
// Preprocess --include regardless of position
LinkedList<String> nonincludes = new LinkedList<>();
while (arglist.peekFirst() != null) {
String word = arglist.peekFirst();
if (word.startsWith("--") && word.contains("=")) {
@ -141,56 +143,23 @@ public class NBCLIOptions {
arglist.offerFirst(split[0]);
continue;
}
switch (word) {
case INCLUDE:
arglist.removeFirst();
String include = readWordOrThrow(arglist, "path to include");
wantsToIncludePaths.add(include);
break;
case SHOW_SCRIPT:
arglist.removeFirst();
showScript = true;
break;
case SCRIPT_FRAGMENT:
Cmd fragment = parseFragmentCmd(arglist);
cmdList.add(fragment);
break;
case ACTIVITY:
arglist.removeFirst();
arglist.addFirst("run");
case START_ACTIVITY:
case RUN_ACTIVITY:
Cmd activity = parseActivityCmd(arglist);
cmdList.add(activity);
break;
case METRICS:
arglist.removeFirst();
arglist.addFirst("start");
Cmd introspectActivity = parseActivityCmd(arglist);
wantsMetricsForActivity = introspectActivity.cmdSpec;
break;
case AWAIT_ACTIVITY:
String awaitCmdType = arglist.removeFirst();
String activityToAwait = readWordOrThrow(arglist, "activity alias to await");
assertNotParameter(activityToAwait);
assertNotReserved(activityToAwait);
Cmd awaitActivityCmd = new Cmd(CmdType.valueOf(awaitCmdType), activityToAwait);
cmdList.add(awaitActivityCmd);
break;
case STOP_ACTIVITY:
String stopCmdType = readWordOrThrow(arglist, "stop command");
String activityToStop = readWordOrThrow(arglist, "activity alias to await");
assertNotParameter(activityToStop);
assertNotReserved(activityToStop);
Cmd stopActivityCmd = new Cmd(CmdType.valueOf(stopCmdType), activityToStop);
cmdList.add(stopActivityCmd);
break;
case WAIT_MILLIS:
String waitMillisCmdType = readWordOrThrow(arglist, "wait millis");
String millisCount = readWordOrThrow(arglist, "millis count");
Long.parseLong(millisCount); // sanity check
Cmd awaitMillisCmd = new Cmd(CmdType.valueOf(waitMillisCmdType), millisCount);
cmdList.add(awaitMillisCmd);
break;
case SCRIPT:
Cmd cmd = parseScriptCmd(arglist);
cmdList.add(cmd);
break;
case SESSION_NAME:
arglist.removeFirst();
sessionName = readWordOrThrow(arglist, "a session name");
@ -317,12 +286,67 @@ public class NBCLIOptions {
break;
case LIST_WORKLOADS:
arglist.removeFirst();
wantsWorkloadsList =true;
wantsWorkloadsList = true;
break;
case COPY_WORKLOAD:
arglist.removeFirst();
wantsToCopyWorkload = readWordOrThrow(arglist, "workload to copy");
break;
default:
nonincludes.addLast(arglist.removeFirst());
}
}
arglist = nonincludes;
while (arglist.peekFirst() != null) {
String word = arglist.peekFirst();
if (word.startsWith("--") && word.contains("=")) {
String wordToSplit = arglist.removeFirst();
String[] split = wordToSplit.split("=", 2);
arglist.offerFirst(split[1]);
arglist.offerFirst(split[0]);
continue;
}
switch (word) {
case SCRIPT_FRAGMENT:
Cmd fragment = parseFragmentCmd(arglist);
cmdList.add(fragment);
break;
case ACTIVITY:
arglist.removeFirst();
arglist.addFirst("run");
case START_ACTIVITY:
case RUN_ACTIVITY:
Cmd activity = parseActivityCmd(arglist);
cmdList.add(activity);
break;
case AWAIT_ACTIVITY:
String awaitCmdType = arglist.removeFirst();
String activityToAwait = readWordOrThrow(arglist, "activity alias to await");
assertNotParameter(activityToAwait);
assertNotReserved(activityToAwait);
Cmd awaitActivityCmd = new Cmd(CmdType.valueOf(awaitCmdType), activityToAwait);
cmdList.add(awaitActivityCmd);
break;
case STOP_ACTIVITY:
String stopCmdType = readWordOrThrow(arglist, "stop command");
String activityToStop = readWordOrThrow(arglist, "activity alias to await");
assertNotParameter(activityToStop);
assertNotReserved(activityToStop);
Cmd stopActivityCmd = new Cmd(CmdType.valueOf(stopCmdType), activityToStop);
cmdList.add(stopActivityCmd);
break;
case WAIT_MILLIS:
String waitMillisCmdType = readWordOrThrow(arglist, "wait millis");
String millisCount = readWordOrThrow(arglist, "millis count");
Long.parseLong(millisCount); // sanity check
Cmd awaitMillisCmd = new Cmd(CmdType.valueOf(waitMillisCmdType), millisCount);
cmdList.add(awaitMillisCmd);
break;
case SCRIPT:
Cmd cmd = parseScriptCmd(arglist);
cmdList.add(cmd);
break;
default:
Optional<Content<?>> scriptfile = NBIO.local()
.prefix("scripts/auto")
@ -338,8 +362,10 @@ public class NBCLIOptions {
Cmd script = parseScriptCmd(arglist);
cmdList.add(script);
//Scripted yaml
} else if (NBCLIScenarioParser.isFoundWorkload(word)) {
NBCLIScenarioParser.parseScenarioCommand(arglist, RESERVED_WORDS);
} else if (
NBCLIScenarioParser.isFoundWorkload(word, wantsIncludes())
) {
NBCLIScenarioParser.parseScenarioCommand(arglist, RESERVED_WORDS, wantsIncludes());
} else {
throw new InvalidParameterException("unrecognized option:" + word);
}
@ -349,6 +375,10 @@ public class NBCLIOptions {
}
public String[] wantsIncludes() {
return wantsToIncludePaths.toArray(new String[0]);
}
private Map<String, Level> parseLogLevelOverrides(String levelsSpec) {
Map<String, Level> levels = new HashMap<>();
Arrays.stream(levelsSpec.split("[,;]")).forEach(kp -> {
@ -498,8 +528,36 @@ public class NBCLIOptions {
while (arglist.size() > 0 &&
!RESERVED_WORDS.contains(arglist.peekFirst())
&& arglist.peekFirst().contains("=")) {
activitydef.add(arglist.removeFirst());
String arg = arglist.removeFirst();
String yaml = "";
if (arg.startsWith("yaml=")) {
yaml = arg.substring("yaml=".length());
} else if (arg.startsWith("workload=")) {
yaml = arg.substring("workload=".length());
}
if (!yaml.isEmpty()) {
Optional<Content<?>> found = NBIO.local().prefix("activities")
.prefix(wantsMetricsPrefix())
.name(yaml)
.first();
if (found.isPresent()) {
if (!found.get().asPath().toString().equals(yaml)) {
logger.info("rewrote path for " + yaml + " as " + found.get().asPath().toString());
activitydef.add("workload=" + found.get().asPath().toString());
} else {
logger.debug("kept path for " + yaml + " as " + found.get().asPath().toString());
activitydef.add("workload=" + yaml);
}
} else {
logger.debug("unable to find " + yaml + " for pathqualification");
activitydef.add("workload=" + yaml);
}
} else {
activitydef.add(arg);
}
}
return new Cmd(CmdType.valueOf(cmdType), activitydef.stream().map(s -> s + ";").collect(Collectors.joining()));
}
@ -582,7 +640,7 @@ public class NBCLIOptions {
}
public boolean wantsToCopyResource() {
return wantsToCopyWorkload!=null;
return wantsToCopyWorkload != null;
}
public String wantsToCopyResourceNamed() {

2
nb-api/src/main/java/io/nosqlbench/nb/api/content/fluent/NBPathsAPI.java
View File

@ -52,7 +52,7 @@ public interface NBPathsAPI {
* @param prefixPaths A list of paths to include in the search
* @return this builder
*/
GetName prefix(String... prefixPaths);
GetPrefix prefix(String... prefixPaths);
}
public static interface GetName extends GetExtension {