mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
new feature WIP nb ui
This commit is contained in:
phact
@@ -0,0 +1,300 @@
|
||||
package io.nosqlbench.engine.api.scenarios;
|
||||
|
||||
import io.nosqlbench.docsys.core.PathWalker;
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.util.Synonyms;
|
||||
import io.nosqlbench.nb.api.content.Content;
|
||||
import io.nosqlbench.nb.api.content.NBIO;
|
||||
import io.nosqlbench.nb.api.content.fluent.NBPathsAPI;
|
||||
import io.nosqlbench.nb.api.errors.BasicError;
|
||||
import io.nosqlbench.nb.api.pathutil.NBPaths;
|
||||
import io.nosqlbench.engine.api.util.StrInterpolator;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class NBCLIScenarioParser {
|
||||
|
||||
public final static String SILENT_LOCKED = "==";
|
||||
public final static String VERBOSE_LOCKED = "===";
|
||||
public final static String UNLOCKED = "=";
|
||||
|
||||
private final static Logger logger = LoggerFactory.getLogger(NBCLIScenarioParser.class);
|
||||
|
||||
public static boolean isFoundWorkload(String word) {
|
||||
Optional<Content<?>> found = NBIO.all().prefix("activities").exact().name(word).extension("yaml").first();
|
||||
return found.isPresent();
|
||||
// Optional<Path> workloadPath = NBPathOldUtil.findOptionalPathIn(word, "yaml", false, "activities");
|
||||
// return workloadPath.isPresent();
|
||||
}
|
||||
|
||||
public static void parseScenarioCommand(LinkedList<String> arglist, Set<String> RESERVED_WORDS) {
|
||||
|
||||
String workloadName = arglist.removeFirst();
|
||||
Optional<Path> workloadPathSearch = NBPaths.findOptionalPath(workloadName, "yaml", false, "activities");
|
||||
Path workloadPath = workloadPathSearch.orElseThrow();
|
||||
|
||||
List<String> scenarioNames = new ArrayList<>();
|
||||
while (arglist.size() > 0
|
||||
&& !arglist.peekFirst().contains("=")
|
||||
&& !arglist.peekFirst().startsWith("-")
|
||||
&& RESERVED_WORDS.contains(arglist.peekFirst())) {
|
||||
scenarioNames.add(arglist.removeFirst());
|
||||
}
|
||||
if (scenarioNames.size() == 0) {
|
||||
scenarioNames.add("default");
|
||||
}
|
||||
|
||||
// Load in user's CLI options
|
||||
LinkedHashMap<String, String> userParams = new LinkedHashMap<>();
|
||||
while (arglist.size() > 0
|
||||
&& arglist.peekFirst().contains("=")
|
||||
&& !arglist.peekFirst().startsWith("-")) {
|
||||
String[] arg = arglist.removeFirst().split("=");
|
||||
arg[0] = Synonyms.canonicalize(arg[0], logger);
|
||||
if (userParams.containsKey(arg[0])) {
|
||||
throw new BasicError("duplicate occurrence of option on command line: " + arg[0]);
|
||||
}
|
||||
userParams.put(arg[0], arg[1]);
|
||||
}
|
||||
StrInterpolator userParamsInterp = new StrInterpolator(userParams);
|
||||
|
||||
// This will hold the command to be prepended to the main arglist
|
||||
LinkedList<String> buildCmdBuffer = new LinkedList<>();
|
||||
|
||||
for (String scenarioName : scenarioNames) {
|
||||
|
||||
// Load in named scenario
|
||||
StmtsDocList stmts = StatementsLoader.load(logger, workloadPath.toString());
|
||||
Scenarios scenarios = stmts.getDocScenarios();
|
||||
List<String> cmds = scenarios.getNamedScenario(scenarioName);
|
||||
if (cmds == null) {
|
||||
throw new BasicError("Unable to find named scenario '" + scenarioName + "' in workload '" + workloadName
|
||||
+ "', but you can pick from " + String.join(",", scenarios.getScenarioNames()));
|
||||
}
|
||||
|
||||
Pattern cmdpattern = Pattern.compile("(?<name>\\w+)((?<oper>=+)(?<val>.+))?");
|
||||
for (String cmd : cmds) { // each command line of the named scenario
|
||||
cmd = userParamsInterp.apply(cmd);
|
||||
LinkedHashMap<String, String> usersCopy = new LinkedHashMap<>(userParams);
|
||||
LinkedHashMap<String, CmdArg> cmdline = new LinkedHashMap<>();
|
||||
|
||||
String[] cmdparts = cmd.split(" ");
|
||||
for (String cmdpart : cmdparts) {
|
||||
Matcher matcher = cmdpattern.matcher(cmdpart);
|
||||
if (!matcher.matches()) {
|
||||
throw new BasicError("Unable to recognize scenario cmd spec in '" + cmdpart + "'");
|
||||
}
|
||||
String name = Synonyms.canonicalize(matcher.group("name"), logger);
|
||||
String oper = matcher.group("oper");
|
||||
String val = matcher.group("val");
|
||||
cmdline.put(name, new CmdArg(name, oper, val));
|
||||
}
|
||||
|
||||
LinkedHashMap<String, String> builtcmd = new LinkedHashMap<>();
|
||||
|
||||
for (CmdArg cmdarg : cmdline.values()) {
|
||||
if (usersCopy.containsKey(cmdarg.getName())) {
|
||||
cmdarg = cmdarg.override(usersCopy.remove(cmdarg.getName()));
|
||||
}
|
||||
builtcmd.put(cmdarg.getName(), cmdarg.toString());
|
||||
}
|
||||
usersCopy.forEach((k, v) -> builtcmd.put(k, k + "=" + v));
|
||||
if (!builtcmd.containsKey("workload")) {
|
||||
builtcmd.put("workload", "workload=" + workloadPath.toString());
|
||||
}
|
||||
|
||||
// Undefine any keys with a value of 'undef'
|
||||
List<String> undefKeys = builtcmd.entrySet()
|
||||
.stream()
|
||||
.filter(e -> e.getValue().toLowerCase().endsWith("=undef"))
|
||||
.map(Map.Entry::getKey)
|
||||
.collect(Collectors.toList());
|
||||
undefKeys.forEach(builtcmd::remove);
|
||||
|
||||
logger.debug("Named scenario built command: " + String.join(" ", builtcmd.values()));
|
||||
buildCmdBuffer.addAll(builtcmd.values());
|
||||
}
|
||||
|
||||
}
|
||||
buildCmdBuffer.descendingIterator().forEachRemaining(arglist::addFirst);
|
||||
|
||||
}
|
||||
|
||||
private final static class CmdArg {
|
||||
private final String name;
|
||||
private final String operator;
|
||||
private final String value;
|
||||
private String scenarioName;
|
||||
|
||||
public CmdArg(String name, String operator, String value) {
|
||||
this.name = name;
|
||||
this.operator = operator;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public boolean isReassignable() {
|
||||
return "=".equals(operator);
|
||||
}
|
||||
public boolean isFinalSilent() {
|
||||
return "==".equals(operator);
|
||||
}
|
||||
public boolean isFinalVerbose() {
|
||||
return "===".equals(operator);
|
||||
}
|
||||
|
||||
|
||||
public CmdArg override(String value) {
|
||||
if (isReassignable()) {
|
||||
return new CmdArg(this.name, this.operator, value);
|
||||
} else if (isFinalSilent()) {
|
||||
return this;
|
||||
} else if (isFinalVerbose()) {
|
||||
throw new BasicError("Unable to reassign value for locked param '" + name + operator + value + "'");
|
||||
} else {
|
||||
throw new RuntimeException("impossible!");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return name + (operator != null ? "=" : "") + (value != null ? value : "");
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
}
|
||||
|
||||
// private static void parseWorkloadYamlCmds(String yamlPath, LinkedList<String> arglist, String scenarioName) {
|
||||
// StmtsDocList stmts = StatementsLoader.load(logger, yamlPath);
|
||||
//
|
||||
// Scenarios scenarios = stmts.getDocScenarios();
|
||||
//
|
||||
// String scenarioName = "default";
|
||||
// if (scenarioName != null) {
|
||||
// scenarioName = scenarioName;
|
||||
// }
|
||||
//
|
||||
// List<String> cmds = scenarios.getNamedScenario(scenarioName);
|
||||
//
|
||||
//
|
||||
// Map<String, String> paramMap = new HashMap<>();
|
||||
// while (arglist.size() > 0 && arglist.peekFirst().contains("=")) {
|
||||
// String arg = arglist.removeFirst();
|
||||
// String oldArg = arg;
|
||||
// arg = Synonyms.canonicalize(arg, logger);
|
||||
//
|
||||
// for (int i = 0; i < cmds.size(); i++) {
|
||||
// String yamlCmd = cmds.get(i);
|
||||
// String[] argArray = arg.split("=");
|
||||
// String argKey = argArray[0];
|
||||
// String argValue = argArray[1];
|
||||
// if (!yamlCmd.contains(argKey)) {
|
||||
// cmds.set(i, yamlCmd + " " + arg);
|
||||
// } else {
|
||||
// paramMap.put(argKey, argValue);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if (cmds == null) {
|
||||
// List<String> names = scenarios.getScenarioNames();
|
||||
// throw new RuntimeException("Unknown scenario name, make sure the scenario name you provide exists in the workload definition (yaml):\n" + String.join(",", names));
|
||||
// }
|
||||
//
|
||||
// for (String cmd : cmds) {
|
||||
// String[] cmdArray = cmd.split(" ");
|
||||
//
|
||||
// for (String parameter : cmdArray) {
|
||||
// if (parameter.contains("=")) {
|
||||
// if (!parameter.contains("TEMPLATE(") && !parameter.contains("<<")) {
|
||||
// String[] paramArray = parameter.split("=");
|
||||
// paramMap.put(paramArray[0], paramArray[1]);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// StrSubstitutor sub1 = new StrSubstitutor(paramMap, "<<", ">>", '\\', ",");
|
||||
// StrSubstitutor sub2 = new StrSubstitutor(paramMap, "TEMPLATE(", ")", '\\', ",");
|
||||
//
|
||||
// cmd = sub2.replace(sub1.replace(cmd));
|
||||
//
|
||||
// if (cmd.contains("yaml=") || cmd.contains("workload=")) {
|
||||
// parse(cmd.split(" "));
|
||||
// } else {
|
||||
// parse((cmd + " workload=" + yamlPath).split(" "));
|
||||
// }
|
||||
//
|
||||
// // Is there a better way to do this than regex?
|
||||
//
|
||||
// }
|
||||
// }
|
||||
|
||||
private static Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
|
||||
private static Pattern templatePattern2 = Pattern.compile("<<(.+?)>>");
|
||||
|
||||
|
||||
public static List<WorkloadDesc> getWorkloadsWithScenarioScripts() {
|
||||
|
||||
String dir = "activities/";
|
||||
|
||||
NBPathsAPI.ForContentSource content = NBIO.all(dir).prefix("activities").exact().(".yaml");
|
||||
/*
|
||||
Path basePath = NBPaths.findPathIn(dir);
|
||||
List<Path> yamlPathList = PathWalker.findAll(basePath)
|
||||
.stream()
|
||||
.filter(f -> f.toString().endsWith(".yaml"))
|
||||
.filter(f -> f.toString().contains("activities"))
|
||||
.collect(Collectors.toList());
|
||||
*/
|
||||
|
||||
List<WorkloadDesc> workloadDescriptions = new ArrayList<>();
|
||||
for (Path yamlPath : yamlPathList) {
|
||||
String substring = yamlPath.toString().substring(1);
|
||||
StmtsDocList stmts = StatementsLoader.load(logger, substring);
|
||||
|
||||
Set<String> templates = new HashSet<>();
|
||||
try {
|
||||
List<String> lines = Files.readAllLines(yamlPath);
|
||||
for (String line : lines) {
|
||||
Matcher matcher = templatePattern.matcher(line);
|
||||
|
||||
while (matcher.find()) {
|
||||
templates.add(matcher.group(1));
|
||||
}
|
||||
matcher = templatePattern2.matcher(line);
|
||||
|
||||
while (matcher.find()) {
|
||||
templates.add(matcher.group(1));
|
||||
}
|
||||
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
Scenarios scenarios = stmts.getDocScenarios();
|
||||
|
||||
List<String> scenarioNames = scenarios.getScenarioNames();
|
||||
|
||||
if (scenarioNames != null && scenarioNames.size() >0){
|
||||
workloadDescriptions.add(new WorkloadDesc(yamlPath.getFileName().toString(), scenarioNames, templates));
|
||||
}
|
||||
}
|
||||
|
||||
return workloadDescriptions;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package io.nosqlbench.engine.api.scenarios;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public class WorkloadDesc {
|
||||
private final String yamlPath;
|
||||
private final List<String> scenarioNames;
|
||||
private final Set<String> templates;
|
||||
|
||||
public WorkloadDesc(String yamlPath, List<String> scenarioNames, Set<String> templates) {
|
||||
this.yamlPath = yamlPath;
|
||||
this.scenarioNames = scenarioNames;
|
||||
this.templates = templates;
|
||||
}
|
||||
|
||||
public String getYamlPath() {
|
||||
return yamlPath;
|
||||
}
|
||||
|
||||
public String getWorkloadName(){
|
||||
return getYamlPath().replaceAll("\\.yaml", "");
|
||||
|
||||
}
|
||||
|
||||
public List<String> getScenarioNames() {
|
||||
return scenarioNames;
|
||||
}
|
||||
|
||||
public Set<String> getTemplates() {
|
||||
return templates;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package io.nosqlbench.engine.api.util;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* This class is just a central reference point for the names of parameters
|
||||
* or other configuration-level primitives which have been given better names.
|
||||
* For the sake of backwards compatibility, the old names are retained, but
|
||||
* deprecated and warned against.
|
||||
*/
|
||||
public class Synonyms {
|
||||
|
||||
/**
|
||||
* Each entry in this list is a list of synonyms in configuration.
|
||||
*/
|
||||
public final static Map<String, Set<String>> PARAM_SYNONYMS = new HashMap<>() {{
|
||||
put("hosts",Set.of("host"));
|
||||
put("workload",Set.of("yaml"));
|
||||
put("driver",Set.of("type"));
|
||||
put("cyclerate",Set.of("targetrate"));
|
||||
}};
|
||||
|
||||
/**
|
||||
* use this method to convert deprecated
|
||||
* @param input A configuration string from a user or file
|
||||
* @param synonyms A list of known synonym lists with the preferred values first, like {@link #PARAM_SYNONYMS}
|
||||
* @param warnings An BiConsumer which can handle (deprecated, preferred) for subsitutions.
|
||||
* @return The configuration string in canonicalized form, with the preferred names used where possible
|
||||
*/
|
||||
public static String canonicalize(String input, Map<String, Set<String>> synonyms, BiConsumer<String,String> warnings) {
|
||||
String replaced = input;
|
||||
for (Map.Entry<String, Set<String>> syns : synonyms.entrySet()) {
|
||||
String preferred = syns.getKey();
|
||||
for (String deprecated : syns.getValue()) {
|
||||
Pattern p = Pattern.compile("\\b" + deprecated + "\\b");
|
||||
String prior = replaced;
|
||||
replaced = replaced.replaceAll(p.pattern(),preferred);
|
||||
if (!prior.equals(replaced) && warnings!=null) {
|
||||
warnings.accept(deprecated,preferred);
|
||||
}
|
||||
}
|
||||
}
|
||||
return replaced;
|
||||
}
|
||||
|
||||
public static String canonicalize(String arg, Logger logger) {
|
||||
return canonicalize(arg, PARAM_SYNONYMS, (d, p) -> logger.warn(
|
||||
"Identified deprecated use of '" + d + ", please use '" + p + "' as the preferred form to remove this warning."
|
||||
));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user