IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

sanitize named scenario aliasing

Browse Source
This commit is contained in:
Jonathan Shook 2020-05-31 23:58:16 -05:00
parent b248c86684
commit d64c084092
2 changed files with 60 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -8,7 +8,6 @@ import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.nb.api.content.Content; import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO; import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.DiscreteProbabilityBuffer;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -34,11 +33,11 @@ public class NBCLIScenarioParser {
public static boolean isFoundWorkload(String workload, public static boolean isFoundWorkload(String workload,
String... includes) { String... includes) {
Optional<Content<?>> found = NBIO.all() Optional<Content<?>> found = NBIO.all()
.prefix("activities") .prefix("activities")
.prefix(includes) .prefix(includes)
.name(workload) .name(workload)
.extension("yaml") .extension("yaml")
.first(); .first();
return found.isPresent(); return found.isPresent();
} }
@ -48,11 +47,11 @@ public class NBCLIScenarioParser {
String workloadName = arglist.removeFirst(); String workloadName = arglist.removeFirst();
Optional<Content<?>> found = NBIO.all() Optional<Content<?>> found = NBIO.all()
.prefix("activities") .prefix("activities")
.prefix(includes) .prefix(includes)
.name(workloadName) .name(workloadName)
.extension("yaml") .extension("yaml")
.first(); .first();
// //
Content<?> workloadContent = found.orElseThrow(); Content<?> workloadContent = found.orElseThrow();
@ -62,9 +61,9 @@ public class NBCLIScenarioParser {
// Buffer in CLI word from user, but only until the next command // Buffer in CLI word from user, but only until the next command
List<String> scenarioNames = new ArrayList<>(); List<String> scenarioNames = new ArrayList<>();
while (arglist.size() > 0 while (arglist.size() > 0
&& !arglist.peekFirst().contains("=") && !arglist.peekFirst().contains("=")
&& !arglist.peekFirst().startsWith("-") && !arglist.peekFirst().startsWith("-")
&& !RESERVED_WORDS.contains(arglist.peekFirst())) { && !RESERVED_WORDS.contains(arglist.peekFirst())) {
scenarioNames.add(arglist.removeFirst()); scenarioNames.add(arglist.removeFirst());
} }
if (scenarioNames.size() == 0) { if (scenarioNames.size() == 0) {
@ -74,8 +73,8 @@ public class NBCLIScenarioParser {
// Parse CLI command into keyed parameters, in order // Parse CLI command into keyed parameters, in order
LinkedHashMap<String, String> userParams = new LinkedHashMap<>(); LinkedHashMap<String, String> userParams = new LinkedHashMap<>();
while (arglist.size() > 0 while (arglist.size() > 0
&& arglist.peekFirst().contains("=") && arglist.peekFirst().contains("=")
&& !arglist.peekFirst().startsWith("-")) { && !arglist.peekFirst().startsWith("-")) {
String[] arg = arglist.removeFirst().split("="); String[] arg = arglist.removeFirst().split("=");
arg[0] = Synonyms.canonicalize(arg[0], logger); arg[0] = Synonyms.canonicalize(arg[0], logger);
if (userParams.containsKey(arg[0])) { if (userParams.containsKey(arg[0])) {
@ -93,11 +92,11 @@ public class NBCLIScenarioParser {
// Load in named scenario // Load in named scenario
Content<?> yamlWithNamedScenarios = NBIO.all() Content<?> yamlWithNamedScenarios = NBIO.all()
.prefix(SEARCH_IN) .prefix(SEARCH_IN)
.prefix(includes) .prefix(includes)
.name(workloadName) .name(workloadName)
.extension("yaml") .extension("yaml")
.one(); .one();
StmtsDocList stmts = StatementsLoader.load(logger, yamlWithNamedScenarios); StmtsDocList stmts = StatementsLoader.load(logger, yamlWithNamedScenarios);
@ -107,7 +106,7 @@ public class NBCLIScenarioParser {
if (namedSteps == null) { if (namedSteps == null) {
throw new BasicError("Unable to find named scenario '" + scenarioName + "' in workload '" + workloadName throw new BasicError("Unable to find named scenario '" + scenarioName + "' in workload '" + workloadName
+ "', but you can pick from " + String.join(",", scenarios.getScenarioNames())); + "', but you can pick from " + String.join(",", scenarios.getScenarioNames()));
} }
// each named command line step of the named scenario // each named command line step of the named scenario
@ -136,37 +135,38 @@ public class NBCLIScenarioParser {
// Undefine any keys with a value of 'undef' // Undefine any keys with a value of 'undef'
List<String> undefKeys = buildingCmd.entrySet() List<String> undefKeys = buildingCmd.entrySet()
.stream() .stream()
.filter(e -> e.getValue().toLowerCase().endsWith("=undef")) .filter(e -> e.getValue().toLowerCase().endsWith("=undef"))
.map(Map.Entry::getKey) .map(Map.Entry::getKey)
.collect(Collectors.toList()); .collect(Collectors.toList());
undefKeys.forEach(buildingCmd::remove); undefKeys.forEach(buildingCmd::remove);
if (!buildingCmd.containsKey("workload")) { if (!buildingCmd.containsKey("workload")) {
String relativeWorkloadPathFromRoot = yamlWithNamedScenarios.asPath().toString(); String relativeWorkloadPathFromRoot = yamlWithNamedScenarios.asPath().toString();
relativeWorkloadPathFromRoot = relativeWorkloadPathFromRoot.startsWith("/") ? relativeWorkloadPathFromRoot = relativeWorkloadPathFromRoot.startsWith("/") ?
relativeWorkloadPathFromRoot.substring(1) : relativeWorkloadPathFromRoot; relativeWorkloadPathFromRoot.substring(1) : relativeWorkloadPathFromRoot;
buildingCmd.put("workload", "workload=" + relativeWorkloadPathFromRoot); buildingCmd.put("workload", "workload=" + relativeWorkloadPathFromRoot);
} }
if (!buildingCmd.containsKey("alias")) { if (!buildingCmd.containsKey("alias")) {
buildingCmd.put("alias", "alias="+WORKLOAD_SCENARIO_STEP); buildingCmd.put("alias", "alias=" + WORKLOAD_SCENARIO_STEP);
} }
String alias = buildingCmd.get("alias"); String alias = buildingCmd.get("alias");
for (String token : new String[]{"WORKLOAD", "SCENARIO", "STEP"}) { for (String token : new String[]{"WORKLOAD", "SCENARIO", "STEP"}) {
if (!alias.contains(token)) { if (!alias.contains(token)) {
logger.warn("Your alias template '" + alias + "' does not contain " + token + ", which will " + logger.warn("Your alias template '" + alias + "' does not contain " + token + ", which will " +
"cause your metrics to be combined under the same name. It is strongly advised that you " + "cause your metrics to be combined under the same name. It is strongly advised that you " +
"include them in a template like " + WORKLOAD_SCENARIO_STEP + "."); "include them in a template like " + WORKLOAD_SCENARIO_STEP + ".");
} }
} }
alias = alias.replaceAll("WORKLOAD", workloadContent.asPath().getFileName().toString().replaceAll( String workloadToken = workloadContent.asPath().getFileName().toString();
".yaml",""));
alias = alias.replaceAll("SCENARIO", scenarioName); alias = alias.replaceAll("WORKLOAD", sanitize(workloadToken));
alias = alias.replaceAll("STEP", stepName); alias = alias.replaceAll("SCENARIO", sanitize(scenarioName));
alias=(alias.startsWith("alias=") ? alias : "alias="+alias); alias = alias.replaceAll("STEP", sanitize(stepName));
alias = (alias.startsWith("alias=") ? alias : "alias=" + alias);
buildingCmd.put("alias", alias); buildingCmd.put("alias", alias);
logger.debug("Named scenario built command: " + String.join(" ", buildingCmd.values())); logger.debug("Named scenario built command: " + String.join(" ", buildingCmd.values()));
@ -178,6 +178,13 @@ public class NBCLIScenarioParser {
} }
public static String sanitize(String word) {
String sanitized = word;
sanitized = sanitized.replaceAll("\\..+$","");
sanitized = sanitized.replaceAll("[^a-zA-Z0-9]+","");
return sanitized;
}
private static final Pattern WordAndMaybeAssignment = Pattern.compile("(?<name>\\w+)((?<oper>=+)(?<val>.+))?"); private static final Pattern WordAndMaybeAssignment = Pattern.compile("(?<name>\\w+)((?<oper>=+)(?<val>.+))?");
private static LinkedHashMap<String, CmdArg> parseStep(String cmd) { private static LinkedHashMap<String, CmdArg> parseStep(String cmd) {
@ -252,10 +259,10 @@ public class NBCLIScenarioParser {
public static List<WorkloadDesc> getWorkloadsWithScenarioScripts(String... includes) { public static List<WorkloadDesc> getWorkloadsWithScenarioScripts(String... includes) {
List<Content<?>> activities = NBIO.all() List<Content<?>> activities = NBIO.all()
.prefix(SEARCH_IN) .prefix(SEARCH_IN)
.prefix(includes) .prefix(includes)
.extension("yaml") .extension("yaml")
.list(); .list();
List<Path> yamlPathList = activities.stream().map(Content::asPath).collect(Collectors.toList()); List<Path> yamlPathList = activities.stream().map(Content::asPath).collect(Collectors.toList());
@ -264,11 +271,11 @@ public class NBCLIScenarioParser {
for (Path yamlPath : yamlPathList) { for (Path yamlPath : yamlPathList) {
String referenced = yamlPath.toString(); String referenced = yamlPath.toString();
referenced = referenced.startsWith("/") ? referenced.substring(1) : referenced = referenced.startsWith("/") ? referenced.substring(1) :
referenced; referenced;
Content<?> content = NBIO.all().prefix(SEARCH_IN) Content<?> content = NBIO.all().prefix(SEARCH_IN)
.name(referenced).extension("yaml") .name(referenced).extension("yaml")
.one(); .one();
StmtsDocList stmts = StatementsLoader.load(logger, content); StmtsDocList stmts = StatementsLoader.load(logger, content);
@ -290,11 +297,11 @@ public class NBCLIScenarioParser {
if (scenarioNames != null && scenarioNames.size() > 0) { if (scenarioNames != null && scenarioNames.size() > 0) {
String path = yamlPath.toString(); String path = yamlPath.toString();
path = path.startsWith(FileSystems.getDefault().getSeparator()) ? path.substring(1) : path; path = path.startsWith(FileSystems.getDefault().getSeparator()) ? path.substring(1) : path;
LinkedHashMap<String,String> sortedTemplates = new LinkedHashMap<>(); LinkedHashMap<String, String> sortedTemplates = new LinkedHashMap<>();
ArrayList<String> keyNames = new ArrayList<>(templates.keySet()); ArrayList<String> keyNames = new ArrayList<>(templates.keySet());
Collections.sort(keyNames); Collections.sort(keyNames);
for (String keyName : keyNames) { for (String keyName : keyNames) {
sortedTemplates.put(keyName,templates.get(keyName)); sortedTemplates.put(keyName, templates.get(keyName));
} }
String description = stmts.getDescription(); String description = stmts.getDescription();
@ -328,10 +335,9 @@ public class NBCLIScenarioParser {
while (matcher.find()) { while (matcher.find()) {
String match = matcher.group(1); String match = matcher.group(1);
String[] matchArray = match.split(":"); String[] matchArray = match.split(":");
if (matchArray.length==1) { if (matchArray.length == 1) {
templates.put(matchArray[0], "-none-"); templates.put(matchArray[0], "-none-");
} } else {
else {
templates.put(matchArray[0], matchArray[1]); templates.put(matchArray[0], matchArray[1]);
} }
} }

7
engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTest.java
View File

@ -1,5 +1,6 @@
package io.nosqlbench.engine.cli; package io.nosqlbench.engine.cli;
import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.nb.api.errors.BasicError;
import org.junit.Test; import org.junit.Test;
@ -98,4 +99,10 @@ public class NBCLIScenarioParserTest {
} }
@Test
public void testSanitizer() {
String sanitized = NBCLIScenarioParser.sanitize("A-b,c_d");
assertThat(sanitized).isEqualTo("Abcd");
}
} }