IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-01-12 00:42:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

centralized path functions

Browse Source
This commit is contained in:
Jonathan Shook 2020-04-01 18:24:38 -05:00
parent 378f6b9844
commit e2d3d3825c
13 changed files with 101 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/statements/core/YamlCQLStatementLoader.java
View File

@ -1,7 +1,7 @@
package io.nosqlbench.activitytype.cql.statements.core;
import io.nosqlbench.engine.api.activityimpl.ActivityInitializationError;
import io.nosqlbench.engine.api.util.NBFiles;
import io.nosqlbench.nb.api.pathutil.NBFiles;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.TypeDescription;

2
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoader.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.activityconfig.snakecharmer.SnakeYamlCharmer;
import io.nosqlbench.engine.api.activityimpl.ActivityInitializationError;
import io.nosqlbench.engine.api.util.NBFiles;
import io.nosqlbench.nb.api.pathutil.NBFiles;
import org.slf4j.Logger;
import org.yaml.snakeyaml.TypeDescription;
import org.yaml.snakeyaml.Yaml;

1
engine-api/src/test/java/io/nosqlbench/engine/api/util/NBFilesTest.java
View File

@ -17,6 +17,7 @@
package io.nosqlbench.engine.api.util;
import io.nosqlbench.nb.api.pathutil.NBFiles;
import org.junit.Test;
import java.io.InputStream;

6
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -6,7 +6,7 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogIm
import io.nosqlbench.engine.api.activityapi.input.InputType;
import io.nosqlbench.engine.api.activityapi.output.OutputType;
import io.nosqlbench.engine.api.exceptions.BasicError;
import io.nosqlbench.engine.api.util.NBFiles;
import io.nosqlbench.nb.api.pathutil.NBFiles;
import io.nosqlbench.engine.core.MarkdownDocInfo;
import io.nosqlbench.engine.core.ScenarioLogger;
import io.nosqlbench.engine.core.ScenariosResults;
@ -233,8 +233,8 @@ public class NBCLI {
}
public void printWorkloads() {
List<NBFiles.WorkloadDesc> workloads = NBFiles.getWorkloadsWithScenarioScripts();
for (NBFiles.WorkloadDesc workload : workloads) {
List<WorkloadDesc> workloads = NBCLIScenarioParser.getWorkloadsWithScenarioScripts();
for (WorkloadDesc workload : workloads) {
System.out.println("\n# from: "+ workload.getYamlPath());
List<String> scenarioList = workload.getScenarioNames();
String workloadName = workload.getYamlPath().replaceAll("\\.yaml", "") ;

2
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@ -2,7 +2,7 @@ package io.nosqlbench.engine.cli;
import ch.qos.logback.classic.Level;
import io.nosqlbench.engine.api.metrics.IndicatorMode;
import io.nosqlbench.engine.api.util.NBFiles;
import io.nosqlbench.nb.api.pathutil.NBFiles;
import io.nosqlbench.engine.api.util.Unit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

60
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIScenarioParser.java
View File

@ -1,18 +1,23 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.docsys.core.PathWalker;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.exceptions.BasicError;
import io.nosqlbench.engine.api.util.NBFiles;
import io.nosqlbench.nb.api.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBFiles;
import io.nosqlbench.engine.api.util.StrInterpolator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class NBCLIScenarioParser {
@ -223,4 +228,57 @@ public class NBCLIScenarioParser {
// }
// }
private static Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
private static Pattern templatePattern2 = Pattern.compile("<<(.+?)>>");
public static List<WorkloadDesc> getWorkloadsWithScenarioScripts() {
String dir = "activities/";
Path basePath = VirtDataResources.findPathIn(dir);
List<Path> yamlPathList = PathWalker.findAll(basePath)
.stream()
.filter(f -> f.toString().endsWith(".yaml"))
.filter(f -> f.toString().contains("activities"))
.collect(Collectors.toList());
List<WorkloadDesc> workloadDescriptions = new ArrayList<>();
for (Path yamlPath : yamlPathList) {
String substring = yamlPath.toString().substring(1);
StmtsDocList stmts = StatementsLoader.load(logger, substring);
Set<String> templates = new HashSet<>();
try {
List<String> lines = Files.readAllLines(yamlPath);
for (String line : lines) {
Matcher matcher = templatePattern.matcher(line);
while (matcher.find()) {
templates.add(matcher.group(1));
}
matcher = templatePattern2.matcher(line);
while (matcher.find()) {
templates.add(matcher.group(1));
}
}
} catch (IOException e) {
e.printStackTrace();
}
Scenarios scenarios = stmts.getDocScenarios();
List<String> scenarioNames = scenarios.getScenarioNames();
if (scenarioNames != null && scenarioNames.size() >0){
workloadDescriptions.add(new WorkloadDesc(yamlPath.getFileName().toString(), scenarioNames, templates));
}
}
return workloadDescriptions;
}
}

2
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIScriptAssembly.java
View File

@ -1,7 +1,7 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.util.NBFiles;
import io.nosqlbench.nb.api.pathutil.NBFiles;
import io.nosqlbench.engine.api.util.StrInterpolator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

28
engine-cli/src/main/java/io/nosqlbench/engine/cli/WorkloadDesc.java Normal file
View File

@ -0,0 +1,28 @@
package io.nosqlbench.engine.cli;
import java.util.List;
import java.util.Set;
public class WorkloadDesc {
private final String yamlPath;
private final List<String> scenarioNames;
private final Set<String> temlpates;
public WorkloadDesc(String yamlPath, List<String> scenarioNames, Set<String> templates) {
this.yamlPath = yamlPath;
this.scenarioNames = scenarioNames;
this.temlpates = templates;
}
public String getYamlPath() {
return yamlPath;
}
public List<String> getScenarioNames() {
return scenarioNames;
}
public Set<String> getTemlpates() {
return temlpates;
}
}

2
engine-docker/src/main/java/io/nosqlbench/engine/docker/DockerMetricsManager.java
View File

@ -12,7 +12,7 @@ import com.github.dockerjava.api.model.ContainerNetworkSettings;
import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.core.async.ResultCallbackTemplate;
import com.github.dockerjava.core.command.LogContainerResultCallback;
import io.nosqlbench.engine.api.util.NBFiles;
import io.nosqlbench.nb.api.pathutil.NBFiles;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

2
engine-docker/src/main/java/io/nosqlbench/engine/docker/RestHelper.java
View File

@ -1,7 +1,7 @@
package io.nosqlbench.engine.docker;
import io.nosqlbench.engine.api.exceptions.BasicError;
import io.nosqlbench.engine.api.util.NBFiles;
import io.nosqlbench.nb.api.pathutil.NBFiles;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

2
engine-extensions/src/main/java/io/nosqlbench/engine/extensions/files/FileAccess.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.extensions.files;
import io.nosqlbench.engine.api.util.NBFiles;
import io.nosqlbench.nb.api.pathutil.NBFiles;
public class FileAccess extends FileAccessPluginData {

84
engine-api/src/main/java/io/nosqlbench/engine/api/util/NBFiles.java → nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/NBFiles.java
View File

@ -15,30 +15,21 @@
* /
*/
package io.nosqlbench.engine.api.util;
package io.nosqlbench.nb.api.pathutil;
import io.nosqlbench.docsys.core.PathWalker;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.nb.api.VirtDataResources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class NBFiles {
private final static Logger logger = LoggerFactory.getLogger(NBFiles.class);
private static Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
private static Pattern templatePattern2 = Pattern.compile("<<(.+?)>>");
public static InputStream findRequiredStreamOrFile(String basename, String extension, String... searchPaths) {
@ -160,76 +151,5 @@ public class NBFiles {
}
}
public static List<WorkloadDesc> getWorkloadsWithScenarioScripts() {
String dir = "activities/";
Path basePath = VirtDataResources.findPathIn(dir);
List<Path> yamlPathList = PathWalker.findAll(basePath)
.stream()
.filter(f -> f.toString().endsWith(".yaml"))
.filter(f -> f.toString().contains("activities"))
.collect(Collectors.toList());
List<WorkloadDesc> workloadDescriptions = new ArrayList<>();
for (Path yamlPath : yamlPathList) {
String substring = yamlPath.toString().substring(1);
StmtsDocList stmts = StatementsLoader.load(logger, substring);
Set<String> templates = new HashSet<>();
try {
List<String> lines = Files.readAllLines(yamlPath);
for (String line : lines) {
Matcher matcher = templatePattern.matcher(line);
while (matcher.find()) {
templates.add(matcher.group(1));
}
matcher = templatePattern2.matcher(line);
while (matcher.find()) {
templates.add(matcher.group(1));
}
}
} catch (IOException e) {
e.printStackTrace();
}
Scenarios scenarios = stmts.getDocScenarios();
List<String> scenarioNames = scenarios.getScenarioNames();
if (scenarioNames != null && scenarioNames.size() >0){
workloadDescriptions.add(new WorkloadDesc(yamlPath.getFileName().toString(), scenarioNames, templates));
}
}
return workloadDescriptions;
}
public static class WorkloadDesc {
private final String yamlPath;
private final List<String> scenarioNames;
private final Set<String> temlpates;
public WorkloadDesc(String yamlPath, List<String> scenarioNames, Set<String> templates) {
this.yamlPath = yamlPath;
this.scenarioNames = scenarioNames;
this.temlpates = templates;
}
public String getYamlPath() {
return yamlPath;
}
public List<String> getScenarioNames() {
return scenarioNames;
}
public Set<String> getTemlpates() {
return temlpates;
}
}
}

2
engine-api/src/main/java/io/nosqlbench/engine/api/util/NBPathWalker.java → nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/NBPathWalker.java
View File

@ -1,4 +1,4 @@
package io.nosqlbench.engine.api.util;
package io.nosqlbench.nb.api.pathutil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;