IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

error for unknown scenario, handling of TEMPLATE and <<>> variables in run scripts in the yaml

Browse Source
This commit is contained in:
phact 2020-03-17 12:50:24 -04:00
parent e24af60c88
commit b216bc3164
8 changed files with 50 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java
View File

@ -38,7 +38,7 @@ import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.api.metrics.ExceptionCountMetrics;
import io.nosqlbench.engine.api.metrics.ExceptionHistoMetrics;
import io.nosqlbench.engine.api.util.SimpleConfig;
import io.nosqlbench.engine.api.util.StrInterpolater;
import io.nosqlbench.engine.api.util.StrInterpolator;
import io.nosqlbench.engine.api.util.TagFilter;
import io.nosqlbench.engine.api.util.Unit;
import org.slf4j.Logger;
@ -276,7 +276,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
StmtsDocList doclist = null;
String yaml_loc = activityDef.getParams().getOptionalString("yaml").orElse("default");
StrInterpolater interp = new StrInterpolater(activityDef);
StrInterpolator interp = new StrInterpolator(activityDef);
String yamlVersion = "unset";
if (yaml_loc.endsWith(":1") || yaml_loc.endsWith(":2")) {
@ -329,7 +329,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
}
@Deprecated
private StmtsDocList getVersion1StmtsDoc(StrInterpolater interp, String yaml_loc) {
private StmtsDocList getVersion1StmtsDoc(StrInterpolator interp, String yaml_loc) {
StmtsDocList unfiltered;
List<RawStmtsBlock> blocks = new ArrayList<>();

12
activitytype-cql/src/main/resources/activities/baselines/cql-iot.yaml
View File

@ -1,13 +1,13 @@
# nb -v run type=cql yaml=baselines/cql-iot tags=phase:schema host=dsehost
scenarios:
default:
- run type=cql yaml=baselines/cql-iot tags=phase:schema
- run type=cql yaml=baselines/cql-iot tags=phase:rampup cycles=TEMPLATE(cycles,10000000)
- run type=cql yaml=baselines/cql-iot tags=phase:main cycles=TEMPLATE(cycles,10000000)
- run type=cql yaml=baselines/cql-iot tags=phase:schema hosts=TEMPLATE(hosts,localhost)
- run type=cql yaml=baselines/cql-iot tags=phase:rampup cycles=TEMPLATE(cycles,10000000) hosts=TEMPLATE(hosts,localhost)
- run type=cql yaml=baselines/cql-iot tags=phase:main cycles=TEMPLATE(cycles,10000000) hosts=TEMPLATE(hosts,localhost)
stdout:
- run type=cql yaml=baselines/cql-iot tags=phase:schema
- run type=cql yaml=baselines/cql-iot tags=phase:rampup cycles=TEMPLATE(cycles,10000000)
- run type=cql yaml=baselines/cql-iot tags=phase:main cycles=TEMPLATE(cycles,10000000)
- run type=cql yaml=baselines/cql-iot tags=phase:schema hosts=TEMPLATE(hosts,localhost)
- run type=cql yaml=baselines/cql-iot tags=phase:rampup cycles=TEMPLATE(cycles,10000000) hosts=TEMPLATE(hosts,localhost)
- run type=cql yaml=baselines/cql-iot tags=phase:main cycles=TEMPLATE(cycles,10000000) hosts=TEMPLATE(hosts,localhost)
bindings:
machine_id: Mod(<<sources:10000>>); ToHashedUUID() -> java.util.UUID
sensor_name: HashedLineToString('data/variable_words.txt')

4
activitytype-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
View File

@ -32,7 +32,7 @@ import io.nosqlbench.engine.api.activityimpl.ParameterMap;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.api.metrics.ExceptionMeterMetrics;
import io.nosqlbench.engine.api.util.StrInterpolater;
import io.nosqlbench.engine.api.util.StrInterpolator;
import io.nosqlbench.virtdata.api.BindingsTemplate;
import io.nosqlbench.virtdata.api.templates.StringBindings;
import io.nosqlbench.virtdata.api.templates.StringBindingsTemplate;
@ -71,7 +71,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
public StdoutActivity(ActivityDef activityDef) {
super(activityDef);
StrInterpolater interp = new StrInterpolater(activityDef);
StrInterpolator interp = new StrInterpolator(activityDef);
String yaml_loc = activityDef.getParams().getOptionalString("yaml").orElse("default");
this.showstmts = activityDef.getParams().getOptionalBoolean("showstatements").orElse(false);
this.fileName = activityDef.getParams().getOptionalString("filename").orElse("stdout");

10
engine-api/src/main/java/io/nosqlbench/engine/api/util/StrInterpolater.java → engine-api/src/main/java/io/nosqlbench/engine/api/util/StrInterpolator.java
View File

@ -27,23 +27,23 @@ import java.util.List;
import java.util.Map;
import java.util.function.Function;
public class StrInterpolater implements Function<String, String> {
public class StrInterpolator implements Function<String, String> {
private MultiMap multimap = new MultiMap();
private StrSubstitutor substitutor= new StrSubstitutor(multimap,"<<",">>",'\\');
private StrSubstitutor substitutor2 = new StrSubstitutor(multimap, "TEMPLATE(", ")", '\\');
public StrInterpolater(ActivityDef... activityDefs) {
public StrInterpolator(ActivityDef... activityDefs) {
Arrays.stream(activityDefs)
.map(ad -> ad.getParams().getStringStringMap())
.forEach(multimap::add);
}
public StrInterpolater(Map<String,String> basicMap) {
public StrInterpolator(Map<String,String> basicMap) {
multimap.add(basicMap);
}
// for testing
protected StrInterpolater(List<Map<String, String>> maps) {
protected StrInterpolator(List<Map<String, String>> maps) {
maps.forEach(multimap::add);
}
@ -52,7 +52,7 @@ public class StrInterpolater implements Function<String, String> {
return substitutor.replace(substitutor2.replace(s));
}
private static class MultiMap extends StrLookup<String> {
public static class MultiMap extends StrLookup<String> {
private List<Map<String, String>> maps = new ArrayList<>();
private String warnPrefix = "UNSET";

4
engine-api/src/test/java/io/nosqlbench/engine/api/util/StrInterpolaterTest.java → engine-api/src/test/java/io/nosqlbench/engine/api/util/StrInterpolatorTest.java
View File

@ -27,7 +27,7 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
@Test
public class StrInterpolaterTest {
public class StrInterpolatorTest {
private static List<Map<String, String>> abcd = new ArrayList<Map<String, String>>() {{
add(
@ -54,7 +54,7 @@ public class StrInterpolaterTest {
);
}};
private static StrInterpolater interp = new StrInterpolater(abcd);
private static StrInterpolator interp = new StrInterpolator(abcd);
@Test
public void shouldReturnIdentity() {

29
engine-cli/src/main/java/io/nosqlbench/engine/cli/EBCLIOptions.java
View File

@ -4,9 +4,13 @@ import ch.qos.logback.classic.Level;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.ParameterMap;
import io.nosqlbench.engine.api.metrics.IndicatorMode;
import io.nosqlbench.engine.api.util.NosqlBenchFiles;
import io.nosqlbench.engine.api.util.StrInterpolator;
import io.nosqlbench.engine.api.util.Unit;
import org.apache.commons.text.StrSubstitutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -321,7 +325,7 @@ public class EBCLIOptions {
if(path.isPresent()){
arglist.removeFirst();
String scenarioFilter = null;
if (arglist.size() > 0 && arglist.peekFirst().contains("=")){
if (arglist.size() > 0 && !arglist.peekFirst().contains("=")){
scenarioFilter = arglist.peekFirst();
arglist.removeFirst();
};
@ -349,10 +353,33 @@ public class EBCLIOptions {
List<String> cmds = scenarios.getNamedScenario(scenarioName);
if (cmds == null){
List<String> names = scenarios.getScenarioNames();
throw new RuntimeException("Unknown scenaro name, make sure the scenario name you provide exists in the workload definition (yaml):\n" + String.join(",", names));
}
for (String cmd : cmds) {
String[] cmdArray = cmd.split(" ");
Map<String, String> paramMap = new HashMap<>();
for (String parameter: cmdArray) {
if (parameter.contains("=")){
if ( !parameter.contains("TEMPLATE(") && !parameter.contains("<<")) {
String[] paramArray = parameter.split("=");
paramMap.put(paramArray[0], paramArray[1]);
}
}
}
StrSubstitutor sub1 = new StrSubstitutor(paramMap, "<<", ">>", '\\', ",");
StrSubstitutor sub2 = new StrSubstitutor(paramMap, "TEMPLATE(", ")", '\\', ",");
cmd = sub2.replace(sub1.replace(cmd));
// Is there a better way to do this than regex?
parse(cmd.split(" "));
}
arglist.removeFirst();
}

6
engine-cli/src/main/java/io/nosqlbench/engine/cli/EBCLIScriptAssembly.java
View File

@ -2,7 +2,7 @@ package io.nosqlbench.engine.cli;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.util.NosqlBenchFiles;
import io.nosqlbench.engine.api.util.StrInterpolater;
import io.nosqlbench.engine.api.util.StrInterpolator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -86,8 +86,8 @@ public class EBCLIScriptAssembly {
} catch (Throwable t) {
throw new RuntimeException("Unable to buffer " + cmd.getCmdSpec() + ": " + t);
}
StrInterpolater interpolater = new StrInterpolater(cmd.getCmdArgs());
scriptData = interpolater.apply(scriptData);
StrInterpolator interpolator = new StrInterpolator(cmd.getCmdArgs());
scriptData = interpolator.apply(scriptData);
return new ScriptData(scriptData,cmd.getCmdArgs());
}

2
engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java
View File

@ -195,7 +195,7 @@ public class TestNBCLIOptions {
@Test
public void cqlIotYamlScenarioSchemaOnly() {
EBCLIOptions opts = new EBCLIOptions(new String[]{ "scenario-test", "stdout"});
EBCLIOptions opts = new EBCLIOptions(new String[]{ "scenario-test", "schema-only"});
List<EBCLIOptions.Cmd> cmds = opts.getCommands();
}