IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

parameterize op template

Browse Source
This commit is contained in:
Jonathan Shook 2021-06-11 15:27:11 -05:00
parent adc60e66a2
commit 748eebc04c
18 changed files with 71 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CQLBindHelper.java
View File

@ -2,7 +2,7 @@ package io.nosqlbench.activitytype.cql.core;
import com.datastax.driver.core.*; import com.datastax.driver.core.*;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt; import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.yaml.OpDef; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.net.InetAddress; import java.net.InetAddress;
@ -129,13 +129,10 @@ public class CQLBindHelper {
} }
} }
public static Map<String, String> parseAndGetSpecificBindings(OpDef opDef, ParsedStmt parsed) { public static Map<String, String> parseAndGetSpecificBindings(OpTemplate<?> opDef, ParsedStmt parsed) {
List<String> spans = new ArrayList<>();
String statement = opDef.getStmt(); String statement = opDef.getStmt();
Set<String> extraBindings = new HashSet<>(); Set<String> extraBindings = new HashSet<>(opDef.getBindings().keySet());
extraBindings.addAll(opDef.getBindings().keySet());
Map<String, String> specificBindings = new LinkedHashMap<>(); Map<String, String> specificBindings = new LinkedHashMap<>();
Matcher m = stmtToken.matcher(statement); Matcher m = stmtToken.matcher(statement);
@ -148,7 +145,6 @@ public class CQLBindHelper {
String form2 = m.group(2); String form2 = m.group(2);
String tokenName = (form1 != null && !form1.isEmpty()) ? form1 : form2; String tokenName = (form1 != null && !form1.isEmpty()) ? form1 : form2;
lastMatch = m.end(); lastMatch = m.end();
spans.add(pre);
if (extraBindings.contains(tokenName)) { if (extraBindings.contains(tokenName)) {
if (specificBindings.get(tokenName) != null){ if (specificBindings.get(tokenName) != null){

14
driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java
View File

@ -70,7 +70,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
private final ExceptionHistoMetrics exceptionHistoMetrics; private final ExceptionHistoMetrics exceptionHistoMetrics;
private final ActivityDef activityDef; private final ActivityDef activityDef;
private final Map<String, Writer> namedWriters = new HashMap<>(); private final Map<String, Writer> namedWriters = new HashMap<>();
protected List<OpTemplate> stmts; protected List<OpTemplate<?>> stmts;
Timer retryDelayTimer; Timer retryDelayTimer;
Timer pagesTimer; Timer pagesTimer;
Histogram skippedTokensHisto; Histogram skippedTokensHisto;
@ -180,7 +180,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
Set<String> timerStarts = new HashSet<>(); Set<String> timerStarts = new HashSet<>();
Set<String> timerStops = new HashSet<>(); Set<String> timerStops = new HashSet<>();
for (OpTemplate stmtDef : stmts) { for (OpTemplate<?> stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed(CqlActivity::canonicalizeBindings).orError(); ParsedStmt parsed = stmtDef.getParsed(CqlActivity::canonicalizeBindings).orError();
boolean prepared = stmtDef.getParamOrDefault("prepared", true); boolean prepared = stmtDef.getParamOrDefault("prepared", true);
@ -282,7 +282,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.map(s -> s.split("[,: ]")) .map(s -> s.split("[,: ]"))
.map(Save::new) .map(Save::new)
.ifPresent(save_op -> { .ifPresent(save_op -> {
psummary.append(" save=>").append(save_op.toString()); psummary.append(" save=>").append(save_op);
template.addRowCycleOperators(save_op); template.addRowCycleOperators(save_op);
}); });
@ -291,7 +291,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.stream().flatMap(Arrays::stream) .stream().flatMap(Arrays::stream)
.map(ResultSetCycleOperators::newOperator) .map(ResultSetCycleOperators::newOperator)
.forEach(rso -> { .forEach(rso -> {
psummary.append(" rsop=>").append(rso.toString()); psummary.append(" rsop=>").append(rso);
template.addResultSetOperators(rso); template.addResultSetOperators(rso);
}); });
@ -300,7 +300,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.stream().flatMap(Arrays::stream) .stream().flatMap(Arrays::stream)
.map(RowCycleOperators::newOperator) .map(RowCycleOperators::newOperator)
.forEach(ro -> { .forEach(ro -> {
psummary.append(" rowop=>").append(ro.toString()); psummary.append(" rowop=>").append(ro);
template.addRowCycleOperators(ro); template.addRowCycleOperators(ro);
}); });
@ -326,7 +326,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
VerificationMetrics vmetrics = getVerificationMetrics(); VerificationMetrics vmetrics = getVerificationMetrics();
RowDifferencer.ThreadLocalWrapper differencer = new RowDifferencer.ThreadLocalWrapper(vmetrics, expected, diffType); RowDifferencer.ThreadLocalWrapper differencer = new RowDifferencer.ThreadLocalWrapper(vmetrics, expected, diffType);
psummary.append(" rowop=>verify-fields:").append(differencer.toString()); psummary.append(" rowop=>verify-fields:").append(differencer);
template.addResultSetOperators(new AssertSingleRowResultSet()); template.addResultSetOperators(new AssertSingleRowResultSet());
template.addRowCycleOperators(differencer); template.addRowCycleOperators(differencer);
@ -348,7 +348,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
template.getContextualBindings().getBindingsTemplate().addFieldBindings(stmtDef.getParsed().getBindPoints()); template.getContextualBindings().getBindingsTemplate().addFieldBindings(stmtDef.getParsed().getBindPoints());
if (psummary.length() > 0) { if (psummary.length() > 0) {
logger.info("statement named '" + stmtDef.getName() + "' has custom settings:" + psummary.toString()); logger.info("statement named '" + stmtDef.getName() + "' has custom settings:" + psummary);
} }
planner.addOp(template.resolve(), ratio); planner.addOp(template.resolve(), ratio);

2
driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/VerifierBuilder.java
View File

@ -8,7 +8,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
public class VerifierBuilder { public class VerifierBuilder {
public static BindingsTemplate getExpectedValuesTemplate(OpTemplate stmtDef) { public static BindingsTemplate getExpectedValuesTemplate(OpTemplate<?> stmtDef) {
BindingsTemplate expected = new BindingsTemplate(); BindingsTemplate expected = new BindingsTemplate();

14
driver-cqld3-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java
View File

@ -70,7 +70,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
private final ExceptionHistoMetrics exceptionHistoMetrics; private final ExceptionHistoMetrics exceptionHistoMetrics;
private final ActivityDef activityDef; private final ActivityDef activityDef;
private final Map<String, Writer> namedWriters = new HashMap<>(); private final Map<String, Writer> namedWriters = new HashMap<>();
protected List<OpTemplate> stmts; protected List<OpTemplate<?>> stmts;
Timer retryDelayTimer; Timer retryDelayTimer;
Timer pagesTimer; Timer pagesTimer;
private Histogram triesHisto; private Histogram triesHisto;
@ -179,7 +179,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
Set<String> timerStarts = new HashSet<>(); Set<String> timerStarts = new HashSet<>();
Set<String> timerStops = new HashSet<>(); Set<String> timerStops = new HashSet<>();
for (OpTemplate stmtDef : stmts) { for (OpTemplate<?> stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed(CqlActivity::canonicalizeBindings).orError(); ParsedStmt parsed = stmtDef.getParsed(CqlActivity::canonicalizeBindings).orError();
boolean prepared = stmtDef.getParamOrDefault("prepared", true); boolean prepared = stmtDef.getParamOrDefault("prepared", true);
@ -281,7 +281,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.map(s -> s.split("[,: ]")) .map(s -> s.split("[,: ]"))
.map(Save::new) .map(Save::new)
.ifPresent(save_op -> { .ifPresent(save_op -> {
psummary.append(" save=>").append(save_op.toString()); psummary.append(" save=>").append(save_op);
template.addRowCycleOperators(save_op); template.addRowCycleOperators(save_op);
}); });
@ -290,7 +290,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.stream().flatMap(Arrays::stream) .stream().flatMap(Arrays::stream)
.map(ResultSetCycleOperators::newOperator) .map(ResultSetCycleOperators::newOperator)
.forEach(rso -> { .forEach(rso -> {
psummary.append(" rsop=>").append(rso.toString()); psummary.append(" rsop=>").append(rso);
template.addResultSetOperators(rso); template.addResultSetOperators(rso);
}); });
@ -299,7 +299,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.stream().flatMap(Arrays::stream) .stream().flatMap(Arrays::stream)
.map(RowCycleOperators::newOperator) .map(RowCycleOperators::newOperator)
.forEach(ro -> { .forEach(ro -> {
psummary.append(" rowop=>").append(ro.toString()); psummary.append(" rowop=>").append(ro);
template.addRowCycleOperators(ro); template.addRowCycleOperators(ro);
}); });
@ -325,7 +325,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
VerificationMetrics vmetrics = getVerificationMetrics(); VerificationMetrics vmetrics = getVerificationMetrics();
RowDifferencer.ThreadLocalWrapper differencer = new RowDifferencer.ThreadLocalWrapper(vmetrics, expected, diffType); RowDifferencer.ThreadLocalWrapper differencer = new RowDifferencer.ThreadLocalWrapper(vmetrics, expected, diffType);
psummary.append(" rowop=>verify-fields:").append(differencer.toString()); psummary.append(" rowop=>verify-fields:").append(differencer);
template.addResultSetOperators(new AssertSingleRowResultSet()); template.addResultSetOperators(new AssertSingleRowResultSet());
template.addRowCycleOperators(differencer); template.addRowCycleOperators(differencer);
@ -347,7 +347,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
template.getContextualBindings().getBindingsTemplate().addFieldBindings(stmtDef.getParsed().getBindPoints()); template.getContextualBindings().getBindingsTemplate().addFieldBindings(stmtDef.getParsed().getBindPoints());
if (psummary.length() > 0) { if (psummary.length() > 0) {
logger.info("statement named '" + stmtDef.getName() + "' has custom settings:" + psummary.toString()); logger.info("statement named '" + stmtDef.getName() + "' has custom settings:" + psummary);
} }
planner.addOp(template.resolve(), ratio); planner.addOp(template.resolve(), ratio);

2
driver-cqld3-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/VerifierBuilder.java
View File

@ -8,7 +8,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
public class VerifierBuilder { public class VerifierBuilder {
public static BindingsTemplate getExpectedValuesTemplate(OpTemplate stmtDef) { public static BindingsTemplate getExpectedValuesTemplate(OpTemplate<?> stmtDef) {
BindingsTemplate expected = new BindingsTemplate(); BindingsTemplate expected = new BindingsTemplate();

8
driver-dsegraph-shaded/src/main/java/com/datastax/ebdrivers/dsegraph/GraphActivity.java
View File

@ -46,7 +46,7 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
public Timer resultTimer; public Timer resultTimer;
public Timer logicalGraphOps; public Timer logicalGraphOps;
public Histogram triesHisto; public Histogram triesHisto;
protected List<OpTemplate> stmts; protected List<OpTemplate<?>> stmts;
private int stride; private int stride;
private DseSession session; private DseSession session;
private DseCluster cluster; private DseCluster cluster;
@ -100,7 +100,7 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
throw new RuntimeException("There were no unfiltered statements found for this activity."); throw new RuntimeException("There were no unfiltered statements found for this activity.");
} }
for (OpTemplate stmtDef : stmts) { for (OpTemplate<?> stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed().orError(); ParsedStmt parsed = stmtDef.getParsed().orError();
@ -209,7 +209,7 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
try { try {
cluster = builder.build(); cluster = builder.build();
} catch (Exception e) { } catch (Exception e) {
logger.error("Error while instantiating cluster from builder: " + e.toString(), e); logger.error("Error while instantiating cluster from builder: " + e, e);
throw e; throw e;
} }
activityDef.getParams().getOptionalBoolean("defaultidempotence").map( activityDef.getParams().getOptionalBoolean("defaultidempotence").map(
@ -236,7 +236,7 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
logger.info("cluster-metadata-allhosts:\n" + session.getCluster().getMetadata().getAllHosts()); logger.info("cluster-metadata-allhosts:\n" + session.getCluster().getMetadata().getAllHosts());
return session; return session;
} catch (Exception e) { } catch (Exception e) {
logger.error("Error while creating a session for dsegraph: " + e.toString(), e); logger.error("Error while creating a session for dsegraph: " + e, e);
throw e; throw e;
} }

2
driver-jdbc/src/main/java/io/nosqlbench/activitytype/jdbc/impl/ReadyJDBCOp.java
View File

@ -12,7 +12,7 @@ public class ReadyJDBCOp implements OpDispenser<String> {
public ReadyJDBCOp(OpTemplate stmtDef) { public ReadyJDBCOp(OpTemplate stmtDef) {
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings()); ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints()); BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getCheckedBindPoints());
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings); StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings);
bindings = template.resolve(); bindings = template.resolve();

4
driver-kafka/src/main/java/com/datastax/ebdrivers/kafkaproducer/KafkaProducerActivity.java
View File

@ -73,12 +73,12 @@ public class KafkaProducerActivity extends SimpleActivity {
String tagFilter = activityDef.getParams().getOptionalString("tags").orElse(""); String tagFilter = activityDef.getParams().getOptionalString("tags").orElse("");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, new StrInterpolator(activityDef), StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, new StrInterpolator(activityDef),
"activities"); "activities");
List<OpTemplate> statements = stmtsDocList.getStmts(tagFilter); List<OpTemplate<?>> statements = stmtsDocList.getStmts(tagFilter);
String format = getParams().getOptionalString("format").orElse(null); String format = getParams().getOptionalString("format").orElse(null);
if (statements.size() > 0) { if (statements.size() > 0) {
for (OpTemplate statement : statements) { for (OpTemplate<?> statement : statements) {
sequencer.addOp( sequencer.addOp(
new KafkaStatement(statement, new KafkaStatement(statement,
servers, servers,

4
driver-kafka/src/main/java/com/datastax/ebdrivers/kafkaproducer/KafkaStatement.java
View File

@ -31,9 +31,9 @@ public class KafkaStatement {
private AvroSchema valueSerializerSchema = null; private AvroSchema valueSerializerSchema = null;
private final String key; private final String key;
public KafkaStatement(OpTemplate stmtDef, String servers, String clientId, String schemaRegistryUrl) { public KafkaStatement(OpTemplate<?> stmtDef, String servers, String clientId, String schemaRegistryUrl) {
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings()); ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints()); BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getCheckedBindPoints());
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings); StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings);
this.bindings = template.resolve(); this.bindings = template.resolve();

4
driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java
View File

@ -112,11 +112,11 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
TagFilter tagFilter = new TagFilter(tagfilter); TagFilter tagFilter = new TagFilter(tagfilter);
stmtsDocList.getStmts().stream().map(tagFilter::matchesTaggedResult).forEach(r -> logger.info(r.getLog())); stmtsDocList.getStmts().stream().map(tagFilter::matchesTaggedResult).forEach(r -> logger.info(r.getLog()));
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter); List<OpTemplate<?>> stmts = stmtsDocList.getStmts(tagfilter);
if (stmts.isEmpty()) { if (stmts.isEmpty()) {
logger.error("No statements found for this activity"); logger.error("No statements found for this activity");
} else { } else {
for (OpTemplate stmt : stmts) { for (OpTemplate<?> stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError(); ParsedStmt parsed = stmt.getParsed().orError();
String statement = parsed.getPositionalStatement(Function.identity()); String statement = parsed.getPositionalStatement(Function.identity());
Objects.requireNonNull(statement); Objects.requireNonNull(statement);

4
driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java
View File

@ -14,9 +14,9 @@ public class ReadyMongoStatement {
private final StringBindings bindings; private final StringBindings bindings;
private final ReadPreference readPreference; private final ReadPreference readPreference;
public ReadyMongoStatement(OpTemplate stmtDef) { public ReadyMongoStatement(OpTemplate<?> stmtDef) {
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings()); ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints()); BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getCheckedBindPoints());
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings); StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings);
this.bindings = template.resolve(); this.bindings = template.resolve();

10
driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
View File

@ -81,7 +81,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
public void shutdownActivity() { public void shutdownActivity() {
try { try {
if (pw != null) { if (pw != null) {
if (!fileName.toLowerCase().equals("stdout")) { if (!fileName.equalsIgnoreCase("stdout")) {
logger.trace("Closing non-stdout output stream."); logger.trace("Closing non-stdout output stream.");
pw.close(); pw.close();
} }
@ -112,7 +112,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
protected Writer createPrintWriter() { protected Writer createPrintWriter() {
PrintWriter pw; PrintWriter pw;
if (fileName.toLowerCase().equals("stdout")) { if (fileName.equalsIgnoreCase("stdout")) {
pw = getConsoleOut(); pw = getConsoleOut();
} else { } else {
try { try {
@ -134,7 +134,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
SequencePlanner<StringBindings> sequencer = new SequencePlanner<>(sequencerType); SequencePlanner<StringBindings> sequencer = new SequencePlanner<>(sequencerType);
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse(""); String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter); List<OpTemplate<?>> stmts = stmtsDocList.getStmts(tagfilter);
String format = getParams().getOptionalString("format").orElse(null); String format = getParams().getOptionalString("format").orElse(null);
@ -155,7 +155,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
String bindings = getActivityDef().getParams().getOptionalString("bindings").orElse("doc"); String bindings = getActivityDef().getParams().getOptionalString("bindings").orElse("doc");
activeBindingNames.addAll(stmtsDocList.getDocBindings().keySet()); activeBindingNames.addAll(stmtsDocList.getDocBindings().keySet());
Pattern bindingsFilter = Pattern.compile(bindings.toLowerCase().equals("doc") ? ".*" : bindings); Pattern bindingsFilter = Pattern.compile(bindings.equalsIgnoreCase("doc") ? ".*" : bindings);
Set<String> filteredBindingNames = new LinkedHashSet<>(); Set<String> filteredBindingNames = new LinkedHashSet<>();
activeBindingNames activeBindingNames
.stream() .stream()
@ -179,7 +179,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
sequencer.addOp(sb, 1L); sequencer.addOp(sb, 1L);
} }
} else if (stmts.size() > 0) { } else if (stmts.size() > 0) {
for (OpTemplate stmt : stmts) { for (OpTemplate<?> stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError(); ParsedStmt parsed = stmt.getParsed().orError();
BindingsTemplate bt = new BindingsTemplate(parsed.getBindPoints()); BindingsTemplate bt = new BindingsTemplate(parsed.getBindPoints());
String statement = parsed.getPositionalStatement(Function.identity()); String statement = parsed.getPositionalStatement(Function.identity());

6
driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java
View File

@ -127,7 +127,7 @@ public class WebDriverActivity extends SimpleActivity {
cmdBuilder.append(" value='").append(command.getValue().replaceAll("'","\\\\'")).append("'"); cmdBuilder.append(" value='").append(command.getValue().replaceAll("'","\\\\'")).append("'");
} }
cmdBuilder.append("\n"); cmdBuilder.append("\n");
logger.debug("build cmd: '" + testBuilder.toString() + "'"); logger.debug("build cmd: '" + testBuilder + "'");
String cmdName = sideName+"_"+String.format("%s_%03d", sideName, ++idx); String cmdName = sideName+"_"+String.format("%s_%03d", sideName, ++idx);
commands.put(cmdName,cmdBuilder.toString()); commands.put(cmdName,cmdBuilder.toString());
} }
@ -180,13 +180,13 @@ public class WebDriverActivity extends SimpleActivity {
SequencePlanner<CommandTemplate> planner = new SequencePlanner<>(sequencerType); SequencePlanner<CommandTemplate> planner = new SequencePlanner<>(sequencerType);
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse(""); String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter); List<OpTemplate<?>> stmts = stmtsDocList.getStmts(tagfilter);
if (stmts.size() == 0) { if (stmts.size() == 0) {
throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'"); throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'");
} }
for (OpTemplate optemplate : stmts) { for (OpTemplate<?> optemplate : stmts) {
long ratio = optemplate.getParamOrDefault("ratio", 1); long ratio = optemplate.getParamOrDefault("ratio", 1);
CommandTemplate cmd = new CommandTemplate(optemplate); CommandTemplate cmd = new CommandTemplate(optemplate);
planner.addOp(cmd, ratio); planner.addOp(cmd, ratio);

6
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java
View File

@ -29,7 +29,7 @@ import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.function.Function; import java.util.function.Function;
public class OpDef implements OpTemplate { public class OpDef<T> implements OpTemplate<T> {
private static final String FIELD_DESC = "description"; private static final String FIELD_DESC = "description";
private static final String FIELD_NAME = "name"; private static final String FIELD_NAME = "name";
@ -58,8 +58,8 @@ public class OpDef implements OpTemplate {
} }
@Override @Override
public Object getOp() { public T getOp() {
return rawStmtDef.getOp(); return (T) rawStmtDef.getOp();
} }
@Override @Override

8
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java
View File

@ -121,19 +121,19 @@ import java.util.function.Function;
* p2: v2 * p2: v2
* }</pre> * }</pre>
*/ */
public interface OpTemplate extends Tagged { public interface OpTemplate<T> extends Tagged {
String getName(); String getName();
Object getOp(); T getOp();
Map<String, String> getBindings(); Map<String, String> getBindings();
Map<String, Object> getParams(); Map<String, Object> getParams();
<T> Map<String, T> getParamsAsValueType(Class<? extends T> type); <V> Map<String, V> getParamsAsValueType(Class<? extends V> type);
<V> V removeParamOrDefault(String name, V defaultValue); <VT> VT removeParamOrDefault(String name, VT defaultValue);
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
<V> V getParamOrDefault(String name, V defaultValue); <V> V getParamOrDefault(String name, V defaultValue);

18
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java
View File

@ -20,10 +20,7 @@ package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.util.TagFilter; import io.nosqlbench.engine.api.util.TagFilter;
import java.util.Iterator; import java.util.*;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
public class StmtsDocList implements Iterable<StmtsDoc> { public class StmtsDocList implements Iterable<StmtsDoc> {
@ -47,7 +44,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
public List<OpTemplate> getStmts() { public List<OpTemplate<?>> getStmts() {
return getStmts(""); return getStmts("");
} }
@ -56,14 +53,17 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
* @return The list of all included statements for all included blocks of in this document, * @return The list of all included statements for all included blocks of in this document,
* including the inherited and overridden values from the this doc and the parent block. * including the inherited and overridden values from the this doc and the parent block.
*/ */
public List<OpTemplate> getStmts(String tagFilterSpec) { public List<OpTemplate<?>> getStmts(String tagFilterSpec) {
TagFilter ts = new TagFilter(tagFilterSpec); TagFilter ts = new TagFilter(tagFilterSpec);
List<OpTemplate<?>> opTemplates = new ArrayList<>();
List<OpTemplate> stmts = getStmtDocs().stream()
getStmtDocs().stream()
.flatMap(d -> d.getStmts().stream()) .flatMap(d -> d.getStmts().stream())
.filter(ts::matchesTagged) .filter(ts::matchesTagged)
.collect(Collectors.toList()); .forEach(opTemplates::add);
return stmts;
return opTemplates;
} }

5
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -468,10 +468,11 @@ public class SimpleActivity implements Activity, ProgressCapable {
stmtsDocList = StatementsLoader.loadPath(logger, op_yaml_loc.get(), interp, "activities"); stmtsDocList = StatementsLoader.loadPath(logger, op_yaml_loc.get(), interp, "activities");
} }
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter); List<OpTemplate<?>> stmts = stmtsDocList.getStmts(tagfilter);
List<Long> ratios = new ArrayList<>(stmts.size()); List<Long> ratios = new ArrayList<>(stmts.size());
for (int i = 0; i < stmts.size(); i++) { for (int i = 0; i < stmts.size(); i++) {
long ratio = stmts.get(i).removeParamOrDefault("ratio", 1); OpTemplate<?> opTemplate = stmts.get(i);
long ratio = opTemplate.removeParamOrDefault("ratio", 1);
ratios.add(ratio); ratios.add(ratio);
} }

18
engine-api/src/main/java/io/nosqlbench/engine/api/templating/CommandTemplate.java
View File

@ -48,7 +48,7 @@ public class CommandTemplate {
* *
* @param optpl An OpTemplate * @param optpl An OpTemplate
*/ */
public CommandTemplate(OpTemplate optpl) { public CommandTemplate(OpTemplate<?> optpl) {
this(optpl.getName(), optpl.getOp().toString(), optpl.getParamsAsValueType(String.class), optpl.getBindings(), List.of()); this(optpl.getName(), optpl.getOp().toString(), optpl.getParamsAsValueType(String.class), optpl.getBindings(), List.of());
} }
@ -66,7 +66,7 @@ public class CommandTemplate {
* @param optpl An OpTemplate * @param optpl An OpTemplate
* @param parsers A list of parser functions * @param parsers A list of parser functions
*/ */
public CommandTemplate(OpTemplate optpl, List<Function<String, Map<String, String>>> parsers) { public CommandTemplate(OpTemplate<?> optpl, List<Function<String, Map<String, String>>> parsers) {
this(optpl.getName(), optpl.getOp(), optpl.getParamsAsValueType(String.class), optpl.getBindings(), parsers); this(optpl.getName(), optpl.getOp(), optpl.getParamsAsValueType(String.class), optpl.getBindings(), parsers);
} }
@ -92,7 +92,13 @@ public class CommandTemplate {
* @param bindings A set of named bindings in name:recipe form. * @param bindings A set of named bindings in name:recipe form.
* @param optionalParsers A set of functions which, if provided, will be used to read the oneline form. * @param optionalParsers A set of functions which, if provided, will be used to read the oneline form.
*/ */
public CommandTemplate(String name, Object op, Map<String, String> params, Map<String, String> bindings, List<Function<String, Map<String, String>>> optionalParsers) { public CommandTemplate(
String name,
Object op,
Map<String, String> params,
Map<String, String> bindings,
List<Function<String, Map<String, String>>> optionalParsers
) {
this.name = name; this.name = name;
Map<String, String> cmd = new HashMap<>(); Map<String, String> cmd = new HashMap<>();
@ -103,7 +109,7 @@ public class CommandTemplate {
String oneline; String oneline;
if (op instanceof CharSequence) { if (op instanceof CharSequence) {
oneline=op.toString(); oneline = op.toString();
} else { } else {
throw new BasicError("Unable to create a oneline version of the CommandTemplate with op type of " + op.getClass().getSimpleName()); throw new BasicError("Unable to create a oneline version of the CommandTemplate with op type of " + op.getClass().getSimpleName());
} }
@ -137,8 +143,8 @@ public class CommandTemplate {
cmd.forEach((param, value) -> { cmd.forEach((param, value) -> {
ParsedTemplate paramTemplate = new ParsedTemplate(value, bindings); ParsedTemplate paramTemplate = new ParsedTemplate(value, bindings);
if (paramTemplate.getBindPoints().size() > 0) { if (paramTemplate.getCheckedBindPoints().size() > 0) {
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints()); BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getCheckedBindPoints());
StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve(); StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();
dynamics.put(param, paramStringBindings); dynamics.put(param, paramStringBindings);
} else { } else {