parameterize op template

This commit is contained in:
Jonathan Shook 2021-06-11 15:27:11 -05:00
parent adc60e66a2
commit 748eebc04c
18 changed files with 71 additions and 68 deletions

View File

@ -2,7 +2,7 @@ package io.nosqlbench.activitytype.cql.core;
import com.datastax.driver.core.*;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.yaml.OpDef;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import java.math.BigDecimal;
import java.net.InetAddress;
@ -129,13 +129,10 @@ public class CQLBindHelper {
}
}
public static Map<String, String> parseAndGetSpecificBindings(OpDef opDef, ParsedStmt parsed) {
List<String> spans = new ArrayList<>();
public static Map<String, String> parseAndGetSpecificBindings(OpTemplate<?> opDef, ParsedStmt parsed) {
String statement = opDef.getStmt();
Set<String> extraBindings = new HashSet<>();
extraBindings.addAll(opDef.getBindings().keySet());
Set<String> extraBindings = new HashSet<>(opDef.getBindings().keySet());
Map<String, String> specificBindings = new LinkedHashMap<>();
Matcher m = stmtToken.matcher(statement);
@ -148,7 +145,6 @@ public class CQLBindHelper {
String form2 = m.group(2);
String tokenName = (form1 != null && !form1.isEmpty()) ? form1 : form2;
lastMatch = m.end();
spans.add(pre);
if (extraBindings.contains(tokenName)) {
if (specificBindings.get(tokenName) != null){

View File

@ -70,7 +70,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
private final ExceptionHistoMetrics exceptionHistoMetrics;
private final ActivityDef activityDef;
private final Map<String, Writer> namedWriters = new HashMap<>();
protected List<OpTemplate> stmts;
protected List<OpTemplate<?>> stmts;
Timer retryDelayTimer;
Timer pagesTimer;
Histogram skippedTokensHisto;
@ -180,7 +180,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
Set<String> timerStarts = new HashSet<>();
Set<String> timerStops = new HashSet<>();
for (OpTemplate stmtDef : stmts) {
for (OpTemplate<?> stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed(CqlActivity::canonicalizeBindings).orError();
boolean prepared = stmtDef.getParamOrDefault("prepared", true);
@ -282,7 +282,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.map(s -> s.split("[,: ]"))
.map(Save::new)
.ifPresent(save_op -> {
psummary.append(" save=>").append(save_op.toString());
psummary.append(" save=>").append(save_op);
template.addRowCycleOperators(save_op);
});
@ -291,7 +291,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.stream().flatMap(Arrays::stream)
.map(ResultSetCycleOperators::newOperator)
.forEach(rso -> {
psummary.append(" rsop=>").append(rso.toString());
psummary.append(" rsop=>").append(rso);
template.addResultSetOperators(rso);
});
@ -300,7 +300,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.stream().flatMap(Arrays::stream)
.map(RowCycleOperators::newOperator)
.forEach(ro -> {
psummary.append(" rowop=>").append(ro.toString());
psummary.append(" rowop=>").append(ro);
template.addRowCycleOperators(ro);
});
@ -326,7 +326,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
VerificationMetrics vmetrics = getVerificationMetrics();
RowDifferencer.ThreadLocalWrapper differencer = new RowDifferencer.ThreadLocalWrapper(vmetrics, expected, diffType);
psummary.append(" rowop=>verify-fields:").append(differencer.toString());
psummary.append(" rowop=>verify-fields:").append(differencer);
template.addResultSetOperators(new AssertSingleRowResultSet());
template.addRowCycleOperators(differencer);
@ -348,7 +348,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
template.getContextualBindings().getBindingsTemplate().addFieldBindings(stmtDef.getParsed().getBindPoints());
if (psummary.length() > 0) {
logger.info("statement named '" + stmtDef.getName() + "' has custom settings:" + psummary.toString());
logger.info("statement named '" + stmtDef.getName() + "' has custom settings:" + psummary);
}
planner.addOp(template.resolve(), ratio);

View File

@ -8,7 +8,7 @@ import java.util.List;
import java.util.Map;
public class VerifierBuilder {
public static BindingsTemplate getExpectedValuesTemplate(OpTemplate stmtDef) {
public static BindingsTemplate getExpectedValuesTemplate(OpTemplate<?> stmtDef) {
BindingsTemplate expected = new BindingsTemplate();

View File

@ -70,7 +70,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
private final ExceptionHistoMetrics exceptionHistoMetrics;
private final ActivityDef activityDef;
private final Map<String, Writer> namedWriters = new HashMap<>();
protected List<OpTemplate> stmts;
protected List<OpTemplate<?>> stmts;
Timer retryDelayTimer;
Timer pagesTimer;
private Histogram triesHisto;
@ -179,7 +179,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
Set<String> timerStarts = new HashSet<>();
Set<String> timerStops = new HashSet<>();
for (OpTemplate stmtDef : stmts) {
for (OpTemplate<?> stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed(CqlActivity::canonicalizeBindings).orError();
boolean prepared = stmtDef.getParamOrDefault("prepared", true);
@ -281,7 +281,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.map(s -> s.split("[,: ]"))
.map(Save::new)
.ifPresent(save_op -> {
psummary.append(" save=>").append(save_op.toString());
psummary.append(" save=>").append(save_op);
template.addRowCycleOperators(save_op);
});
@ -290,7 +290,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.stream().flatMap(Arrays::stream)
.map(ResultSetCycleOperators::newOperator)
.forEach(rso -> {
psummary.append(" rsop=>").append(rso.toString());
psummary.append(" rsop=>").append(rso);
template.addResultSetOperators(rso);
});
@ -299,7 +299,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
.stream().flatMap(Arrays::stream)
.map(RowCycleOperators::newOperator)
.forEach(ro -> {
psummary.append(" rowop=>").append(ro.toString());
psummary.append(" rowop=>").append(ro);
template.addRowCycleOperators(ro);
});
@ -325,7 +325,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
VerificationMetrics vmetrics = getVerificationMetrics();
RowDifferencer.ThreadLocalWrapper differencer = new RowDifferencer.ThreadLocalWrapper(vmetrics, expected, diffType);
psummary.append(" rowop=>verify-fields:").append(differencer.toString());
psummary.append(" rowop=>verify-fields:").append(differencer);
template.addResultSetOperators(new AssertSingleRowResultSet());
template.addRowCycleOperators(differencer);
@ -347,7 +347,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
template.getContextualBindings().getBindingsTemplate().addFieldBindings(stmtDef.getParsed().getBindPoints());
if (psummary.length() > 0) {
logger.info("statement named '" + stmtDef.getName() + "' has custom settings:" + psummary.toString());
logger.info("statement named '" + stmtDef.getName() + "' has custom settings:" + psummary);
}
planner.addOp(template.resolve(), ratio);

View File

@ -8,7 +8,7 @@ import java.util.List;
import java.util.Map;
public class VerifierBuilder {
public static BindingsTemplate getExpectedValuesTemplate(OpTemplate stmtDef) {
public static BindingsTemplate getExpectedValuesTemplate(OpTemplate<?> stmtDef) {
BindingsTemplate expected = new BindingsTemplate();

View File

@ -46,7 +46,7 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
public Timer resultTimer;
public Timer logicalGraphOps;
public Histogram triesHisto;
protected List<OpTemplate> stmts;
protected List<OpTemplate<?>> stmts;
private int stride;
private DseSession session;
private DseCluster cluster;
@ -100,7 +100,7 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
throw new RuntimeException("There were no unfiltered statements found for this activity.");
}
for (OpTemplate stmtDef : stmts) {
for (OpTemplate<?> stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed().orError();
@ -209,7 +209,7 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
try {
cluster = builder.build();
} catch (Exception e) {
logger.error("Error while instantiating cluster from builder: " + e.toString(), e);
logger.error("Error while instantiating cluster from builder: " + e, e);
throw e;
}
activityDef.getParams().getOptionalBoolean("defaultidempotence").map(
@ -236,7 +236,7 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
logger.info("cluster-metadata-allhosts:\n" + session.getCluster().getMetadata().getAllHosts());
return session;
} catch (Exception e) {
logger.error("Error while creating a session for dsegraph: " + e.toString(), e);
logger.error("Error while creating a session for dsegraph: " + e, e);
throw e;
}

View File

@ -12,7 +12,7 @@ public class ReadyJDBCOp implements OpDispenser<String> {
public ReadyJDBCOp(OpTemplate stmtDef) {
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getCheckedBindPoints());
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings);
bindings = template.resolve();

View File

@ -73,12 +73,12 @@ public class KafkaProducerActivity extends SimpleActivity {
String tagFilter = activityDef.getParams().getOptionalString("tags").orElse("");
StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yamlLoc, new StrInterpolator(activityDef),
"activities");
List<OpTemplate> statements = stmtsDocList.getStmts(tagFilter);
List<OpTemplate<?>> statements = stmtsDocList.getStmts(tagFilter);
String format = getParams().getOptionalString("format").orElse(null);
if (statements.size() > 0) {
for (OpTemplate statement : statements) {
for (OpTemplate<?> statement : statements) {
sequencer.addOp(
new KafkaStatement(statement,
servers,

View File

@ -31,9 +31,9 @@ public class KafkaStatement {
private AvroSchema valueSerializerSchema = null;
private final String key;
public KafkaStatement(OpTemplate stmtDef, String servers, String clientId, String schemaRegistryUrl) {
public KafkaStatement(OpTemplate<?> stmtDef, String servers, String clientId, String schemaRegistryUrl) {
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getCheckedBindPoints());
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings);
this.bindings = template.resolve();

View File

@ -112,11 +112,11 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
TagFilter tagFilter = new TagFilter(tagfilter);
stmtsDocList.getStmts().stream().map(tagFilter::matchesTaggedResult).forEach(r -> logger.info(r.getLog()));
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
List<OpTemplate<?>> stmts = stmtsDocList.getStmts(tagfilter);
if (stmts.isEmpty()) {
logger.error("No statements found for this activity");
} else {
for (OpTemplate stmt : stmts) {
for (OpTemplate<?> stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
String statement = parsed.getPositionalStatement(Function.identity());
Objects.requireNonNull(statement);

View File

@ -14,9 +14,9 @@ public class ReadyMongoStatement {
private final StringBindings bindings;
private final ReadPreference readPreference;
public ReadyMongoStatement(OpTemplate stmtDef) {
public ReadyMongoStatement(OpTemplate<?> stmtDef) {
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getCheckedBindPoints());
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings);
this.bindings = template.resolve();

View File

@ -81,7 +81,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
public void shutdownActivity() {
try {
if (pw != null) {
if (!fileName.toLowerCase().equals("stdout")) {
if (!fileName.equalsIgnoreCase("stdout")) {
logger.trace("Closing non-stdout output stream.");
pw.close();
}
@ -112,7 +112,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
protected Writer createPrintWriter() {
PrintWriter pw;
if (fileName.toLowerCase().equals("stdout")) {
if (fileName.equalsIgnoreCase("stdout")) {
pw = getConsoleOut();
} else {
try {
@ -134,7 +134,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
SequencePlanner<StringBindings> sequencer = new SequencePlanner<>(sequencerType);
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
List<OpTemplate<?>> stmts = stmtsDocList.getStmts(tagfilter);
String format = getParams().getOptionalString("format").orElse(null);
@ -155,7 +155,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
String bindings = getActivityDef().getParams().getOptionalString("bindings").orElse("doc");
activeBindingNames.addAll(stmtsDocList.getDocBindings().keySet());
Pattern bindingsFilter = Pattern.compile(bindings.toLowerCase().equals("doc") ? ".*" : bindings);
Pattern bindingsFilter = Pattern.compile(bindings.equalsIgnoreCase("doc") ? ".*" : bindings);
Set<String> filteredBindingNames = new LinkedHashSet<>();
activeBindingNames
.stream()
@ -179,7 +179,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
sequencer.addOp(sb, 1L);
}
} else if (stmts.size() > 0) {
for (OpTemplate stmt : stmts) {
for (OpTemplate<?> stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
BindingsTemplate bt = new BindingsTemplate(parsed.getBindPoints());
String statement = parsed.getPositionalStatement(Function.identity());

View File

@ -127,7 +127,7 @@ public class WebDriverActivity extends SimpleActivity {
cmdBuilder.append(" value='").append(command.getValue().replaceAll("'","\\\\'")).append("'");
}
cmdBuilder.append("\n");
logger.debug("build cmd: '" + testBuilder.toString() + "'");
logger.debug("build cmd: '" + testBuilder + "'");
String cmdName = sideName+"_"+String.format("%s_%03d", sideName, ++idx);
commands.put(cmdName,cmdBuilder.toString());
}
@ -180,13 +180,13 @@ public class WebDriverActivity extends SimpleActivity {
SequencePlanner<CommandTemplate> planner = new SequencePlanner<>(sequencerType);
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
List<OpTemplate<?>> stmts = stmtsDocList.getStmts(tagfilter);
if (stmts.size() == 0) {
throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'");
}
for (OpTemplate optemplate : stmts) {
for (OpTemplate<?> optemplate : stmts) {
long ratio = optemplate.getParamOrDefault("ratio", 1);
CommandTemplate cmd = new CommandTemplate(optemplate);
planner.addOp(cmd, ratio);

View File

@ -29,7 +29,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
public class OpDef implements OpTemplate {
public class OpDef<T> implements OpTemplate<T> {
private static final String FIELD_DESC = "description";
private static final String FIELD_NAME = "name";
@ -58,8 +58,8 @@ public class OpDef implements OpTemplate {
}
@Override
public Object getOp() {
return rawStmtDef.getOp();
public T getOp() {
return (T) rawStmtDef.getOp();
}
@Override

View File

@ -121,19 +121,19 @@ import java.util.function.Function;
* p2: v2
* }</pre>
*/
public interface OpTemplate extends Tagged {
public interface OpTemplate<T> extends Tagged {
String getName();
Object getOp();
T getOp();
Map<String, String> getBindings();
Map<String, Object> getParams();
<T> Map<String, T> getParamsAsValueType(Class<? extends T> type);
<V> Map<String, V> getParamsAsValueType(Class<? extends V> type);
<V> V removeParamOrDefault(String name, V defaultValue);
<VT> VT removeParamOrDefault(String name, VT defaultValue);
@SuppressWarnings("unchecked")
<V> V getParamOrDefault(String name, V defaultValue);

View File

@ -20,10 +20,7 @@ package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.util.TagFilter;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
public class StmtsDocList implements Iterable<StmtsDoc> {
@ -47,7 +44,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
.collect(Collectors.toList());
}
public List<OpTemplate> getStmts() {
public List<OpTemplate<?>> getStmts() {
return getStmts("");
}
@ -56,14 +53,17 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
* @return The list of all included statements for all included blocks of in this document,
* including the inherited and overridden values from the this doc and the parent block.
*/
public List<OpTemplate> getStmts(String tagFilterSpec) {
public List<OpTemplate<?>> getStmts(String tagFilterSpec) {
TagFilter ts = new TagFilter(tagFilterSpec);
List<OpTemplate<?>> opTemplates = new ArrayList<>();
List<OpTemplate> stmts = getStmtDocs().stream()
getStmtDocs().stream()
.flatMap(d -> d.getStmts().stream())
.filter(ts::matchesTagged)
.collect(Collectors.toList());
return stmts;
.forEach(opTemplates::add);
return opTemplates;
}

View File

@ -468,10 +468,11 @@ public class SimpleActivity implements Activity, ProgressCapable {
stmtsDocList = StatementsLoader.loadPath(logger, op_yaml_loc.get(), interp, "activities");
}
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
List<OpTemplate<?>> stmts = stmtsDocList.getStmts(tagfilter);
List<Long> ratios = new ArrayList<>(stmts.size());
for (int i = 0; i < stmts.size(); i++) {
long ratio = stmts.get(i).removeParamOrDefault("ratio", 1);
OpTemplate<?> opTemplate = stmts.get(i);
long ratio = opTemplate.removeParamOrDefault("ratio", 1);
ratios.add(ratio);
}

View File

@ -48,7 +48,7 @@ public class CommandTemplate {
*
* @param optpl An OpTemplate
*/
public CommandTemplate(OpTemplate optpl) {
public CommandTemplate(OpTemplate<?> optpl) {
this(optpl.getName(), optpl.getOp().toString(), optpl.getParamsAsValueType(String.class), optpl.getBindings(), List.of());
}
@ -66,7 +66,7 @@ public class CommandTemplate {
* @param optpl An OpTemplate
* @param parsers A list of parser functions
*/
public CommandTemplate(OpTemplate optpl, List<Function<String, Map<String, String>>> parsers) {
public CommandTemplate(OpTemplate<?> optpl, List<Function<String, Map<String, String>>> parsers) {
this(optpl.getName(), optpl.getOp(), optpl.getParamsAsValueType(String.class), optpl.getBindings(), parsers);
}
@ -92,7 +92,13 @@ public class CommandTemplate {
* @param bindings A set of named bindings in name:recipe form.
* @param optionalParsers A set of functions which, if provided, will be used to read the oneline form.
*/
public CommandTemplate(String name, Object op, Map<String, String> params, Map<String, String> bindings, List<Function<String, Map<String, String>>> optionalParsers) {
public CommandTemplate(
String name,
Object op,
Map<String, String> params,
Map<String, String> bindings,
List<Function<String, Map<String, String>>> optionalParsers
) {
this.name = name;
Map<String, String> cmd = new HashMap<>();
@ -103,7 +109,7 @@ public class CommandTemplate {
String oneline;
if (op instanceof CharSequence) {
oneline=op.toString();
oneline = op.toString();
} else {
throw new BasicError("Unable to create a oneline version of the CommandTemplate with op type of " + op.getClass().getSimpleName());
}
@ -137,8 +143,8 @@ public class CommandTemplate {
cmd.forEach((param, value) -> {
ParsedTemplate paramTemplate = new ParsedTemplate(value, bindings);
if (paramTemplate.getBindPoints().size() > 0) {
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
if (paramTemplate.getCheckedBindPoints().size() > 0) {
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getCheckedBindPoints());
StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();
dynamics.put(param, paramStringBindings);
} else {