From 0164c49f97c1ab3bb8d94f898ff7374717e93fcd Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 23 May 2022 17:08:30 -0500 Subject: [PATCH] Provide activity parameter default strict=true for driver adapters, which requires strict usage checking on op fields. Unused op fields will cause an error by default. --- .../api/activityconfig/yaml/OpTemplate.java | 23 +++++++++++++++ .../uniform/BaseDriverAdapter.java | 1 + .../activitytype/http/HttpActivity.java | 2 +- .../activitytype/jdbc/api/JDBCActivity.java | 2 +- .../io/nosqlbench/driver/jms/JmsActivity.java | 2 +- .../io/nosqlbench/driver/jmx/JMXActivity.java | 2 +- .../driver/pulsar/PulsarActivity.java | 2 +- .../api/activityimpl/SimpleActivity.java | 28 ++++++++++++------- .../uniform/StandardActivity.java | 5 ++-- 9 files changed, 50 insertions(+), 17 deletions(-) diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java index a9fe2639e..78c11e6ba 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java @@ -21,6 +21,7 @@ import com.google.gson.GsonBuilder; import io.nosqlbench.engine.api.util.Tagged; import io.nosqlbench.nb.api.config.params.Element; import io.nosqlbench.nb.api.config.params.NBParams; +import io.nosqlbench.nb.api.errors.OpConfigError; import io.nosqlbench.virtdata.core.templates.ParsedTemplate; import java.util.LinkedHashMap; @@ -332,4 +333,26 @@ public abstract class OpTemplate implements Tagged { public Element getParamReader() { return NBParams.one(getName(),getParams()); } + + /** + * @return the size of remaining fields from the op template and the params map. + */ + public int size() { + return getOp().map(Map::size).orElse(0) + getParams().size(); + } + + /** + * @return the map of all remaining fields from the op template and the params map. + */ + public Map remainingFields() { + Map remaining = new LinkedHashMap<>(getOp().orElse(Map.of())); + remaining.putAll(getParams()); + return remaining; + } + + public void assertConsumed() { + if (size()>0) { + throw new OpConfigError("The op template named '" + getName() + "' was not fully consumed. These fields are not being applied:" + remainingFields()); + } + } } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java index bcf13941a..98dfcba0b 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java @@ -120,6 +120,7 @@ public abstract class BaseDriverAdapter implements DriverAdapter public NBConfigModel getConfigModel() { return ConfigModel.of(BaseDriverAdapter.class) .add(Param.optional("alias")) + .add(Param.defaultTo("strict",true,"strict op field mode, which requires that provided op fields are recognized and used")) .add(Param.optional(List.of("op", "stmt", "statement"), String.class, "op template in statement form")) .add(Param.optional(List.of("workload", "yaml"), String.class, "location of workload yaml file")) .add(Param.optional("tags", String.class, "tags to be used to filter operations")) diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java b/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java index 1215c31b4..df2ea7f01 100644 --- a/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java @@ -75,7 +75,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe statusCodeHisto = ActivityMetrics.histogram(activityDef, "statuscode"); skippedTokens = ActivityMetrics.histogram(activityDef, "skipped-tokens"); resultSuccessTimer = ActivityMetrics.timer(activityDef, "result-success"); - this.sequencer = createOpSequence(ReadyHttpOp::new); + this.sequencer = createOpSequence(ReadyHttpOp::new, false); setDefaultsFromOpSequence(sequencer); onActivityDefUpdate(activityDef); } diff --git a/driver-jdbc/src/main/java/io/nosqlbench/activitytype/jdbc/api/JDBCActivity.java b/driver-jdbc/src/main/java/io/nosqlbench/activitytype/jdbc/api/JDBCActivity.java index 40dc3b96a..31e04ef42 100644 --- a/driver-jdbc/src/main/java/io/nosqlbench/activitytype/jdbc/api/JDBCActivity.java +++ b/driver-jdbc/src/main/java/io/nosqlbench/activitytype/jdbc/api/JDBCActivity.java @@ -88,7 +88,7 @@ public abstract class JDBCActivity extends SimpleActivity { resultSuccessTimer = ActivityMetrics.timer(getActivityDef(), "result-success"); triesHisto = ActivityMetrics.histogram(getActivityDef(), "tries"); - opSequence = createOpSequence(ReadyJDBCOp::new); + opSequence = createOpSequence(ReadyJDBCOp::new,false); setDefaultsFromOpSequence(opSequence); onActivityDefUpdate(getActivityDef()); diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java index e5dd92d9d..d2c5a9cf8 100644 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java +++ b/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java @@ -114,7 +114,7 @@ public class JmsActivity extends SimpleActivity { messagesizeHistogram = ActivityMetrics.histogram(activityDef, "messagesize"); if (StringUtils.equalsIgnoreCase(jmsProviderType, JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label )) { - this.sequence = createOpSequence((ot) -> new ReadyPulsarJmsOp(ot, this)); + this.sequence = createOpSequence((ot) -> new ReadyPulsarJmsOp(ot, this), false); } setDefaultsFromOpSequence(sequence); diff --git a/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/JMXActivity.java b/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/JMXActivity.java index d6c625025..3540dee46 100644 --- a/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/JMXActivity.java +++ b/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/JMXActivity.java @@ -39,7 +39,7 @@ public class JMXActivity extends SimpleActivity implements Activity { @Override public void initActivity() { super.initActivity(); - this.sequence = createOpSequenceFromCommands(ReadyJmxOp::new); + this.sequence = createOpSequenceFromCommands(ReadyJmxOp::new, false); setDefaultsFromOpSequence(sequence); NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams()); this.sslContext= SSLKsFactory.get().getContext(sslCfg); diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivity.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivity.java index 117a247bb..81b98f2b9 100644 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivity.java +++ b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivity.java @@ -136,7 +136,7 @@ public class PulsarActivity extends SimpleActivity implements ActivityDefObserve createPulsarSchemaFromConf(); - this.sequencer = createOpSequence((ot) -> new ReadyPulsarOp(ot, pulsarCache, this)); + this.sequencer = createOpSequence((ot) -> new ReadyPulsarOp(ot, pulsarCache, this), false); setDefaultsFromOpSequence(sequencer); onActivityDefUpdate(activityDef); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java index ffd7d1839..357b4bb4e 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java @@ -38,7 +38,6 @@ import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.engine.api.metrics.ActivityMetrics; import io.nosqlbench.engine.api.templating.CommandTemplate; import io.nosqlbench.engine.api.templating.ParsedOp; -import io.nosqlbench.engine.api.templating.StrInterpolator; import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.nb.api.errors.OpConfigError; @@ -324,7 +323,7 @@ public class SimpleActivity implements Activity, ProgressCapable { activityDef.getParams().getOptionalNamedParameter("cyclerate", "targetrate", "rate") .map(RateSpec::new).ifPresent( - spec -> cycleLimiter = RateLimiters.createOrUpdate(this.getActivityDef(), "cycles", cycleLimiter, spec)); + spec -> cycleLimiter = RateLimiters.createOrUpdate(this.getActivityDef(), "cycles", cycleLimiter, spec)); activityDef.getParams().getOptionalNamedParameter("phaserate") .map(RateSpec::new) @@ -424,32 +423,38 @@ public class SimpleActivity implements Activity, ProgressCapable { * Given a function that can create an op of type from a CommandTemplate, generate * an indexed sequence of ready to call operations. * - * This method works almost exactly like the {@link #createOpSequenceFromCommands(Function)}, + * This method works almost exactly like the {@link #createOpSequenceFromCommands(Function, boolean)}, * except that it uses the {@link CommandTemplate} semantics, which are more general and allow * for map-based specification of operations with bindings in each field. * * It is recommended to use the CommandTemplate form * than the * - * @param opinit * @param + * @param opinit + * @param strict * @return */ - protected OpSequence> createOpSequenceFromCommands(Function> opinit) { + protected OpSequence> createOpSequenceFromCommands( + Function> opinit, + boolean strict + ) { Function f = CommandTemplate::new; Function> opTemplateOFunction = f.andThen(opinit); - return createOpSequence(opTemplateOFunction); + return createOpSequence(opTemplateOFunction, strict); } protected OpSequence> createOpSourceFromCommands( Function> opinit, NBConfiguration cfg, - List, Map>> parsers + List, Map>> parsers, + boolean strict ) { Function f = t -> new ParsedOp(t, cfg, parsers); Function> opTemplateOFunction = f.andThen(opinit); - return createOpSequence(opTemplateOFunction); + + return createOpSequence(opTemplateOFunction, strict); } /** @@ -475,7 +480,7 @@ public class SimpleActivity implements Activity, ProgressCapable { * @return The sequence of operations as determined by filtering and ratios */ @Deprecated(forRemoval = true) - protected OpSequence> createOpSequence(Function> opinit) { + protected OpSequence> createOpSequence(Function> opinit, boolean strict) { String tagfilter = activityDef.getParams().getOptionalString("tags").orElse(""); // StrInterpolator interp = new StrInterpolator(activityDef); SequencerType sequencerType = getParams() @@ -517,10 +522,13 @@ public class SimpleActivity implements Activity, ProgressCapable { long ratio = ratios.get(i); OpTemplate optemplate = stmts.get(i); OpDispenser driverSpecificReadyOp = opinit.apply(optemplate); + if (strict) { + optemplate.assertConsumed(); + } planner.addOp(driverSpecificReadyOp, ratio); } } catch (Exception e) { - throw new OpConfigError(e.getMessage(),workloadSource,e); + throw new OpConfigError(e.getMessage(), workloadSource, e); } return planner.resolve(); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java index 010ef0e66..816b3887a 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java @@ -29,7 +29,7 @@ import java.util.Map; import java.util.function.Function; /** - * This is a typed activity which is expected to be come the standard + * This is a typed activity which is expected to become the standard * core of all new activity types. Extant NB drivers should also migrate * to this when possible. * @@ -48,7 +48,8 @@ public class StandardActivity extends SimpleActivity { try { OpMapper opmapper = adapter.getOpMapper(); Function, Map> preprocessor = adapter.getPreprocessor(); - sequence = createOpSourceFromCommands(opmapper, adapter.getConfiguration(), List.of(preprocessor)); + boolean strict = activityDef.getParams().getOptionalBoolean("strict").orElse(true); + sequence = createOpSourceFromCommands(opmapper, adapter.getConfiguration(), List.of(preprocessor), strict); } catch (Exception e) { if (e instanceof OpConfigError) { throw e;