diff --git a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java index 3ebb73b79..617aedca4 100644 --- a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java @@ -16,6 +16,11 @@ import io.nosqlbench.activitytype.cql.statements.core.*; import io.nosqlbench.activitytype.cql.statements.modifiers.StatementModifier; import io.nosqlbench.activitytype.cql.statements.rowoperators.RowCycleOperators; import io.nosqlbench.activitytype.cql.statements.rowoperators.Save; +import io.nosqlbench.activitytype.cql.statements.rowoperators.verification.DiffType; +import io.nosqlbench.activitytype.cql.statements.rowoperators.verification.RowDifferencer; +import io.nosqlbench.activitytype.cql.statements.rowoperators.verification.VerificationMetrics; +import io.nosqlbench.activitytype.cql.statements.rowoperators.verification.VerifierBuilder; +import io.nosqlbench.activitytype.cql.statements.rsoperators.AssertSingleRowResultSet; import io.nosqlbench.activitytype.cql.statements.rsoperators.ResultSetCycleOperators; import io.nosqlbench.activitytype.cql.statements.rsoperators.TraceLogger; import io.nosqlbench.engine.api.activityapi.core.Activity; @@ -44,6 +49,7 @@ import io.nosqlbench.engine.api.util.TagFilter; import io.nosqlbench.engine.api.util.Unit; import io.nosqlbench.nb.api.config.params.Element; import io.nosqlbench.nb.api.errors.BasicError; +import io.nosqlbench.virtdata.core.bindings.Bindings; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -88,6 +94,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef private long maxRetryDelay; private boolean retryReplace; private String pooling; + private VerificationMetrics verificationMetrics; public CqlActivity(ActivityDef activityDef) { @@ -141,7 +148,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef Map fconfig = Map.of("session", session); SequencerType sequencerType = SequencerType.valueOf( - getParams().getOptionalString("seq").orElse("bucket") + getParams().getOptionalString("seq").orElse("bucket") ); SequencePlanner planner = new SequencePlanner<>(sequencerType); @@ -180,7 +187,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef boolean instrument = stmtDef.getOptionalStringParam("instrument", Boolean.class) .or(() -> getParams().getOptionalBoolean("instrument")) - .orElse(false); + .orElse(false); String logresultcsv = stmtDef.getParamOrDefault("logresultcsv", ""); @@ -215,11 +222,11 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef }); CqlBinderTypes binderType = stmtDef.getOptionalStringParam("binder") - .map(CqlBinderTypes::valueOf) - .orElse(CqlBinderTypes.DEFAULT); + .map(CqlBinderTypes::valueOf) + .orElse(CqlBinderTypes.DEFAULT); template = new ReadyCQLStatementTemplate(fconfig, binderType, getSession(), prepare, ratio, - parsed.getName()); + parsed.getName()); } else { SimpleStatement simpleStatement = new SimpleStatement(stmtForDriver); cl.ifPresent((conlvl) -> { @@ -271,22 +278,51 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef }); stmtDef.getOptionalStringParam("rsoperators") - .map(s -> s.split(",")) - .stream().flatMap(Arrays::stream) - .map(ResultSetCycleOperators::newOperator) - .forEach(rso -> { - psummary.append(" rsop=>").append(rso.toString()); - template.addResultSetOperators(rso); - }); + .map(s -> s.split(",")) + .stream().flatMap(Arrays::stream) + .map(ResultSetCycleOperators::newOperator) + .forEach(rso -> { + psummary.append(" rsop=>").append(rso.toString()); + template.addResultSetOperators(rso); + }); stmtDef.getOptionalStringParam("rowoperators") - .map(s -> s.split(",")) - .stream().flatMap(Arrays::stream) - .map(RowCycleOperators::newOperator) - .forEach(ro -> { - psummary.append(" rowop=>").append(ro.toString()); - template.addRowCycleOperators(ro); - }); + .map(s -> s.split(",")) + .stream().flatMap(Arrays::stream) + .map(RowCycleOperators::newOperator) + .forEach(ro -> { + psummary.append(" rowop=>").append(ro.toString()); + template.addRowCycleOperators(ro); + }); + + + // If verify is set on activity, assume all fields should be verified for every + // statement, otherwise, do per-statement verification for ops which have + // a verify param + + if (activityDef.getParams().containsKey("verify") || + stmtDef.getParams().containsKey("verify") || + stmtDef.getParams().containsKey("verify-fields")) { + + String verify = stmtDef.getOptionalStringParam("verify") + .or(() -> stmtDef.getOptionalStringParam("verify-fields")) + .or(() -> activityDef.getParams().getOptionalString("verify")) + .orElse("*"); + + DiffType diffType = stmtDef.getOptionalStringParam("compare") + .or(() -> activityDef.getParams().getOptionalString("compare")) + .map(DiffType::valueOf).orElse(DiffType.reffields); + + Bindings expected = VerifierBuilder.getExpectedValuesTemplate(stmtDef).resolveBindings(); + VerificationMetrics vmetrics = getVerificationMetrics(); + + RowDifferencer.ThreadLocalWrapper differencer = new RowDifferencer.ThreadLocalWrapper(vmetrics, expected, diffType); + psummary.append(" rowop=>verify-fields:").append(differencer.toString()); + + template.addResultSetOperators(new AssertSingleRowResultSet()); + template.addRowCycleOperators(differencer); + } + if (instrument) { logger.info("Adding per-statement success and error and resultset-size timers to statement '" + parsed.getName() + "'"); @@ -318,6 +354,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef } + private synchronized VerificationMetrics getVerificationMetrics() { + if (verificationMetrics == null) { + verificationMetrics = new VerificationMetrics(this.activityDef); + } + return verificationMetrics; + } + private StmtsDocList loadStmtsYaml() { StmtsDocList doclist = null; @@ -336,9 +379,9 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef case "1": doclist = getVersion1StmtsDoc(interp, yaml_loc); logger.warn("DEPRECATED-FORMAT: Loaded yaml " + yaml_loc + " with compatibility mode. " + - "This will be deprecated in a future release."); + "This will be deprecated in a future release."); logger.warn("DEPRECATED-FORMAT: Please refer to " + - "http://docs.engineblock.io/user-guide/standard_yaml/ for more details."); + "http://docs.engineblock.io/user-guide/standard_yaml/ for more details."); break; case "2": doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities"); @@ -346,22 +389,22 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef case "unset": try { logger.debug("You can suffix your yaml filename or url with the " + - "format version, such as :1 or :2. Assuming version 2."); + "format version, such as :1 or :2. Assuming version 2."); doclist = StatementsLoader.loadPath(null, yaml_loc, interp, "activities"); } catch (Exception ignored) { try { doclist = getVersion1StmtsDoc(interp, yaml_loc); logger.warn("DEPRECATED-FORMAT: Loaded yaml " + yaml_loc + - " with compatibility mode. This will be deprecated in a future release."); + " with compatibility mode. This will be deprecated in a future release."); logger.warn("DEPRECATED-FORMAT: Please refer to " + - "http://docs.engineblock.io/user-guide/standard_yaml/ for more details."); + "http://docs.engineblock.io/user-guide/standard_yaml/ for more details."); } catch (Exception compatError) { logger.warn("Tried to load yaml in compatibility mode, " + - "since it failed to load with the standard format, " + - "but found an error:" + compatError); + "since it failed to load with the standard format, " + + "but found an error:" + compatError); logger.warn("The following detailed errors are provided only " + - "for the standard format. To force loading version 1 with detailed logging, add" + - " a version qualifier to your yaml filename or url like ':1'"); + "for the standard format. To force loading version 1 with detailed logging, add" + + " a version qualifier to your yaml filename or url like ':1'"); // retrigger the error again, this time with logging enabled. doclist = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities"); } @@ -369,7 +412,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef break; default: throw new RuntimeException("Unrecognized yaml format version, expected :1 or :2 " + - "at end of yaml file, but got " + yamlVersion + " instead."); + "at end of yaml file, but got " + yamlVersion + " instead."); } return doclist; @@ -429,13 +472,33 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef return exceptionCountMetrics; } + @Override + public void shutdownActivity() { + super.shutdownActivity(); + + if (verificationMetrics != null) { + + VerificationMetrics metrics = getVerificationMetrics(); + long unverifiedValues = metrics.unverifiedValuesCounter.getCount(); + long unverifiedRows = metrics.unverifiedRowsCounter.getCount(); + + if (unverifiedRows > 0 || unverifiedValues > 0) { + throw new RuntimeException( + "There were " + unverifiedValues + " unverified values across " + unverifiedRows + " unverified rows." + ); + } + logger.info("verified " + metrics.verifiedValuesCounter.getCount() + " values across " + metrics.verifiedRowsCounter.getCount() + " verified rows"); + } + + } + @Override public String toString() { return "CQLActivity {" + - "activityDef=" + activityDef + - ", session=" + session + - ", opSequence=" + this.opsequence + - '}'; + "activityDef=" + activityDef + + ", session=" + session + + ", opSequence=" + this.opsequence + + '}'; } @Override @@ -451,7 +514,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef Cluster cluster = getSession().getCluster(); if (fetchSizeOption.isPresent()) { int fetchSize = fetchSizeOption.flatMap(Unit::bytesFor).map(Double::intValue).orElseThrow(() -> new RuntimeException( - "Unable to parse fetch size from " + fetchSizeOption.get() + "Unable to parse fetch size from " + fetchSizeOption.get() )); if (fetchSize > 10000000 && fetchSize < 1000000000) { logger.warn("Setting the fetchsize to " + fetchSize + " is unlikely to give good performance."); @@ -470,8 +533,8 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef this.maxpages = params.getOptionalInteger("maxpages").orElse(1); this.statementFilter = params.getOptionalString("tokens") - .map(s -> new TokenRangeStmtFilter(cluster, s)) - .orElse(null); + .map(s -> new TokenRangeStmtFilter(cluster, s)) + .orElse(null); if (statementFilter != null) { logger.info("filtering statements" + statementFilter); @@ -480,13 +543,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef errorHandler = configureErrorHandler(); params.getOptionalString("trace") - .map(SimpleConfig::new) - .map(TraceLogger::new) - .ifPresent( - tl -> { - addResultSetCycleOperator(tl); - addStatementModifier(tl); - }); + .map(SimpleConfig::new) + .map(TraceLogger::new) + .ifPresent( + tl -> { + addResultSetCycleOperator(tl); + addStatementModifier(tl); + }); this.maxTotalOpsInFlight = params.getOptionalLong("async").orElse(1L); @@ -543,8 +606,8 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef HashedCQLErrorHandler newerrorHandler = new HashedCQLErrorHandler(exceptionCountMetrics); String errors = activityDef.getParams() - .getOptionalString("errors") - .orElse("stop,retryable->retry,unverified->stop"); + .getOptionalString("errors") + .orElse("stop,retryable->retry,unverified->stop"); String[] handlerSpecs = errors.split(","); @@ -554,12 +617,12 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef String verb = keyval[0]; ErrorResponse errorResponse = getErrorResponseOrBasicError(verb); newerrorHandler.setDefaultHandler( - new NBCycleErrorHandler( - errorResponse, - exceptionCountMetrics, - exceptionHistoMetrics, - !getParams().getOptionalLong("async").isPresent() - ) + new NBCycleErrorHandler( + errorResponse, + exceptionCountMetrics, + exceptionHistoMetrics, + !getParams().getOptionalLong("async").isPresent() + ) ); } else { String pattern = keyval[0]; @@ -567,21 +630,21 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef if (newerrorHandler.getGroupNames().contains(pattern)) { ErrorResponse errorResponse = getErrorResponseOrBasicError(verb); NBCycleErrorHandler handler = - new NBCycleErrorHandler( - errorResponse, - exceptionCountMetrics, - exceptionHistoMetrics, - !getParams().getOptionalLong("async").isPresent() - ); + new NBCycleErrorHandler( + errorResponse, + exceptionCountMetrics, + exceptionHistoMetrics, + !getParams().getOptionalLong("async").isPresent() + ); logger.info("Handling error group '" + pattern + "' with handler:" + handler); newerrorHandler.setHandlerForGroup(pattern, handler); } else { ErrorResponse errorResponse = ErrorResponse.valueOf(keyval[1]); NBCycleErrorHandler handler = new NBCycleErrorHandler( - errorResponse, - exceptionCountMetrics, - exceptionHistoMetrics, - !getParams().getOptionalLong("async").isPresent() + errorResponse, + exceptionCountMetrics, + exceptionHistoMetrics, + !getParams().getOptionalLong("async").isPresent() ); logger.info("Handling error pattern '" + pattern + "' with handler:" + handler); newerrorHandler.setHandlerForPattern(keyval[0], handler); diff --git a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/RowCycleOperators.java b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/RowCycleOperators.java index 37264a2fe..2e05ca7f5 100644 --- a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/RowCycleOperators.java +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/RowCycleOperators.java @@ -15,7 +15,6 @@ public enum RowCycleOperators { this.implClass = traceLoggerClass; } - public Class getImplementation() { return implClass; } diff --git a/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/DiffType.java b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/DiffType.java similarity index 80% rename from driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/DiffType.java rename to driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/DiffType.java index f882bf38c..385d692dc 100644 --- a/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/DiffType.java +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/DiffType.java @@ -1,16 +1,19 @@ -package io.nosqlbench.activitytype.cqlverify; +package io.nosqlbench.activitytype.cql.statements.rowoperators.verification; public enum DiffType { + /// Verify nothing for this statement + none(0), + /// Verify that fields named in the row are present in the reference map. rowfields(0x1), /// Verify that fields in the reference map are present in the row data. - reffields(0x1<<1), + reffields(0x1 << 1), /// Verify that all fields present in either the row or the reference data /// are also present in the other. - fields(0x1|0x1<<1), + fields(0x1 | 0x1 << 1), /// Verify that all values of the same named field are equal, according to /// {@link Object#equals(Object)}}. diff --git a/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/RowDifferencer.java b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/RowDifferencer.java similarity index 99% rename from driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/RowDifferencer.java rename to driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/RowDifferencer.java index 349ec476c..8f8d6ea15 100644 --- a/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/RowDifferencer.java +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/RowDifferencer.java @@ -1,4 +1,4 @@ -package io.nosqlbench.activitytype.cqlverify; +package io.nosqlbench.activitytype.cql.statements.rowoperators.verification; import com.datastax.driver.core.*; import io.nosqlbench.activitytype.cql.api.RowCycleOperator; @@ -295,7 +295,7 @@ public class RowDifferencer implements RowCycleOperator { private final VerificationMetrics metrics; private final Bindings bindings; private final DiffType diffType; - private ThreadLocal tl; + private final ThreadLocal tl; public ThreadLocalWrapper(VerificationMetrics metrics, Bindings bindings, DiffType diffType) { this.metrics = metrics; diff --git a/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/VerificationMetrics.java b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/VerificationMetrics.java similarity index 91% rename from driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/VerificationMetrics.java rename to driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/VerificationMetrics.java index 9e0c7803a..2f6ea4e47 100644 --- a/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/VerificationMetrics.java +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/VerificationMetrics.java @@ -1,4 +1,4 @@ -package io.nosqlbench.activitytype.cqlverify; +package io.nosqlbench.activitytype.cql.statements.rowoperators.verification; import com.codahale.metrics.Counter; import io.nosqlbench.engine.api.activityimpl.ActivityDef; diff --git a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/VerifierBuilder.java b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/VerifierBuilder.java new file mode 100644 index 000000000..bac12f7db --- /dev/null +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/verification/VerifierBuilder.java @@ -0,0 +1,53 @@ +package io.nosqlbench.activitytype.cql.statements.rowoperators.verification; + +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; +import io.nosqlbench.virtdata.core.bindings.BindingsTemplate; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class VerifierBuilder { + public static BindingsTemplate getExpectedValuesTemplate(OpTemplate stmtDef) { + + BindingsTemplate expected = new BindingsTemplate(); + + if (!stmtDef.getParams().containsKey("verify-fields") && !stmtDef.getParams().containsKey("verify")) { + throw new RuntimeException("Unable to create expected values template with no 'verify' param"); + } + + Map reading = stmtDef.getBindings(); + + List fields = new ArrayList<>(); + String fieldSpec = stmtDef.getOptionalStringParam("verify-fields") + .or(() -> stmtDef.getOptionalStringParam("verify")) + .orElse("*"); + + String[] vfields = fieldSpec.split("\\s*,\\s*"); + for (String vfield : vfields) { + if (vfield.equals("*")) { + reading.forEach((k, v) -> fields.add(k)); + } else if (vfield.startsWith("+")) { + fields.add(vfield.substring(1)); + } else if (vfield.startsWith("-")) { + fields.remove(vfield.substring(1)); + } else if (vfield.matches("\\w+(\\w+->[\\w-]+)?")) { + fields.add(vfield); + } else { + throw new RuntimeException("unknown verify-fields format: '" + vfield + "'"); + } + } + for (String vfield : fields) { + String[] fieldNameAndBindingName = vfield.split("\\s*->\\s*", 2); + String fieldName = fieldNameAndBindingName[0]; + String bindingName = fieldNameAndBindingName.length == 1 ? fieldName : fieldNameAndBindingName[1]; + if (!reading.containsKey(bindingName)) { + throw new RuntimeException("binding name '" + bindingName + + "' referenced in verify-fields, but it is not present in available bindings."); + } + expected.addFieldBinding(fieldName, reading.get(bindingName)); + } + return expected; + } + +} diff --git a/driver-cql-shaded/src/main/resources/cql.md b/driver-cql-shaded/src/main/resources/cql.md index 8190c365b..3f2c0e461 100644 --- a/driver-cql-shaded/src/main/resources/cql.md +++ b/driver-cql-shaded/src/main/resources/cql.md @@ -277,6 +277,21 @@ now **they are limited to a YAML params block**: # that statement for both successes and errors, # using the given statement name. + verify: * + compare: all + # Adds two operators to the operation: + # 1) verify that there is a single row result set in the response. + # 2) verify some or all of the field values by name and/or value. + # If this option is used on any statement, then the activity will + # provide verification metrics and exceptions, including details + # of verification in the log once the activity is completed. + # For full details on this field, see the docs on cqlverify. + + /// Cross-verify all fields and field values between the reference data and + /// the actual data. + all(0x1|0x1<<1|0x1<<2); + + logresultcsv: true OR logresultcsv: myfilename.csv diff --git a/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/CqlVerifyActivity.java b/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/CqlVerifyActivity.java index 487008c89..706b3bb9e 100644 --- a/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/CqlVerifyActivity.java +++ b/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/CqlVerifyActivity.java @@ -3,21 +3,17 @@ package io.nosqlbench.activitytype.cqlverify; import io.nosqlbench.activitytype.cql.core.CqlActivity; import io.nosqlbench.activitytype.cql.statements.rsoperators.AssertSingleRowResultSet; import io.nosqlbench.engine.api.activityimpl.ActivityDef; -import io.nosqlbench.virtdata.core.bindings.Bindings; -import io.nosqlbench.virtdata.core.bindings.BindingsTemplate; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; - +/** + * This activity is just a thin wrapper at this point. + * Most of the functionality it used to have has been + * generalized into the cql activity proper at this point. + */ public class CqlVerifyActivity extends CqlActivity { private final static Logger logger = LogManager.getLogger(CqlVerifyActivity.class); - private BindingsTemplate expectedValuesTemplate; - private VerificationMetrics verificationMetrics; public CqlVerifyActivity(ActivityDef activityDef) { super(activityDef); @@ -25,85 +21,22 @@ public class CqlVerifyActivity extends CqlActivity { @Override public synchronized void initActivity() { - this.verificationMetrics = new VerificationMetrics(getActivityDef()); + + if (!super.getActivityDef().getParams().contains("verify") && + !super.getActivityDef().getParams().contains("verify-fields")) { + logger.info("Pre-configuring activity param 'verify=*' since none was provided."); + logger.info("To control this on a per-statement basis, use the verify param."); + super.getActivityDef().getParams().put("verify", "*"); + } + + if (!super.getActivityDef().getParams().contains("compare")) { + super.getActivityDef().getParams().put("compare", "all"); + logger.info("Pre-configuring activity param 'compare=all' since none was provided."); + logger.info("To control this on a per-statement basis, use the compare param."); + } super.initActivity(); - if (this.stmts.size() > 1) { - throw new RuntimeException("More than one statement was configured as active. " - + this.getActivityDef().getActivityType() + " requires exactly one."); - } - - Optional randomMapper = stmts.stream() - .flatMap(s -> s.getBindings().values().stream()) - .filter(t -> t.matches(".*Random.*") || t.matches(".*random.*")) - .findAny(); - - - if (randomMapper.isPresent()) { - throw new RuntimeException( - "You should not try to verify data generated with random mapping " + - "functions, like " + randomMapper.get() + " as it does not " + - "produce stable results in different invocation order."); - } - - } - - public synchronized BindingsTemplate getExpectedValuesTemplate() { - if (expectedValuesTemplate==null) { - expectedValuesTemplate = new BindingsTemplate(); - Map bindings = stmts.get(0).getBindings(); - if (stmts.get(0).getParams().containsKey("verify-fields")) { - List fields = new ArrayList<>(); - String fieldSpec= stmts.get(0).getParamOrDefault("verify-fields",""); - String[] vfields = fieldSpec.split("\\s*,\\s*"); - for (String vfield : vfields) { - if (vfield.equals("*")) { - bindings.forEach((k,v)->fields.add(k)); - } else if (vfield.startsWith("+")) { - fields.add(vfield.substring(1)); - } else if (vfield.startsWith("-")) { - fields.remove(vfield.substring(1)); - } else if (vfield.matches("\\w+(\\w+->[\\w-]+)?")) { - fields.add(vfield); - } else { - throw new RuntimeException("unknown verify-fields format: '" + vfield + "'"); - } - } - for (String vfield : fields) { - String[] fieldNameAndBindingName = vfield.split("\\s*->\\s*", 2); - String fieldName = fieldNameAndBindingName[0]; - String bindingName = fieldNameAndBindingName.length==1 ? fieldName : fieldNameAndBindingName[1]; - if (!bindings.containsKey(bindingName)) { - throw new RuntimeException("binding name '" + bindingName + - "' referenced in verify-fields, but it is not present in available bindings."); - } - expectedValuesTemplate.addFieldBinding(fieldName,bindings.get(bindingName)); - } - } else { - bindings.forEach((k,v)->expectedValuesTemplate.addFieldBinding(k,v)); - } - } - return expectedValuesTemplate; - } - - public synchronized VerificationMetrics getVerificationMetrics() { - return verificationMetrics; - } - - @Override - public void shutdownActivity() { - super.shutdownActivity(); - VerificationMetrics metrics = getVerificationMetrics(); - long unverifiedValues = metrics.unverifiedValuesCounter.getCount(); - long unverifiedRows = metrics.unverifiedRowsCounter.getCount(); - - if (unverifiedRows > 0 || unverifiedValues > 0) { - throw new RuntimeException( - "There were " + unverifiedValues + " unverified values across " + unverifiedRows + " unverified rows." - ); - } - logger.info("verified " + metrics.verifiedValuesCounter.getCount() + " values across " + metrics.verifiedRowsCounter.getCount() + " verified rows"); } @Override @@ -111,14 +44,5 @@ public class CqlVerifyActivity extends CqlActivity { super.onActivityDefUpdate(activityDef); addResultSetCycleOperator(new AssertSingleRowResultSet()); - String verify = activityDef.getParams() - .getOptionalString("compare").orElse("all"); - DiffType diffType = DiffType.valueOf(verify); - Bindings verifyBindings = getExpectedValuesTemplate().resolveBindings(); - var differ = new RowDifferencer.ThreadLocalWrapper( - getVerificationMetrics(), - verifyBindings, - diffType); - addRowCycleOperator(differ); } } diff --git a/driver-cqlverify/src/main/resources/cqlverify.md b/driver-cqlverify/src/main/resources/cqlverify.md index 8100b0be9..e85e65bbb 100644 --- a/driver-cqlverify/src/main/resources/cqlverify.md +++ b/driver-cqlverify/src/main/resources/cqlverify.md @@ -1,8 +1,23 @@ # cqlverify -This activity type allows you to read values from a database and compare them to -the generated values that were expected to be written, row-by-row, producing a -comparative result between the two. +This driver allows you to read values from a database and compare them to +the generated values that were expected to be written, row-by-row, +producing a comparative result between the two. + +In practice, this is a wrapper activity type which simply sets the +defaults on the cql driver. The defaults will assume that all statements +are reads and that all fields referenced in their bindings should be +verified. If this is not the desired behavior, then you can control the +behavior on a per-statement basis by setting the verify and compare +statement parameters. If these are not set on a given statement, then the +activity level defaults are taken as `verify=*` +and `compare=all`. + +Going forward, it is suggested that you use the cql driver directly and +use the statement parameters. The cqlverify activity is not officially +deprecated, but it offers no specific functionality over the cql activity +apart from implied defaults, so it will likely be removed in lieu of the +direct cql verify parameters in the future. The verification options include: @@ -12,87 +27,97 @@ The verification options include: according to the Java equals implementation for the object type specified in that field's metadata. -The data bindings are used to generate the expected values that would be used -for an upsert. Each row is verified according to these values, and any -discrepancy is treated as an error that can be counted, logged, etc. +The data bindings are used to generate the expected values that would be +used for an upsert. Each row is verified according to these values, and +any discrepancy is treated as an error that can be counted, logged, etc. ### Using cqlverify -The cqlverify activity type is built on top of the cql activity type. As such, -it has all of the same capabilities and options, and then some. See the cql -activity type documentation for the usual details. This doc page only covers how -the cqlverify activity extends it. +The cqlverify activity type is built on top of the cql activity type. As +such, it has all of the same capabilities and options, and then some. See +the cql activity type documentation for the usual details. This doc page +only covers how the cqlverify activity extends it. -The differences between the cql and cqlverify activity types are mostly in how -how you configure for verifiable data and error handling. +The differences between the cql and cqlverify activity types are mostly in +how how you configure for verifiable data and error handling. ##### Writing verifiable data -The cqlverify driver does not retain logged data for verification. Still, it is able to compare data as if it had a -separate data set to compare to. This is possible only because the data generation facilities used by NoSQLBench provide -realistic and voluminous synthetic data that can be recalled from a recipe and accessed dynamically. +The cqlverify driver does not retain logged data for verification. Still, +it is able to compare data as if it had a separate data set to compare to. +This is possible only because the data generation facilities used by +NoSQLBench provide realistic and voluminous synthetic data that can be +recalled from a recipe and accessed dynamically. -That means, however, that you must avoid using the non-stable data mapping functions when writing data. The rule of -thumb is to avoid using any data mapping functions containing the word "Random", as these are the ones that have -historically used internal RNG state. Instead, swap in their replacements that start with "Hashed". There is a hashed -equivalent to all of the original random functions. The rng-based functions will be deprecated in a future release. +That means, however, that you must avoid using the non-stable data mapping +functions when writing data. The rule of thumb is to avoid using any data +mapping functions containing the word "Random", as these are the ones that +have historically used internal RNG state. Instead, swap in their +replacements that start with "Hashed". There is a hashed equivalent to all +of the original random functions. The rng-based functions will be +deprecated in a future release. -In a typical cql activity, you are allowed to name the bindings however you like, so long as the binding names match the -anchor names in your statement template. Because we need to match reference field data to actual row data pair-wise by -field name, there is a more strict requirement for cqlverify activities. The binding names themselves are now required -to match the field names that they are expected to be compared to. +In a typical cql activity, you are allowed to name the bindings however +you like, so long as the binding names match the anchor names in your +statement template. Because we need to match reference field data to +actual row data pair-wise by field name, there is a more strict +requirement for cqlverify activities. The binding names themselves are now +required to match the field names that they are expected to be compared +to. The simplest way to do this is to follow this recipe: -1. Make the binding names the same as the field names that you use in - in your write statements. +1. Make the binding names the same as the field names that you use in in + your write statements. 2. When you configure your read statement for the cqlverify activity, simply include the same bindings as-is, using the partition and clustering fields in the appropriate where clauses. -*note*: It used to be an error to have bindings names in excess of what anchor -names would match. Now, it is only an error if an anchor is not qualified with -a matching binding name. This allows you to simply copy your bindings as-is -directly from your write statement with no issues. +*note*: It used to be an error to have bindings names in excess of what +anchor names would match. Now, it is only an error if an anchor is not +qualified with a matching binding name. This allows you to simply copy +your bindings as-is directly from your write statement with no issues. ### Configuring the verification Reader -A cqlverify activity is almost exactly like any other cql activity. However, you -configure a single read statement to access the row data you want to verify. The -bindings for the read statement should include the data mappings that you have -for the write statement. That's pretty much all you have to do. +A cqlverify activity is almost exactly like any other cql activity. +However, you configure a single read statement to access the row data you +want to verify. The bindings for the read statement should include the +data mappings that you have for the congruent write statement. That's +pretty much all you have to do. -The names of the bindings and the values they produce are considered, depending -on the *compare* setting explained below. This means that you need to make sure -that the bindings that are provided for the statement are exactly the same as -you expect the row structure, irrespective of field order. For some statements -which use the same value in more than one place, you must name these uniquely -as well. +The names of the bindings and the values they produce are considered, +depending on the *compare* setting explained below. This means that you +need to make sure that the bindings that are provided for the statement +are exactly the same as you expect the row structure, irrespective of +field order. For some statements which use the same value in more than one +place, you must name these uniquely as well. -If more than one statement is active for a cqlverify activity, then an error is -thrown. This may change in the future, but for now it is a requirement. +If more than one statement is active for a cqlverify activity, then an +error is thrown. This may change in the future, but for now it is a +requirement. ### Handling Verification Errors -The cqlverify activity extends on the error handling stack mechanism that is -used by the cql activity type, by introducing a new error category: +The cqlverify activity extends on the error handling stack mechanism that +is used by the cql activity type, by introducing a new error category: *unverified*. The default configuration for this error category is unverified=stop -However, the usual options, including "stop", "warn", "retry", "histogram", -"count", and "ignore" are also allowed. +However, the usual options, including "stop", "warn", "retry", "histogram" +, "count", and "ignore" are also allowed. -Care should be taken to set the other error handling categories to be strict -enough to avoid false negatives in testing. The verification on a row can only -be done if the row is actually read first. If you set the error handler stack to -only count real errors, for example, then you will be preempting the read -verifier. Therefore, there is a default setting for the cqlverify activity for -the catch-all error handler parameter *errors*. +Care should be taken to set the other error handling categories to be +strict enough to avoid false negatives in testing. The verification on a +row can only be done if the row is actually read first. If you set the +error handler stack to only count real errors, for example, then you will +be preempting the read verifier. Therefore, there is a default setting for +the cqlverify activity for the catch-all error handler parameter *errors*. -This means that the default error handling behavior will cause an exception to -be thrown and the client will exit by default. If you wish for something less -dramatic, then set it to +This means that the default error handling behavior will cause an +exception to be thrown and the client will exit by default. If you wish +for something less dramatic, then set it to errors=...,unverified->count @@ -102,9 +127,9 @@ or ##### rows to verify -Currently, every read operation in a cqlverify activity must have a single row -in the result set. If there is no row, then the row fails validation. The same -happens if there is more than one row. +Currently, every read operation in a cqlverify activity must have a single +row in the result set. If there is no row, then the row fails validation. +The same happens if there is more than one row. A future release may allow for paged reads for quicker verification. @@ -118,7 +143,8 @@ Verify the the same 100K cycles of telemetry data ... run driver=cqlverify alias=verify workload=cql-iot tags=group:verify cycles=100000 host=... -To see how these examples work, consult the telemetry.yaml file in the nosqlbench.jar. +To see how these examples work, consult the telemetry.yaml file in the +nosqlbench.jar. ### CQLVerify ActivityType Parameters @@ -126,34 +152,37 @@ To see how these examples work, consult the telemetry.yaml file in the nosqlbenc - **compare** - what to verify. Valid values are "reffields", "rowfields", "fields", "values", or "all" - (default: all) - - rowfields - Verify that fields in the row, by name, are - not in excess of what is provided in the reference data. - - reffields - Verify that fields in the row, by name, are - present for all all of those provided in the reference data. - - fields - A synonym for rowfields AND reffields - (full set equivalence) - - values - Verify that all the pair-wise fields have equal - values, according to the type-specific equals method for - the data type identified in the row metadata by field name. - - all - A synonym for fields AND values + (default: all) + - rowfields - Verify that fields in the row, by name, are not in + excess of what is provided in the reference data. + - reffields - Verify that fields in the row, by name, are present for + all all of those provided in the reference data. + - fields - A synonym for rowfields AND reffields + (full set equivalence) + - values - Verify that all the pair-wise fields have equal values, + according to the type-specific equals method for the data type + identified in the row metadata by field name. + - all - A synonym for fields AND values ### CQLVerify Statement Parameters -- **verify-fields** - an optional modifier of fields to verify for a statement. - If this parameter is not provided, then it is presumed to be '*' by default. - This is a string which consists of comma-separate values. If the value - is '*', then all the bindings that are visible for the statement will be - used as expected values. - If it is a word that starts with '-', like '-field2', then the name after the - dash is removed from the list of fields to verify. - If it is a word that starts with a '+', like '+field3', or a simple word, - then the field is added to the list of fields to verify. - This parameter is useful if you have a set of default bindings and want - to specify which subset of them of them will be used just for this statement. +- **verify** - an optional modifier of fields to verify for a statement. + If this parameter is not provided, then it is presumed to be '*' by + default. This is a string which consists of comma-separate values. If + the value is '*', then all the bindings that are visible for the + statement will be used as expected values. If it is a word that starts + with '-', like '-field2', then the name after the dash is removed from + the list of fields to verify. If it is a word that starts with a ' + +', like '+field3', or a simple word, then the field is added to the + list of fields to verify. This parameter is useful if you have a set of + default bindings and want to specify which subset of them will be used + just for this statement. - If any of the added fields is in the form "f->b", then it is taken as a mapping - from the field name _f_ in the schema to a binding _b_. + If any of the added fields is in the form "f->b", then it is taken as a + mapping from the field name _f_ in the schema to a binding _b_. + + For backwards compatibility `verify-fields` is also recognized for this + option. ### Metrics @@ -162,5 +191,6 @@ The cqlverify activity type adds some verification-specific metrics: - alias.verifiedrows - A counter for how many rows passed verification - alias.unverifiedrows - A counter for how many rows failed verification - alias.verifiedvalues - A counter for how many field values were verified -- alias.unverifiedvalues - A counter for how many field values were unverified +- alias.unverifiedvalues - A counter for how many field values were + unverified diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ScenarioResult.java b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ScenarioResult.java index 0f0530922..1dc37f0ca 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ScenarioResult.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ScenarioResult.java @@ -18,10 +18,10 @@ package io.nosqlbench.engine.core.lifecycle; -import com.codahale.metrics.ConsoleReporter; -import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.*; import io.nosqlbench.engine.api.metrics.ActivityMetrics; import io.nosqlbench.engine.core.logging.Log4JMetricsReporter; +import io.nosqlbench.engine.core.metrics.NBMetricsSummary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -117,4 +117,22 @@ public class ScenarioResult { reporter.report(); logger.debug("-- END METRICS DETAIL --"); } + + public void reportCountsTo(PrintStream printStream) { + StringBuilder sb = new StringBuilder(); + + ActivityMetrics.getMetricRegistry().getMetrics().forEach((k, v) -> { + if (v instanceof Counting) { + long count = ((Counting) v).getCount(); + if (count > 0) { + NBMetricsSummary.summarize(sb, k, v); + } + } + }); + + printStream.println("-- BEGIN NON-ZERO metric counts (run longer for full report):"); + printStream.print(sb.toString()); + printStream.println("-- END NON-ZERO metric counts:"); + + } }