mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
nosqlbench-266 cql verify should allow multiple verification statements
This commit is contained in:
Jonathan Shook
parent
9a974bac00
commit
eaf30ad9aa
@ -16,6 +16,11 @@ import io.nosqlbench.activitytype.cql.statements.core.*;
|
||||
import io.nosqlbench.activitytype.cql.statements.modifiers.StatementModifier;
|
||||
import io.nosqlbench.activitytype.cql.statements.rowoperators.RowCycleOperators;
|
||||
import io.nosqlbench.activitytype.cql.statements.rowoperators.Save;
|
||||
import io.nosqlbench.activitytype.cql.statements.rowoperators.verification.DiffType;
|
||||
import io.nosqlbench.activitytype.cql.statements.rowoperators.verification.RowDifferencer;
|
||||
import io.nosqlbench.activitytype.cql.statements.rowoperators.verification.VerificationMetrics;
|
||||
import io.nosqlbench.activitytype.cql.statements.rowoperators.verification.VerifierBuilder;
|
||||
import io.nosqlbench.activitytype.cql.statements.rsoperators.AssertSingleRowResultSet;
|
||||
import io.nosqlbench.activitytype.cql.statements.rsoperators.ResultSetCycleOperators;
|
||||
import io.nosqlbench.activitytype.cql.statements.rsoperators.TraceLogger;
|
||||
import io.nosqlbench.engine.api.activityapi.core.Activity;
|
||||
@ -44,6 +49,7 @@ import io.nosqlbench.engine.api.util.TagFilter;
|
||||
import io.nosqlbench.engine.api.util.Unit;
|
||||
import io.nosqlbench.nb.api.config.params.Element;
|
||||
import io.nosqlbench.nb.api.errors.BasicError;
|
||||
import io.nosqlbench.virtdata.core.bindings.Bindings;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
@ -88,6 +94,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
|
||||
private long maxRetryDelay;
|
||||
private boolean retryReplace;
|
||||
private String pooling;
|
||||
private VerificationMetrics verificationMetrics;
|
||||
|
||||
|
||||
public CqlActivity(ActivityDef activityDef) {
|
||||
@ -288,6 +295,35 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
|
||||
template.addRowCycleOperators(ro);
|
||||
});
|
||||
|
||||
|
||||
// If verify is set on activity, assume all fields should be verified for every
|
||||
// statement, otherwise, do per-statement verification for ops which have
|
||||
// a verify param
|
||||
|
||||
if (activityDef.getParams().containsKey("verify") ||
|
||||
stmtDef.getParams().containsKey("verify") ||
|
||||
stmtDef.getParams().containsKey("verify-fields")) {
|
||||
|
||||
String verify = stmtDef.getOptionalStringParam("verify")
|
||||
.or(() -> stmtDef.getOptionalStringParam("verify-fields"))
|
||||
.or(() -> activityDef.getParams().getOptionalString("verify"))
|
||||
.orElse("*");
|
||||
|
||||
DiffType diffType = stmtDef.getOptionalStringParam("compare")
|
||||
.or(() -> activityDef.getParams().getOptionalString("compare"))
|
||||
.map(DiffType::valueOf).orElse(DiffType.reffields);
|
||||
|
||||
Bindings expected = VerifierBuilder.getExpectedValuesTemplate(stmtDef).resolveBindings();
|
||||
VerificationMetrics vmetrics = getVerificationMetrics();
|
||||
|
||||
RowDifferencer.ThreadLocalWrapper differencer = new RowDifferencer.ThreadLocalWrapper(vmetrics, expected, diffType);
|
||||
psummary.append(" rowop=>verify-fields:").append(differencer.toString());
|
||||
|
||||
template.addResultSetOperators(new AssertSingleRowResultSet());
|
||||
template.addRowCycleOperators(differencer);
|
||||
}
|
||||
|
||||
|
||||
if (instrument) {
|
||||
logger.info("Adding per-statement success and error and resultset-size timers to statement '" + parsed.getName() + "'");
|
||||
template.instrument(this);
|
||||
@ -318,6 +354,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
|
||||
|
||||
}
|
||||
|
||||
private synchronized VerificationMetrics getVerificationMetrics() {
|
||||
if (verificationMetrics == null) {
|
||||
verificationMetrics = new VerificationMetrics(this.activityDef);
|
||||
}
|
||||
return verificationMetrics;
|
||||
}
|
||||
|
||||
private StmtsDocList loadStmtsYaml() {
|
||||
StmtsDocList doclist = null;
|
||||
|
||||
@ -429,6 +472,26 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
|
||||
return exceptionCountMetrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdownActivity() {
|
||||
super.shutdownActivity();
|
||||
|
||||
if (verificationMetrics != null) {
|
||||
|
||||
VerificationMetrics metrics = getVerificationMetrics();
|
||||
long unverifiedValues = metrics.unverifiedValuesCounter.getCount();
|
||||
long unverifiedRows = metrics.unverifiedRowsCounter.getCount();
|
||||
|
||||
if (unverifiedRows > 0 || unverifiedValues > 0) {
|
||||
throw new RuntimeException(
|
||||
"There were " + unverifiedValues + " unverified values across " + unverifiedRows + " unverified rows."
|
||||
);
|
||||
}
|
||||
logger.info("verified " + metrics.verifiedValuesCounter.getCount() + " values across " + metrics.verifiedRowsCounter.getCount() + " verified rows");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "CQLActivity {" +
|
||||
|
||||
@ -15,7 +15,6 @@ public enum RowCycleOperators {
|
||||
this.implClass = traceLoggerClass;
|
||||
}
|
||||
|
||||
|
||||
public Class<? extends RowCycleOperator> getImplementation() {
|
||||
return implClass;
|
||||
}
|
||||
|
||||
@ -1,7 +1,10 @@
|
||||
package io.nosqlbench.activitytype.cqlverify;
|
||||
package io.nosqlbench.activitytype.cql.statements.rowoperators.verification;
|
||||
|
||||
public enum DiffType {
|
||||
|
||||
/// Verify nothing for this statement
|
||||
none(0),
|
||||
|
||||
/// Verify that fields named in the row are present in the reference map.
|
||||
rowfields(0x1),
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package io.nosqlbench.activitytype.cqlverify;
|
||||
package io.nosqlbench.activitytype.cql.statements.rowoperators.verification;
|
||||
|
||||
import com.datastax.driver.core.*;
|
||||
import io.nosqlbench.activitytype.cql.api.RowCycleOperator;
|
||||
@ -295,7 +295,7 @@ public class RowDifferencer implements RowCycleOperator {
|
||||
private final VerificationMetrics metrics;
|
||||
private final Bindings bindings;
|
||||
private final DiffType diffType;
|
||||
private ThreadLocal<RowDifferencer> tl;
|
||||
private final ThreadLocal<RowDifferencer> tl;
|
||||
|
||||
public ThreadLocalWrapper(VerificationMetrics metrics, Bindings bindings, DiffType diffType) {
|
||||
this.metrics = metrics;
|
||||
@ -1,4 +1,4 @@
|
||||
package io.nosqlbench.activitytype.cqlverify;
|
||||
package io.nosqlbench.activitytype.cql.statements.rowoperators.verification;
|
||||
|
||||
import com.codahale.metrics.Counter;
|
||||
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
|
||||
@ -0,0 +1,53 @@
|
||||
package io.nosqlbench.activitytype.cql.statements.rowoperators.verification;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class VerifierBuilder {
|
||||
public static BindingsTemplate getExpectedValuesTemplate(OpTemplate stmtDef) {
|
||||
|
||||
BindingsTemplate expected = new BindingsTemplate();
|
||||
|
||||
if (!stmtDef.getParams().containsKey("verify-fields") && !stmtDef.getParams().containsKey("verify")) {
|
||||
throw new RuntimeException("Unable to create expected values template with no 'verify' param");
|
||||
}
|
||||
|
||||
Map<String, String> reading = stmtDef.getBindings();
|
||||
|
||||
List<String> fields = new ArrayList<>();
|
||||
String fieldSpec = stmtDef.getOptionalStringParam("verify-fields")
|
||||
.or(() -> stmtDef.getOptionalStringParam("verify"))
|
||||
.orElse("*");
|
||||
|
||||
String[] vfields = fieldSpec.split("\\s*,\\s*");
|
||||
for (String vfield : vfields) {
|
||||
if (vfield.equals("*")) {
|
||||
reading.forEach((k, v) -> fields.add(k));
|
||||
} else if (vfield.startsWith("+")) {
|
||||
fields.add(vfield.substring(1));
|
||||
} else if (vfield.startsWith("-")) {
|
||||
fields.remove(vfield.substring(1));
|
||||
} else if (vfield.matches("\\w+(\\w+->[\\w-]+)?")) {
|
||||
fields.add(vfield);
|
||||
} else {
|
||||
throw new RuntimeException("unknown verify-fields format: '" + vfield + "'");
|
||||
}
|
||||
}
|
||||
for (String vfield : fields) {
|
||||
String[] fieldNameAndBindingName = vfield.split("\\s*->\\s*", 2);
|
||||
String fieldName = fieldNameAndBindingName[0];
|
||||
String bindingName = fieldNameAndBindingName.length == 1 ? fieldName : fieldNameAndBindingName[1];
|
||||
if (!reading.containsKey(bindingName)) {
|
||||
throw new RuntimeException("binding name '" + bindingName +
|
||||
"' referenced in verify-fields, but it is not present in available bindings.");
|
||||
}
|
||||
expected.addFieldBinding(fieldName, reading.get(bindingName));
|
||||
}
|
||||
return expected;
|
||||
}
|
||||
|
||||
}
|
||||
@ -277,6 +277,21 @@ now **they are limited to a YAML params block**:
|
||||
# that statement for both successes and errors,
|
||||
# using the given statement name.
|
||||
|
||||
verify: *
|
||||
compare: all
|
||||
# Adds two operators to the operation:
|
||||
# 1) verify that there is a single row result set in the response.
|
||||
# 2) verify some or all of the field values by name and/or value.
|
||||
# If this option is used on any statement, then the activity will
|
||||
# provide verification metrics and exceptions, including details
|
||||
# of verification in the log once the activity is completed.
|
||||
# For full details on this field, see the docs on cqlverify.
|
||||
|
||||
/// Cross-verify all fields and field values between the reference data and
|
||||
/// the actual data.
|
||||
all(0x1|0x1<<1|0x1<<2);
|
||||
|
||||
|
||||
logresultcsv: true
|
||||
OR
|
||||
logresultcsv: myfilename.csv
|
||||
|
||||
@ -3,21 +3,17 @@ package io.nosqlbench.activitytype.cqlverify;
|
||||
import io.nosqlbench.activitytype.cql.core.CqlActivity;
|
||||
import io.nosqlbench.activitytype.cql.statements.rsoperators.AssertSingleRowResultSet;
|
||||
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
|
||||
import io.nosqlbench.virtdata.core.bindings.Bindings;
|
||||
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* This activity is just a thin wrapper at this point.
|
||||
* Most of the functionality it used to have has been
|
||||
* generalized into the cql activity proper at this point.
|
||||
*/
|
||||
public class CqlVerifyActivity extends CqlActivity {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(CqlVerifyActivity.class);
|
||||
private BindingsTemplate expectedValuesTemplate;
|
||||
private VerificationMetrics verificationMetrics;
|
||||
|
||||
public CqlVerifyActivity(ActivityDef activityDef) {
|
||||
super(activityDef);
|
||||
@ -25,85 +21,22 @@ public class CqlVerifyActivity extends CqlActivity {
|
||||
|
||||
@Override
|
||||
public synchronized void initActivity() {
|
||||
this.verificationMetrics = new VerificationMetrics(getActivityDef());
|
||||
|
||||
if (!super.getActivityDef().getParams().contains("verify") &&
|
||||
!super.getActivityDef().getParams().contains("verify-fields")) {
|
||||
logger.info("Pre-configuring activity param 'verify=*' since none was provided.");
|
||||
logger.info("To control this on a per-statement basis, use the verify param.");
|
||||
super.getActivityDef().getParams().put("verify", "*");
|
||||
}
|
||||
|
||||
if (!super.getActivityDef().getParams().contains("compare")) {
|
||||
super.getActivityDef().getParams().put("compare", "all");
|
||||
logger.info("Pre-configuring activity param 'compare=all' since none was provided.");
|
||||
logger.info("To control this on a per-statement basis, use the compare param.");
|
||||
}
|
||||
|
||||
super.initActivity();
|
||||
|
||||
if (this.stmts.size() > 1) {
|
||||
throw new RuntimeException("More than one statement was configured as active. "
|
||||
+ this.getActivityDef().getActivityType() + " requires exactly one.");
|
||||
}
|
||||
|
||||
Optional<String> randomMapper = stmts.stream()
|
||||
.flatMap(s -> s.getBindings().values().stream())
|
||||
.filter(t -> t.matches(".*Random.*") || t.matches(".*random.*"))
|
||||
.findAny();
|
||||
|
||||
|
||||
if (randomMapper.isPresent()) {
|
||||
throw new RuntimeException(
|
||||
"You should not try to verify data generated with random mapping " +
|
||||
"functions, like " + randomMapper.get() + " as it does not " +
|
||||
"produce stable results in different invocation order.");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public synchronized BindingsTemplate getExpectedValuesTemplate() {
|
||||
if (expectedValuesTemplate==null) {
|
||||
expectedValuesTemplate = new BindingsTemplate();
|
||||
Map<String, String> bindings = stmts.get(0).getBindings();
|
||||
if (stmts.get(0).getParams().containsKey("verify-fields")) {
|
||||
List<String> fields = new ArrayList<>();
|
||||
String fieldSpec= stmts.get(0).getParamOrDefault("verify-fields","");
|
||||
String[] vfields = fieldSpec.split("\\s*,\\s*");
|
||||
for (String vfield : vfields) {
|
||||
if (vfield.equals("*")) {
|
||||
bindings.forEach((k,v)->fields.add(k));
|
||||
} else if (vfield.startsWith("+")) {
|
||||
fields.add(vfield.substring(1));
|
||||
} else if (vfield.startsWith("-")) {
|
||||
fields.remove(vfield.substring(1));
|
||||
} else if (vfield.matches("\\w+(\\w+->[\\w-]+)?")) {
|
||||
fields.add(vfield);
|
||||
} else {
|
||||
throw new RuntimeException("unknown verify-fields format: '" + vfield + "'");
|
||||
}
|
||||
}
|
||||
for (String vfield : fields) {
|
||||
String[] fieldNameAndBindingName = vfield.split("\\s*->\\s*", 2);
|
||||
String fieldName = fieldNameAndBindingName[0];
|
||||
String bindingName = fieldNameAndBindingName.length==1 ? fieldName : fieldNameAndBindingName[1];
|
||||
if (!bindings.containsKey(bindingName)) {
|
||||
throw new RuntimeException("binding name '" + bindingName +
|
||||
"' referenced in verify-fields, but it is not present in available bindings.");
|
||||
}
|
||||
expectedValuesTemplate.addFieldBinding(fieldName,bindings.get(bindingName));
|
||||
}
|
||||
} else {
|
||||
bindings.forEach((k,v)->expectedValuesTemplate.addFieldBinding(k,v));
|
||||
}
|
||||
}
|
||||
return expectedValuesTemplate;
|
||||
}
|
||||
|
||||
public synchronized VerificationMetrics getVerificationMetrics() {
|
||||
return verificationMetrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdownActivity() {
|
||||
super.shutdownActivity();
|
||||
VerificationMetrics metrics = getVerificationMetrics();
|
||||
long unverifiedValues = metrics.unverifiedValuesCounter.getCount();
|
||||
long unverifiedRows = metrics.unverifiedRowsCounter.getCount();
|
||||
|
||||
if (unverifiedRows > 0 || unverifiedValues > 0) {
|
||||
throw new RuntimeException(
|
||||
"There were " + unverifiedValues + " unverified values across " + unverifiedRows + " unverified rows."
|
||||
);
|
||||
}
|
||||
logger.info("verified " + metrics.verifiedValuesCounter.getCount() + " values across " + metrics.verifiedRowsCounter.getCount() + " verified rows");
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -111,14 +44,5 @@ public class CqlVerifyActivity extends CqlActivity {
|
||||
super.onActivityDefUpdate(activityDef);
|
||||
addResultSetCycleOperator(new AssertSingleRowResultSet());
|
||||
|
||||
String verify = activityDef.getParams()
|
||||
.getOptionalString("compare").orElse("all");
|
||||
DiffType diffType = DiffType.valueOf(verify);
|
||||
Bindings verifyBindings = getExpectedValuesTemplate().resolveBindings();
|
||||
var differ = new RowDifferencer.ThreadLocalWrapper(
|
||||
getVerificationMetrics(),
|
||||
verifyBindings,
|
||||
diffType);
|
||||
addRowCycleOperator(differ);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,8 +1,23 @@
|
||||
# cqlverify
|
||||
|
||||
This activity type allows you to read values from a database and compare them to
|
||||
the generated values that were expected to be written, row-by-row, producing a
|
||||
comparative result between the two.
|
||||
This driver allows you to read values from a database and compare them to
|
||||
the generated values that were expected to be written, row-by-row,
|
||||
producing a comparative result between the two.
|
||||
|
||||
In practice, this is a wrapper activity type which simply sets the
|
||||
defaults on the cql driver. The defaults will assume that all statements
|
||||
are reads and that all fields referenced in their bindings should be
|
||||
verified. If this is not the desired behavior, then you can control the
|
||||
behavior on a per-statement basis by setting the verify and compare
|
||||
statement parameters. If these are not set on a given statement, then the
|
||||
activity level defaults are taken as `verify=*`
|
||||
and `compare=all`.
|
||||
|
||||
Going forward, it is suggested that you use the cql driver directly and
|
||||
use the statement parameters. The cqlverify activity is not officially
|
||||
deprecated, but it offers no specific functionality over the cql activity
|
||||
apart from implied defaults, so it will likely be removed in lieu of the
|
||||
direct cql verify parameters in the future.
|
||||
|
||||
The verification options include:
|
||||
|
||||
@ -12,87 +27,97 @@ The verification options include:
|
||||
according to the Java equals implementation for the object type
|
||||
specified in that field's metadata.
|
||||
|
||||
The data bindings are used to generate the expected values that would be used
|
||||
for an upsert. Each row is verified according to these values, and any
|
||||
discrepancy is treated as an error that can be counted, logged, etc.
|
||||
The data bindings are used to generate the expected values that would be
|
||||
used for an upsert. Each row is verified according to these values, and
|
||||
any discrepancy is treated as an error that can be counted, logged, etc.
|
||||
|
||||
### Using cqlverify
|
||||
|
||||
The cqlverify activity type is built on top of the cql activity type. As such,
|
||||
it has all of the same capabilities and options, and then some. See the cql
|
||||
activity type documentation for the usual details. This doc page only covers how
|
||||
the cqlverify activity extends it.
|
||||
The cqlverify activity type is built on top of the cql activity type. As
|
||||
such, it has all of the same capabilities and options, and then some. See
|
||||
the cql activity type documentation for the usual details. This doc page
|
||||
only covers how the cqlverify activity extends it.
|
||||
|
||||
The differences between the cql and cqlverify activity types are mostly in how
|
||||
how you configure for verifiable data and error handling.
|
||||
The differences between the cql and cqlverify activity types are mostly in
|
||||
how how you configure for verifiable data and error handling.
|
||||
|
||||
##### Writing verifiable data
|
||||
|
||||
The cqlverify driver does not retain logged data for verification. Still, it is able to compare data as if it had a
|
||||
separate data set to compare to. This is possible only because the data generation facilities used by NoSQLBench provide
|
||||
realistic and voluminous synthetic data that can be recalled from a recipe and accessed dynamically.
|
||||
The cqlverify driver does not retain logged data for verification. Still,
|
||||
it is able to compare data as if it had a separate data set to compare to.
|
||||
This is possible only because the data generation facilities used by
|
||||
NoSQLBench provide realistic and voluminous synthetic data that can be
|
||||
recalled from a recipe and accessed dynamically.
|
||||
|
||||
That means, however, that you must avoid using the non-stable data mapping functions when writing data. The rule of
|
||||
thumb is to avoid using any data mapping functions containing the word "Random", as these are the ones that have
|
||||
historically used internal RNG state. Instead, swap in their replacements that start with "Hashed". There is a hashed
|
||||
equivalent to all of the original random functions. The rng-based functions will be deprecated in a future release.
|
||||
That means, however, that you must avoid using the non-stable data mapping
|
||||
functions when writing data. The rule of thumb is to avoid using any data
|
||||
mapping functions containing the word "Random", as these are the ones that
|
||||
have historically used internal RNG state. Instead, swap in their
|
||||
replacements that start with "Hashed". There is a hashed equivalent to all
|
||||
of the original random functions. The rng-based functions will be
|
||||
deprecated in a future release.
|
||||
|
||||
In a typical cql activity, you are allowed to name the bindings however you like, so long as the binding names match the
|
||||
anchor names in your statement template. Because we need to match reference field data to actual row data pair-wise by
|
||||
field name, there is a more strict requirement for cqlverify activities. The binding names themselves are now required
|
||||
to match the field names that they are expected to be compared to.
|
||||
In a typical cql activity, you are allowed to name the bindings however
|
||||
you like, so long as the binding names match the anchor names in your
|
||||
statement template. Because we need to match reference field data to
|
||||
actual row data pair-wise by field name, there is a more strict
|
||||
requirement for cqlverify activities. The binding names themselves are now
|
||||
required to match the field names that they are expected to be compared
|
||||
to.
|
||||
|
||||
The simplest way to do this is to follow this recipe:
|
||||
|
||||
1. Make the binding names the same as the field names that you use in
|
||||
in your write statements.
|
||||
1. Make the binding names the same as the field names that you use in in
|
||||
your write statements.
|
||||
2. When you configure your read statement for the cqlverify activity,
|
||||
simply include the same bindings as-is, using the partition and
|
||||
clustering fields in the appropriate where clauses.
|
||||
|
||||
*note*: It used to be an error to have bindings names in excess of what anchor
|
||||
names would match. Now, it is only an error if an anchor is not qualified with
|
||||
a matching binding name. This allows you to simply copy your bindings as-is
|
||||
directly from your write statement with no issues.
|
||||
*note*: It used to be an error to have bindings names in excess of what
|
||||
anchor names would match. Now, it is only an error if an anchor is not
|
||||
qualified with a matching binding name. This allows you to simply copy
|
||||
your bindings as-is directly from your write statement with no issues.
|
||||
|
||||
### Configuring the verification Reader
|
||||
|
||||
A cqlverify activity is almost exactly like any other cql activity. However, you
|
||||
configure a single read statement to access the row data you want to verify. The
|
||||
bindings for the read statement should include the data mappings that you have
|
||||
for the write statement. That's pretty much all you have to do.
|
||||
A cqlverify activity is almost exactly like any other cql activity.
|
||||
However, you configure a single read statement to access the row data you
|
||||
want to verify. The bindings for the read statement should include the
|
||||
data mappings that you have for the congruent write statement. That's
|
||||
pretty much all you have to do.
|
||||
|
||||
The names of the bindings and the values they produce are considered, depending
|
||||
on the *compare* setting explained below. This means that you need to make sure
|
||||
that the bindings that are provided for the statement are exactly the same as
|
||||
you expect the row structure, irrespective of field order. For some statements
|
||||
which use the same value in more than one place, you must name these uniquely
|
||||
as well.
|
||||
The names of the bindings and the values they produce are considered,
|
||||
depending on the *compare* setting explained below. This means that you
|
||||
need to make sure that the bindings that are provided for the statement
|
||||
are exactly the same as you expect the row structure, irrespective of
|
||||
field order. For some statements which use the same value in more than one
|
||||
place, you must name these uniquely as well.
|
||||
|
||||
If more than one statement is active for a cqlverify activity, then an error is
|
||||
thrown. This may change in the future, but for now it is a requirement.
|
||||
If more than one statement is active for a cqlverify activity, then an
|
||||
error is thrown. This may change in the future, but for now it is a
|
||||
requirement.
|
||||
|
||||
### Handling Verification Errors
|
||||
|
||||
The cqlverify activity extends on the error handling stack mechanism that is
|
||||
used by the cql activity type, by introducing a new error category:
|
||||
The cqlverify activity extends on the error handling stack mechanism that
|
||||
is used by the cql activity type, by introducing a new error category:
|
||||
*unverified*. The default configuration for this error category is
|
||||
|
||||
unverified=stop
|
||||
|
||||
However, the usual options, including "stop", "warn", "retry", "histogram",
|
||||
"count", and "ignore" are also allowed.
|
||||
However, the usual options, including "stop", "warn", "retry", "histogram"
|
||||
, "count", and "ignore" are also allowed.
|
||||
|
||||
Care should be taken to set the other error handling categories to be strict
|
||||
enough to avoid false negatives in testing. The verification on a row can only
|
||||
be done if the row is actually read first. If you set the error handler stack to
|
||||
only count real errors, for example, then you will be preempting the read
|
||||
verifier. Therefore, there is a default setting for the cqlverify activity for
|
||||
the catch-all error handler parameter *errors*.
|
||||
Care should be taken to set the other error handling categories to be
|
||||
strict enough to avoid false negatives in testing. The verification on a
|
||||
row can only be done if the row is actually read first. If you set the
|
||||
error handler stack to only count real errors, for example, then you will
|
||||
be preempting the read verifier. Therefore, there is a default setting for
|
||||
the cqlverify activity for the catch-all error handler parameter *errors*.
|
||||
|
||||
This means that the default error handling behavior will cause an exception to
|
||||
be thrown and the client will exit by default. If you wish for something less
|
||||
dramatic, then set it to
|
||||
This means that the default error handling behavior will cause an
|
||||
exception to be thrown and the client will exit by default. If you wish
|
||||
for something less dramatic, then set it to
|
||||
|
||||
errors=...,unverified->count
|
||||
|
||||
@ -102,9 +127,9 @@ or
|
||||
|
||||
##### rows to verify
|
||||
|
||||
Currently, every read operation in a cqlverify activity must have a single row
|
||||
in the result set. If there is no row, then the row fails validation. The same
|
||||
happens if there is more than one row.
|
||||
Currently, every read operation in a cqlverify activity must have a single
|
||||
row in the result set. If there is no row, then the row fails validation.
|
||||
The same happens if there is more than one row.
|
||||
|
||||
A future release may allow for paged reads for quicker verification.
|
||||
|
||||
@ -118,7 +143,8 @@ Verify the the same 100K cycles of telemetry data
|
||||
|
||||
... run driver=cqlverify alias=verify workload=cql-iot tags=group:verify cycles=100000 host=...
|
||||
|
||||
To see how these examples work, consult the telemetry.yaml file in the nosqlbench.jar.
|
||||
To see how these examples work, consult the telemetry.yaml file in the
|
||||
nosqlbench.jar.
|
||||
|
||||
### CQLVerify ActivityType Parameters
|
||||
|
||||
@ -127,33 +153,36 @@ To see how these examples work, consult the telemetry.yaml file in the nosqlbenc
|
||||
- **compare** - what to verify. Valid values are "reffields",
|
||||
"rowfields", "fields", "values", or "all"
|
||||
(default: all)
|
||||
- rowfields - Verify that fields in the row, by name, are
|
||||
not in excess of what is provided in the reference data.
|
||||
- reffields - Verify that fields in the row, by name, are
|
||||
present for all all of those provided in the reference data.
|
||||
- rowfields - Verify that fields in the row, by name, are not in
|
||||
excess of what is provided in the reference data.
|
||||
- reffields - Verify that fields in the row, by name, are present for
|
||||
all all of those provided in the reference data.
|
||||
- fields - A synonym for rowfields AND reffields
|
||||
(full set equivalence)
|
||||
- values - Verify that all the pair-wise fields have equal
|
||||
values, according to the type-specific equals method for
|
||||
the data type identified in the row metadata by field name.
|
||||
- values - Verify that all the pair-wise fields have equal values,
|
||||
according to the type-specific equals method for the data type
|
||||
identified in the row metadata by field name.
|
||||
- all - A synonym for fields AND values
|
||||
|
||||
### CQLVerify Statement Parameters
|
||||
|
||||
- **verify-fields** - an optional modifier of fields to verify for a statement.
|
||||
If this parameter is not provided, then it is presumed to be '*' by default.
|
||||
This is a string which consists of comma-separate values. If the value
|
||||
is '*', then all the bindings that are visible for the statement will be
|
||||
used as expected values.
|
||||
If it is a word that starts with '-', like '-field2', then the name after the
|
||||
dash is removed from the list of fields to verify.
|
||||
If it is a word that starts with a '+', like '+field3', or a simple word,
|
||||
then the field is added to the list of fields to verify.
|
||||
This parameter is useful if you have a set of default bindings and want
|
||||
to specify which subset of them of them will be used just for this statement.
|
||||
- **verify** - an optional modifier of fields to verify for a statement.
|
||||
If this parameter is not provided, then it is presumed to be '*' by
|
||||
default. This is a string which consists of comma-separate values. If
|
||||
the value is '*', then all the bindings that are visible for the
|
||||
statement will be used as expected values. If it is a word that starts
|
||||
with '-', like '-field2', then the name after the dash is removed from
|
||||
the list of fields to verify. If it is a word that starts with a '
|
||||
+', like '+field3', or a simple word, then the field is added to the
|
||||
list of fields to verify. This parameter is useful if you have a set of
|
||||
default bindings and want to specify which subset of them will be used
|
||||
just for this statement.
|
||||
|
||||
If any of the added fields is in the form "f->b", then it is taken as a mapping
|
||||
from the field name _f_ in the schema to a binding _b_.
|
||||
If any of the added fields is in the form "f->b", then it is taken as a
|
||||
mapping from the field name _f_ in the schema to a binding _b_.
|
||||
|
||||
For backwards compatibility `verify-fields` is also recognized for this
|
||||
option.
|
||||
|
||||
### Metrics
|
||||
|
||||
@ -162,5 +191,6 @@ The cqlverify activity type adds some verification-specific metrics:
|
||||
- alias.verifiedrows - A counter for how many rows passed verification
|
||||
- alias.unverifiedrows - A counter for how many rows failed verification
|
||||
- alias.verifiedvalues - A counter for how many field values were verified
|
||||
- alias.unverifiedvalues - A counter for how many field values were unverified
|
||||
- alias.unverifiedvalues - A counter for how many field values were
|
||||
unverified
|
||||
|
||||
|
||||
@ -18,10 +18,10 @@
|
||||
|
||||
package io.nosqlbench.engine.core.lifecycle;
|
||||
|
||||
import com.codahale.metrics.ConsoleReporter;
|
||||
import com.codahale.metrics.MetricFilter;
|
||||
import com.codahale.metrics.*;
|
||||
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
|
||||
import io.nosqlbench.engine.core.logging.Log4JMetricsReporter;
|
||||
import io.nosqlbench.engine.core.metrics.NBMetricsSummary;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
@ -117,4 +117,22 @@ public class ScenarioResult {
|
||||
reporter.report();
|
||||
logger.debug("-- END METRICS DETAIL --");
|
||||
}
|
||||
|
||||
public void reportCountsTo(PrintStream printStream) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
ActivityMetrics.getMetricRegistry().getMetrics().forEach((k, v) -> {
|
||||
if (v instanceof Counting) {
|
||||
long count = ((Counting) v).getCount();
|
||||
if (count > 0) {
|
||||
NBMetricsSummary.summarize(sb, k, v);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
printStream.println("-- BEGIN NON-ZERO metric counts (run longer for full report):");
|
||||
printStream.print(sb.toString());
|
||||
printStream.println("-- END NON-ZERO metric counts:");
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user