Merge pull request #1251 from nosqlbench/jk-test-eng-95-expected-result-verification

Verify expected result with MVEL

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4RawStmtDispenser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4SimpleCqlStmtDispenser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,7 +28,9 @@ import io.nosqlbench.adapter.cqld4.exceptions.UndefinedResultSetException;
 import io.nosqlbench.adapter.cqld4.exceptions.UnexpectedPagingException;
 import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.*;
 
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
 
@@ -42,7 +44,7 @@ import java.util.Map;
 // TODO: add rows histogram resultSetSizeHisto
 
 
-public abstract class Cqld4CqlOp implements CycleOp<ResultSet>, VariableCapture, OpGenerator, OpResultSize {
+public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture, OpGenerator, OpResultSize {
 
     private final CqlSession session;
     private final int maxPages;
@@ -54,6 +56,8 @@ public abstract class Cqld4CqlOp implements CycleOp<ResultSet>, VariableCapture,
     private Cqld4CqlOp nextOp;
     private final RSProcessors processors;
 
+    private final ThreadLocal<List<Row>> results = new ThreadLocal<>();
+
     public Cqld4CqlOp(CqlSession session, int maxPages, boolean retryReplace, int maxLwtRetries, RSProcessors processors) {
         this.session = session;
         this.maxPages = maxPages;
@@ -71,7 +75,7 @@ public abstract class Cqld4CqlOp implements CycleOp<ResultSet>, VariableCapture,
         this.processors = processors;
     }
 
-    public final ResultSet apply(long cycle) {
+    public final List<Row> apply(long cycle) {
 
         Statement<?> stmt = getStmt();
         rs = session.execute(stmt);
@@ -97,22 +101,29 @@ public abstract class Cqld4CqlOp implements CycleOp<ResultSet>, VariableCapture,
 
         Iterator<Row> reader = rs.iterator();
         int pages = 0;
+        // TODO/MVEL: An optimization to this would be to collect the results in a result set processor,
+        // but allow/require this processor to be added to an op _only_ in the event that it would
+        // be needed by a downstream consumer like the MVEL expected result evaluator
+
+        var resultRows = new ArrayList<Row>();
         while (true) {
             int pageRows = rs.getAvailableWithoutFetching();
             for (int i = 0; i < pageRows; i++) {
                 Row row = reader.next();
+                resultRows.add(row);
                 processors.buffer(row);
             }
             if (pages++ > maxPages) {
                 throw new UnexpectedPagingException(rs, getQueryString(), pages, maxPages, stmt.getPageSize());
             }
             if (rs.isFullyFetched()) {
+                results.set(resultRows);
                 break;
             }
             totalRows += pageRows;
         }
         processors.flush();
-        return rs;
+        return results.get();
     }
 
     @Override

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,7 +25,7 @@ public class Cqld4CqlPreparedStatement extends Cqld4CqlOp {
     private final BoundStatement stmt;
 
     public Cqld4CqlPreparedStatement(CqlSession session, BoundStatement stmt, int maxPages, boolean retryReplace, int maxLwtRetries, RSProcessors processors) {
-        super(session,maxPages,retryReplace,maxLwtRetries,processors);
+        super(session, maxPages, retryReplace, maxLwtRetries, processors);
         this.stmt = stmt;
     }
 

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java

@@ -21,11 +21,14 @@ import com.codahale.metrics.Timer;
 import io.nosqlbench.api.config.NBLabeledElement;
 import io.nosqlbench.api.config.NBLabels;
 import io.nosqlbench.api.engine.metrics.ActivityMetrics;
+import io.nosqlbench.api.errors.MVELCompilationError;
 import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
 import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op;
 import io.nosqlbench.engine.api.metrics.ThreadLocalNamedTimers;
 import io.nosqlbench.engine.api.templating.ParsedOp;
+import org.mvel2.MVEL;
 
+import java.io.Serializable;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -40,6 +43,7 @@ import java.util.concurrent.TimeUnit;
 public abstract class BaseOpDispenser<T extends Op, S> implements OpDispenser<T>, NBLabeledElement {
 
     private final String opName;
+    private Serializable expectedResultExpression;
     protected final DriverAdapter<T, S> adapter;
     private final NBLabels labels;
     private boolean instrument;
@@ -65,6 +69,27 @@ public abstract class BaseOpDispenser<T extends Op, S> implements OpDispenser<T>
         if (null != timerStarts)
             for (final String timerStart : this.timerStarts) ThreadLocalNamedTimers.addTimer(op, timerStart);
         this.configureInstrumentation(op);
+        this.configureResultExpectations(op);
+    }
+
+    public Serializable getExpectedResultExpression() {
+        return expectedResultExpression;
+    }
+
+    private void configureResultExpectations(ParsedOp op) {
+        op.getOptionalStaticValue("expected-result", String.class)
+            .map(this::compileExpectedResultExpression)
+            .ifPresent(result -> this.expectedResultExpression = result);
+    }
+
+    private Serializable compileExpectedResultExpression(String expectedResultExpression) {
+        try {
+            return MVEL.compileExpression(expectedResultExpression);
+        } catch (Exception e) {
+            throw new MVELCompilationError(
+                String.format("Failed to compile expected-result expression: \"%s\"", expectedResultExpression), e
+            );
+        }
     }
 
     String getOpName() {

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/OpDispenser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 
 package io.nosqlbench.engine.api.activityimpl;
 
+import java.io.Serializable;
 import java.util.function.LongFunction;
 
 /**
@@ -81,5 +82,6 @@ public interface OpDispenser<T> extends LongFunction<T>, OpResultTracker {
      */
 
     T apply(long value);
+    Serializable getExpectedResultExpression();
 
 }

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java

@@ -170,6 +170,7 @@ public abstract class BaseDriverAdapter<R extends Op, S> implements DriverAdapte
                 .add(Param.optional(List.of("workload", "yaml"), String.class, "location of workload yaml file"))
                 .add(Param.optional("driver", String.class))
                 .add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|none)"))
+                .add(Param.optional("maxtries", Integer.class))
                 .asReadOnly();
     }
 

engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/ErrorMetrics.java

@@ -19,6 +19,7 @@ package io.nosqlbench.engine.api.activityapi.errorhandling;
 import io.nosqlbench.api.config.NBLabeledElement;
 import io.nosqlbench.api.engine.activityimpl.ActivityDef;
 import io.nosqlbench.engine.api.metrics.ExceptionCountMetrics;
+import io.nosqlbench.engine.api.metrics.ExceptionExpectedResultVerificationMetrics;
 import io.nosqlbench.engine.api.metrics.ExceptionHistoMetrics;
 import io.nosqlbench.engine.api.metrics.ExceptionMeterMetrics;
 import io.nosqlbench.engine.api.metrics.ExceptionTimerMetrics;
@@ -32,6 +33,7 @@ public class ErrorMetrics {
     private ExceptionHistoMetrics exceptionHistoMetrics;
     private ExceptionMeterMetrics exceptionMeterMetrics;
     private ExceptionTimerMetrics exceptionTimerMetrics;
+    private ExceptionExpectedResultVerificationMetrics exceptionExpectedResultVerificationMetrics;
 
     public ErrorMetrics(final NBLabeledElement parentLabels) {
         this.parentLabels = parentLabels;
@@ -59,6 +61,12 @@ public class ErrorMetrics {
         return this.exceptionTimerMetrics;
     }
 
+    public synchronized ExceptionExpectedResultVerificationMetrics getExceptionExpectedResultVerificationMetrics() {
+        if (null == exceptionExpectedResultVerificationMetrics)
+            this.exceptionExpectedResultVerificationMetrics = new ExceptionExpectedResultVerificationMetrics(this.parentLabels);
+        return this.exceptionExpectedResultVerificationMetrics;
+    }
+
     public interface Aware {
         void setErrorMetricsSupplier(Supplier<ErrorMetrics> supplier);
     }

engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/handlers/ExpectedResultVerificationErrorHandler.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2022-2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.api.activityapi.errorhandling.modular.handlers;
+
+import io.nosqlbench.api.errors.ExpectedResultVerificationError;
+import io.nosqlbench.engine.api.activityapi.errorhandling.ErrorMetrics;
+import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorDetail;
+import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorHandler;
+import io.nosqlbench.engine.api.metrics.ExceptionExpectedResultVerificationMetrics;
+import io.nosqlbench.nb.annotations.Service;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.function.Supplier;
+
+/**
+ * The expected result verification error handler will create, if needed, two metric
+ * objects for error and retry counts.
+ */
+@Service(value = ErrorHandler.class, selector = "verifyexpected")
+public class ExpectedResultVerificationErrorHandler implements ErrorHandler, ErrorMetrics.Aware {
+    private static final Logger logger = LogManager.getLogger("VERIFY");
+    private ExceptionExpectedResultVerificationMetrics exceptionExpectedResultVerificationMetrics;
+
+    @Override
+    public ErrorDetail handleError(String name, Throwable t, long cycle, long durationInNanos, ErrorDetail detail) {
+        if (t instanceof ExpectedResultVerificationError erve) {
+            if (erve.getTriesLeft() == 0) {
+                logger.warn("Cycle: {} Verification of result did not pass following expression: {}", cycle, erve.getExpectedResultExpression());
+                exceptionExpectedResultVerificationMetrics.countVerificationErrors();
+            } else {
+                logger.info("Cycle: {} Verification of result did not pass. {} retries left.", cycle, erve.getTriesLeft());
+                exceptionExpectedResultVerificationMetrics.countVerificationRetries();
+            }
+        }
+        return detail;
+    }
+
+    @Override
+    public void setErrorMetricsSupplier(Supplier<ErrorMetrics> supplier) {
+        this.exceptionExpectedResultVerificationMetrics = supplier.get().getExceptionExpectedResultVerificationMetrics();
+    }
+
+}

engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/actions/StandardAction.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ package io.nosqlbench.engine.api.activityimpl.uniform.actions;
 
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
+import io.nosqlbench.api.errors.ExpectedResultVerificationError;
 import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
 import io.nosqlbench.engine.api.activityapi.core.SyncAction;
 import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorDetail;
@@ -29,7 +30,9 @@ import io.nosqlbench.engine.api.activityimpl.uniform.StandardActivity;
 import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.*;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.mvel2.MVEL;
 
+import java.io.Serializable;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -87,7 +90,7 @@ public class StandardAction<A extends StandardActivity<R, ?>, R extends Op> impl
         while (op != null) {
 
             int tries = 0;
-            while (tries++ <= maxTries) {
+            while (tries++ < maxTries) {
                 Throwable error = null;
                 long startedAt = System.nanoTime();
 
@@ -104,6 +107,13 @@ public class StandardAction<A extends StandardActivity<R, ?>, R extends Op> impl
                         throw new RuntimeException("The op implementation did not implement any active logic. Implement " +
                             "one of [RunnableOp, CycleOp, or ChainingOp]");
                     }
+                    var expectedResultExpression = dispenser.getExpectedResultExpression();
+                    if (shouldVerifyExpectedResultFor(op, expectedResultExpression)) {
+                        var verified = MVEL.executeExpression(expectedResultExpression, result, boolean.class);
+                        if (!verified) {
+                            throw new ExpectedResultVerificationError(maxTries - tries, expectedResultExpression);
+                        }
+                    }
                 } catch (Exception e) {
                     error = e;
                 } finally {
@@ -139,4 +149,8 @@ public class StandardAction<A extends StandardActivity<R, ?>, R extends Op> impl
     @Override
     public void onActivityDefUpdate(ActivityDef activityDef) {
     }
+
+    private boolean shouldVerifyExpectedResultFor(Op op, Serializable expectedResultExpression) {
+        return !(op instanceof RunnableOp) && expectedResultExpression != null;
+    }
 }

engine-api/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionExpectedResultVerificationMetrics.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2022-2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.api.metrics;
+
+import com.codahale.metrics.Counter;
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.engine.metrics.ActivityMetrics;
+
+
+/**
+ * Use this to provide exception metering during expected result verification.
+ */
+public class ExceptionExpectedResultVerificationMetrics {
+    private final NBLabeledElement parentLabels;
+    private final Counter verificationErrors;
+    private final Counter verificationRetries;
+
+    public ExceptionExpectedResultVerificationMetrics(final NBLabeledElement parentLabels) {
+        this.parentLabels = parentLabels;
+        verificationRetries = ActivityMetrics.counter(parentLabels, "verificationcounts.RETRIES");
+        verificationErrors = ActivityMetrics.counter(parentLabels, "verificationcounts.ERRORS");
+    }
+
+    public void countVerificationRetries() {
+        verificationRetries.inc();
+    }
+
+    public void countVerificationErrors() {
+        verificationErrors.inc();
+    }
+
+    public Counter getVerificationErrors() {
+        return verificationErrors;
+    }
+
+    public Counter getVerificationRetries() {
+        return verificationRetries;
+    }
+}

engine-api/src/test/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/NBErrorHandlerTest.java

@@ -21,6 +21,7 @@ import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Timer;
 import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.errors.ExpectedResultVerificationError;
 import io.nosqlbench.engine.api.activityapi.errorhandling.ErrorMetrics;
 import io.nosqlbench.engine.api.activityapi.errorhandling.modular.handlers.CountErrorHandler;
 import io.nosqlbench.engine.api.activityapi.errorhandling.modular.handlers.CounterErrorHandler;
@@ -30,8 +31,12 @@ import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.Logger;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import java.util.List;
+import java.util.stream.Stream;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
@@ -173,5 +178,54 @@ class NBErrorHandlerTest {
         appender.cleanup(logger);
     }
 
+    @ParameterizedTest(name = "Error with {0}")
+    @MethodSource
+    void testExpectedResultVerificationErrorHandler(String name, Exception error, String log, long retriesCount, long errorsCount, Logger logger) {
+        // given
+        NBMock.LogAppender appender = NBMock.registerTestLogger(ERROR_HANDLER_APPENDER_NAME, logger, Level.INFO);
+        var errorMetrics = new ErrorMetrics(NBLabeledElement.forKV("activity","testalias_result_verification_" + name));
+        var eh = new NBErrorHandler(() -> "verifyexpected", () -> errorMetrics);
+        var retries = errorMetrics.getExceptionExpectedResultVerificationMetrics().getVerificationRetries();
+        var errors = errorMetrics.getExceptionExpectedResultVerificationMetrics().getVerificationErrors();
+
+        assertThat(retries.getCount()).isEqualTo(0);
+        assertThat(errors.getCount()).isEqualTo(0);
+
+        // when
+        eh.handleError(error, 1, 2);
+
+        // then
+        assertThat(retries.getCount()).isEqualTo(retriesCount);
+        assertThat(errors.getCount()).isEqualTo(errorsCount);
+
+        logger.getContext().stop(); // force any async appenders to flush
+        logger.getContext().start(); // resume processing
+
+        assertThat(appender.getFirstEntry()).contains(log);
+        appender.cleanup(logger);
+    }
+
+    @SuppressWarnings("unused")
+    private static Stream<Arguments> testExpectedResultVerificationErrorHandler() {
+        Logger logger = (Logger) LogManager.getLogger("VERIFY");
+        return Stream.of(
+            Arguments.of(
+                "retries left",
+                new ExpectedResultVerificationError(5, "expected"),
+                "Cycle: 1 Verification of result did not pass. 5 retries left.",
+                1,
+                0,
+                logger
+            ),
+            Arguments.of(
+                "no retries left",
+                new ExpectedResultVerificationError(0, "expected"),
+                "Cycle: 1 Verification of result did not pass following expression: expected",
+                0,
+                1,
+                logger
+            )
+        );
+    }
 
 }

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -161,6 +161,7 @@ public class NBCLI implements Function<String[], Integer> {
             .setMaxLogs(globalOptions.getLogsMax())
             .setLogsDirectory(globalOptions.getLogsDirectory())
             .setAnsiEnabled(globalOptions.isEnableAnsi())
+            .setDedicatedVerificationLogger(globalOptions.isDedicatedVerificationLogger())
             .activate();
         ConfigurationFactory.setConfigurationFactory(NBCLI.loggerConfig);
 

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java

@@ -127,6 +127,7 @@ public class NBCLIOptions {
 
     private static final String DEFAULT_CONSOLE_PATTERN = "TERSE";
     private static final String DEFAULT_LOGFILE_PATTERN = "VERBOSE";
+    private final static String ENABLE_DEDICATED_VERIFICATION_LOGGER = "--enable-dedicated-verification-logging";
 
     //    private static final String DEFAULT_CONSOLE_LOGGING_PATTERN = "%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n";
 
@@ -190,6 +191,7 @@ public class NBCLIOptions {
     private String graphitelogLevel = "info";
     private boolean wantsListCommands;
     private boolean wantsListApps;
+    private boolean dedicatedVerificationLogger;
 
     public boolean isWantsListApps() {
         return this.wantsListApps;
@@ -232,6 +234,14 @@ public class NBCLIOptions {
         return graphitelogLevel;
     }
 
+    public boolean isDedicatedVerificationLogger() {
+        return this.dedicatedVerificationLogger;
+    }
+
+    public void enableDedicatedVerificationLogger() {
+        this.dedicatedVerificationLogger = true;
+    }
+
     public enum Mode {
         ParseGlobalsOnly,
         ParseAllOptions
@@ -340,6 +350,10 @@ public class NBCLIOptions {
                     showStackTraces = true;
                     arglist.removeFirst();
                     break;
+                case NBCLIOptions.ENABLE_DEDICATED_VERIFICATION_LOGGER:
+                    enableDedicatedVerificationLogger();
+                    arglist.removeFirst();
+                    break;
                 case NBCLIOptions.ANNOTATE_EVENTS:
                     arglist.removeFirst();
                     final String toAnnotate = this.readWordOrThrow(arglist, "annotated events");

engine-core/src/main/java/io/nosqlbench/engine/core/logging/LoggerConfig.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.Filter;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.apache.logging.log4j.core.appender.ConsoleAppender;
+import org.apache.logging.log4j.core.appender.FileAppender;
 import org.apache.logging.log4j.core.appender.RollingFileAppender;
 import org.apache.logging.log4j.core.config.Configuration;
 import org.apache.logging.log4j.core.config.ConfigurationFactory;
@@ -87,6 +88,7 @@ public class LoggerConfig extends ConfigurationFactory {
     private int maxLogfiles = 100;
     private String logfileLocation;
     private boolean ansiEnabled;
+    private boolean isDedicatedVerificationLoggerEnabled = false;
 
 
     public LoggerConfig() {
@@ -107,6 +109,11 @@ public class LoggerConfig extends ConfigurationFactory {
         return this;
     }
 
+    public LoggerConfig setDedicatedVerificationLogger(boolean enabled) {
+        this.isDedicatedVerificationLoggerEnabled = enabled;
+        return this;
+    }
+
     /**
      * Ensure that what is shown in the logfile includes at a minimum,
      * everything that is shown on console, but allow it to show more
@@ -209,6 +216,11 @@ public class LoggerConfig extends ConfigurationFactory {
                             .addComponent(triggeringPolicy);
             builder.add(logsAppenderBuilder);
 
+            if (isDedicatedVerificationLoggerEnabled) {
+                var verificationLogfilePath = loggerDir.resolve(filebase + "_verification.log").toString();
+                addResultVerificationLoggingChannel(builder, verificationLogfilePath);
+            }
+
             rootBuilder.add(
                     builder.newAppenderRef("SCENARIO_APPENDER")
                             .addAttribute("level", fileLevel)
@@ -367,4 +379,23 @@ public class LoggerConfig extends ConfigurationFactory {
         this.loggerDir = logsDirectory;
         return this;
     }
+
+    private void addResultVerificationLoggingChannel(ConfigurationBuilder<BuiltConfiguration> builder, String verificationLogfilePath) {
+        var appenderName = "RESULTVERIFYLOG";
+        var appender = builder
+            .newAppender(appenderName, FileAppender.PLUGIN_NAME)
+            .addAttribute("append", false)
+            .addAttribute("fileName", verificationLogfilePath)
+            .add(builder
+                .newLayout("PatternLayout")
+                .addAttribute("pattern", "%d %p %C{1.} [%t] %m%n")
+            );
+        var logger = builder
+            .newLogger("VERIFY", Level.INFO)
+            .add(builder.newAppenderRef(appenderName))
+            .addAttribute("additivity", false);
+
+        builder.add(appender);
+        builder.add(logger);
+    }
 }

nb-api/src/main/java/io/nosqlbench/api/errors/ExpectedResultVerificationError.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2022-2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.errors;
+
+import java.io.Serializable;
+
+public class ExpectedResultVerificationError extends RuntimeException {
+    private final int triesLeft;
+    private final Serializable expectedResultExpression;
+
+    public ExpectedResultVerificationError(int triesLeft, Serializable expectedResultExpression) {
+        this.triesLeft = triesLeft;
+        this.expectedResultExpression = expectedResultExpression;
+    }
+
+    public int getTriesLeft() {
+        return triesLeft;
+    }
+
+    public Serializable getExpectedResultExpression() {
+        return expectedResultExpression;
+    }
+
+}

nb-api/src/main/java/io/nosqlbench/api/errors/MVELCompilationError.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2022-2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.errors;
+
+/**
+ * MVELCompilationErrors are those known to occur during the compilation of expected results expressions.
+ */
+public class MVELCompilationError extends RuntimeException {
+    public MVELCompilationError(String message, Throwable cause) {
+        super(message, cause);
+    }
+}