diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4RawStmtDispenser.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4RawStmtDispenser.java index 16b999e60..7043b6715 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4RawStmtDispenser.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4RawStmtDispenser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4SimpleCqlStmtDispenser.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4SimpleCqlStmtDispenser.java index f58ce6dc3..0f71fb262 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4SimpleCqlStmtDispenser.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4SimpleCqlStmtDispenser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java index 67931b83a..8e675b36d 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,9 @@ import io.nosqlbench.adapter.cqld4.exceptions.UndefinedResultSetException; import io.nosqlbench.adapter.cqld4.exceptions.UnexpectedPagingException; import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.*; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.Map; @@ -42,7 +44,7 @@ import java.util.Map; // TODO: add rows histogram resultSetSizeHisto -public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, OpGenerator, OpResultSize { +public abstract class Cqld4CqlOp implements CycleOp>, VariableCapture, OpGenerator, OpResultSize { private final CqlSession session; private final int maxPages; @@ -54,6 +56,8 @@ public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, private Cqld4CqlOp nextOp; private final RSProcessors processors; + private final ThreadLocal> results = new ThreadLocal<>(); + public Cqld4CqlOp(CqlSession session, int maxPages, boolean retryReplace, int maxLwtRetries, RSProcessors processors) { this.session = session; this.maxPages = maxPages; @@ -71,7 +75,7 @@ public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, this.processors = processors; } - public final ResultSet apply(long cycle) { + public final List apply(long cycle) { Statement stmt = getStmt(); rs = session.execute(stmt); @@ -97,22 +101,29 @@ public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, Iterator reader = rs.iterator(); int pages = 0; + // TODO/MVEL: An optimization to this would be to collect the results in a result set processor, + // but allow/require this processor to be added to an op _only_ in the event that it would + // be needed by a downstream consumer like the MVEL expected result evaluator + + var resultRows = new ArrayList(); while (true) { int pageRows = rs.getAvailableWithoutFetching(); for (int i = 0; i < pageRows; i++) { Row row = reader.next(); + resultRows.add(row); processors.buffer(row); } if (pages++ > maxPages) { throw new UnexpectedPagingException(rs, getQueryString(), pages, maxPages, stmt.getPageSize()); } if (rs.isFullyFetched()) { + results.set(resultRows); break; } totalRows += pageRows; } processors.flush(); - return rs; + return results.get(); } @Override diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java index 9e6dfcad9..44a4b85b1 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ public class Cqld4CqlPreparedStatement extends Cqld4CqlOp { private final BoundStatement stmt; public Cqld4CqlPreparedStatement(CqlSession session, BoundStatement stmt, int maxPages, boolean retryReplace, int maxLwtRetries, RSProcessors processors) { - super(session,maxPages,retryReplace,maxLwtRetries,processors); + super(session, maxPages, retryReplace, maxLwtRetries, processors); this.stmt = stmt; } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java index 0f119c91d..959a3ea89 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java index bed84ec36..8c833833f 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java @@ -21,11 +21,14 @@ import com.codahale.metrics.Timer; import io.nosqlbench.api.config.NBLabeledElement; import io.nosqlbench.api.config.NBLabels; import io.nosqlbench.api.engine.metrics.ActivityMetrics; +import io.nosqlbench.api.errors.MVELCompilationError; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.engine.api.metrics.ThreadLocalNamedTimers; import io.nosqlbench.engine.api.templating.ParsedOp; +import org.mvel2.MVEL; +import java.io.Serializable; import java.util.concurrent.TimeUnit; /** @@ -40,6 +43,7 @@ import java.util.concurrent.TimeUnit; public abstract class BaseOpDispenser implements OpDispenser, NBLabeledElement { private final String opName; + private Serializable expectedResultExpression; protected final DriverAdapter adapter; private final NBLabels labels; private boolean instrument; @@ -65,6 +69,27 @@ public abstract class BaseOpDispenser implements OpDispenser if (null != timerStarts) for (final String timerStart : this.timerStarts) ThreadLocalNamedTimers.addTimer(op, timerStart); this.configureInstrumentation(op); + this.configureResultExpectations(op); + } + + public Serializable getExpectedResultExpression() { + return expectedResultExpression; + } + + private void configureResultExpectations(ParsedOp op) { + op.getOptionalStaticValue("expected-result", String.class) + .map(this::compileExpectedResultExpression) + .ifPresent(result -> this.expectedResultExpression = result); + } + + private Serializable compileExpectedResultExpression(String expectedResultExpression) { + try { + return MVEL.compileExpression(expectedResultExpression); + } catch (Exception e) { + throw new MVELCompilationError( + String.format("Failed to compile expected-result expression: \"%s\"", expectedResultExpression), e + ); + } } String getOpName() { diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/OpDispenser.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/OpDispenser.java index 162621902..097744a5f 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/OpDispenser.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/OpDispenser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package io.nosqlbench.engine.api.activityimpl; +import java.io.Serializable; import java.util.function.LongFunction; /** @@ -81,5 +82,6 @@ public interface OpDispenser extends LongFunction, OpResultTracker { */ T apply(long value); + Serializable getExpectedResultExpression(); } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java index 074413728..1e3bb9711 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java @@ -170,6 +170,7 @@ public abstract class BaseDriverAdapter implements DriverAdapte .add(Param.optional(List.of("workload", "yaml"), String.class, "location of workload yaml file")) .add(Param.optional("driver", String.class)) .add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|none)")) + .add(Param.optional("maxtries", Integer.class)) .asReadOnly(); } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/ErrorMetrics.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/ErrorMetrics.java index c408d6c50..acc3e5433 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/ErrorMetrics.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/ErrorMetrics.java @@ -19,6 +19,7 @@ package io.nosqlbench.engine.api.activityapi.errorhandling; import io.nosqlbench.api.config.NBLabeledElement; import io.nosqlbench.api.engine.activityimpl.ActivityDef; import io.nosqlbench.engine.api.metrics.ExceptionCountMetrics; +import io.nosqlbench.engine.api.metrics.ExceptionExpectedResultVerificationMetrics; import io.nosqlbench.engine.api.metrics.ExceptionHistoMetrics; import io.nosqlbench.engine.api.metrics.ExceptionMeterMetrics; import io.nosqlbench.engine.api.metrics.ExceptionTimerMetrics; @@ -32,6 +33,7 @@ public class ErrorMetrics { private ExceptionHistoMetrics exceptionHistoMetrics; private ExceptionMeterMetrics exceptionMeterMetrics; private ExceptionTimerMetrics exceptionTimerMetrics; + private ExceptionExpectedResultVerificationMetrics exceptionExpectedResultVerificationMetrics; public ErrorMetrics(final NBLabeledElement parentLabels) { this.parentLabels = parentLabels; @@ -59,6 +61,12 @@ public class ErrorMetrics { return this.exceptionTimerMetrics; } + public synchronized ExceptionExpectedResultVerificationMetrics getExceptionExpectedResultVerificationMetrics() { + if (null == exceptionExpectedResultVerificationMetrics) + this.exceptionExpectedResultVerificationMetrics = new ExceptionExpectedResultVerificationMetrics(this.parentLabels); + return this.exceptionExpectedResultVerificationMetrics; + } + public interface Aware { void setErrorMetricsSupplier(Supplier supplier); } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/handlers/ExpectedResultVerificationErrorHandler.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/handlers/ExpectedResultVerificationErrorHandler.java new file mode 100644 index 000000000..c2ca0cde9 --- /dev/null +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/handlers/ExpectedResultVerificationErrorHandler.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.api.activityapi.errorhandling.modular.handlers; + +import io.nosqlbench.api.errors.ExpectedResultVerificationError; +import io.nosqlbench.engine.api.activityapi.errorhandling.ErrorMetrics; +import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorDetail; +import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorHandler; +import io.nosqlbench.engine.api.metrics.ExceptionExpectedResultVerificationMetrics; +import io.nosqlbench.nb.annotations.Service; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.Supplier; + +/** + * The expected result verification error handler will create, if needed, two metric + * objects for error and retry counts. + */ +@Service(value = ErrorHandler.class, selector = "verifyexpected") +public class ExpectedResultVerificationErrorHandler implements ErrorHandler, ErrorMetrics.Aware { + private static final Logger logger = LogManager.getLogger("VERIFY"); + private ExceptionExpectedResultVerificationMetrics exceptionExpectedResultVerificationMetrics; + + @Override + public ErrorDetail handleError(String name, Throwable t, long cycle, long durationInNanos, ErrorDetail detail) { + if (t instanceof ExpectedResultVerificationError erve) { + if (erve.getTriesLeft() == 0) { + logger.warn("Cycle: {} Verification of result did not pass following expression: {}", cycle, erve.getExpectedResultExpression()); + exceptionExpectedResultVerificationMetrics.countVerificationErrors(); + } else { + logger.info("Cycle: {} Verification of result did not pass. {} retries left.", cycle, erve.getTriesLeft()); + exceptionExpectedResultVerificationMetrics.countVerificationRetries(); + } + } + return detail; + } + + @Override + public void setErrorMetricsSupplier(Supplier supplier) { + this.exceptionExpectedResultVerificationMetrics = supplier.get().getExceptionExpectedResultVerificationMetrics(); + } + +} diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/actions/StandardAction.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/actions/StandardAction.java index d1f5010e8..b0ab2aa8d 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/actions/StandardAction.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/actions/StandardAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ package io.nosqlbench.engine.api.activityimpl.uniform.actions; import com.codahale.metrics.Histogram; import com.codahale.metrics.Timer; +import io.nosqlbench.api.errors.ExpectedResultVerificationError; import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver; import io.nosqlbench.engine.api.activityapi.core.SyncAction; import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorDetail; @@ -29,7 +30,9 @@ import io.nosqlbench.engine.api.activityimpl.uniform.StandardActivity; import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.mvel2.MVEL; +import java.io.Serializable; import java.util.concurrent.TimeUnit; /** @@ -87,7 +90,7 @@ public class StandardAction, R extends Op> impl while (op != null) { int tries = 0; - while (tries++ <= maxTries) { + while (tries++ < maxTries) { Throwable error = null; long startedAt = System.nanoTime(); @@ -104,6 +107,13 @@ public class StandardAction, R extends Op> impl throw new RuntimeException("The op implementation did not implement any active logic. Implement " + "one of [RunnableOp, CycleOp, or ChainingOp]"); } + var expectedResultExpression = dispenser.getExpectedResultExpression(); + if (shouldVerifyExpectedResultFor(op, expectedResultExpression)) { + var verified = MVEL.executeExpression(expectedResultExpression, result, boolean.class); + if (!verified) { + throw new ExpectedResultVerificationError(maxTries - tries, expectedResultExpression); + } + } } catch (Exception e) { error = e; } finally { @@ -139,4 +149,8 @@ public class StandardAction, R extends Op> impl @Override public void onActivityDefUpdate(ActivityDef activityDef) { } + + private boolean shouldVerifyExpectedResultFor(Op op, Serializable expectedResultExpression) { + return !(op instanceof RunnableOp) && expectedResultExpression != null; + } } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionExpectedResultVerificationMetrics.java b/engine-api/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionExpectedResultVerificationMetrics.java new file mode 100644 index 000000000..e54726dfc --- /dev/null +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionExpectedResultVerificationMetrics.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.api.metrics; + +import com.codahale.metrics.Counter; +import io.nosqlbench.api.config.NBLabeledElement; +import io.nosqlbench.api.engine.metrics.ActivityMetrics; + + +/** + * Use this to provide exception metering during expected result verification. + */ +public class ExceptionExpectedResultVerificationMetrics { + private final NBLabeledElement parentLabels; + private final Counter verificationErrors; + private final Counter verificationRetries; + + public ExceptionExpectedResultVerificationMetrics(final NBLabeledElement parentLabels) { + this.parentLabels = parentLabels; + verificationRetries = ActivityMetrics.counter(parentLabels, "verificationcounts.RETRIES"); + verificationErrors = ActivityMetrics.counter(parentLabels, "verificationcounts.ERRORS"); + } + + public void countVerificationRetries() { + verificationRetries.inc(); + } + + public void countVerificationErrors() { + verificationErrors.inc(); + } + + public Counter getVerificationErrors() { + return verificationErrors; + } + + public Counter getVerificationRetries() { + return verificationRetries; + } +} diff --git a/engine-api/src/test/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/NBErrorHandlerTest.java b/engine-api/src/test/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/NBErrorHandlerTest.java index bff836d46..7534f9966 100644 --- a/engine-api/src/test/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/NBErrorHandlerTest.java +++ b/engine-api/src/test/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/NBErrorHandlerTest.java @@ -21,6 +21,7 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; import io.nosqlbench.api.config.NBLabeledElement; +import io.nosqlbench.api.errors.ExpectedResultVerificationError; import io.nosqlbench.engine.api.activityapi.errorhandling.ErrorMetrics; import io.nosqlbench.engine.api.activityapi.errorhandling.modular.handlers.CountErrorHandler; import io.nosqlbench.engine.api.activityapi.errorhandling.modular.handlers.CounterErrorHandler; @@ -30,8 +31,12 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.Logger; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.util.List; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; @@ -173,5 +178,54 @@ class NBErrorHandlerTest { appender.cleanup(logger); } + @ParameterizedTest(name = "Error with {0}") + @MethodSource + void testExpectedResultVerificationErrorHandler(String name, Exception error, String log, long retriesCount, long errorsCount, Logger logger) { + // given + NBMock.LogAppender appender = NBMock.registerTestLogger(ERROR_HANDLER_APPENDER_NAME, logger, Level.INFO); + var errorMetrics = new ErrorMetrics(NBLabeledElement.forKV("activity","testalias_result_verification_" + name)); + var eh = new NBErrorHandler(() -> "verifyexpected", () -> errorMetrics); + var retries = errorMetrics.getExceptionExpectedResultVerificationMetrics().getVerificationRetries(); + var errors = errorMetrics.getExceptionExpectedResultVerificationMetrics().getVerificationErrors(); + + assertThat(retries.getCount()).isEqualTo(0); + assertThat(errors.getCount()).isEqualTo(0); + + // when + eh.handleError(error, 1, 2); + + // then + assertThat(retries.getCount()).isEqualTo(retriesCount); + assertThat(errors.getCount()).isEqualTo(errorsCount); + + logger.getContext().stop(); // force any async appenders to flush + logger.getContext().start(); // resume processing + + assertThat(appender.getFirstEntry()).contains(log); + appender.cleanup(logger); + } + + @SuppressWarnings("unused") + private static Stream testExpectedResultVerificationErrorHandler() { + Logger logger = (Logger) LogManager.getLogger("VERIFY"); + return Stream.of( + Arguments.of( + "retries left", + new ExpectedResultVerificationError(5, "expected"), + "Cycle: 1 Verification of result did not pass. 5 retries left.", + 1, + 0, + logger + ), + Arguments.of( + "no retries left", + new ExpectedResultVerificationError(0, "expected"), + "Cycle: 1 Verification of result did not pass following expression: expected", + 0, + 1, + logger + ) + ); + } } diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index ed9223463..7f252dd6b 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -161,6 +161,7 @@ public class NBCLI implements Function { .setMaxLogs(globalOptions.getLogsMax()) .setLogsDirectory(globalOptions.getLogsDirectory()) .setAnsiEnabled(globalOptions.isEnableAnsi()) + .setDedicatedVerificationLogger(globalOptions.isDedicatedVerificationLogger()) .activate(); ConfigurationFactory.setConfigurationFactory(NBCLI.loggerConfig); diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java index dde6f53a6..1687a225e 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java @@ -127,6 +127,7 @@ public class NBCLIOptions { private static final String DEFAULT_CONSOLE_PATTERN = "TERSE"; private static final String DEFAULT_LOGFILE_PATTERN = "VERBOSE"; + private final static String ENABLE_DEDICATED_VERIFICATION_LOGGER = "--enable-dedicated-verification-logging"; // private static final String DEFAULT_CONSOLE_LOGGING_PATTERN = "%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n"; @@ -190,6 +191,7 @@ public class NBCLIOptions { private String graphitelogLevel = "info"; private boolean wantsListCommands; private boolean wantsListApps; + private boolean dedicatedVerificationLogger; public boolean isWantsListApps() { return this.wantsListApps; @@ -232,6 +234,14 @@ public class NBCLIOptions { return graphitelogLevel; } + public boolean isDedicatedVerificationLogger() { + return this.dedicatedVerificationLogger; + } + + public void enableDedicatedVerificationLogger() { + this.dedicatedVerificationLogger = true; + } + public enum Mode { ParseGlobalsOnly, ParseAllOptions @@ -340,6 +350,10 @@ public class NBCLIOptions { showStackTraces = true; arglist.removeFirst(); break; + case NBCLIOptions.ENABLE_DEDICATED_VERIFICATION_LOGGER: + enableDedicatedVerificationLogger(); + arglist.removeFirst(); + break; case NBCLIOptions.ANNOTATE_EVENTS: arglist.removeFirst(); final String toAnnotate = this.readWordOrThrow(arglist, "annotated events"); diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/logging/LoggerConfig.java b/engine-core/src/main/java/io/nosqlbench/engine/core/logging/LoggerConfig.java index e97f20ebf..b3f95008a 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/logging/LoggerConfig.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/logging/LoggerConfig.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.ConsoleAppender; +import org.apache.logging.log4j.core.appender.FileAppender; import org.apache.logging.log4j.core.appender.RollingFileAppender; import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.ConfigurationFactory; @@ -87,6 +88,7 @@ public class LoggerConfig extends ConfigurationFactory { private int maxLogfiles = 100; private String logfileLocation; private boolean ansiEnabled; + private boolean isDedicatedVerificationLoggerEnabled = false; public LoggerConfig() { @@ -107,6 +109,11 @@ public class LoggerConfig extends ConfigurationFactory { return this; } + public LoggerConfig setDedicatedVerificationLogger(boolean enabled) { + this.isDedicatedVerificationLoggerEnabled = enabled; + return this; + } + /** * Ensure that what is shown in the logfile includes at a minimum, * everything that is shown on console, but allow it to show more @@ -209,6 +216,11 @@ public class LoggerConfig extends ConfigurationFactory { .addComponent(triggeringPolicy); builder.add(logsAppenderBuilder); + if (isDedicatedVerificationLoggerEnabled) { + var verificationLogfilePath = loggerDir.resolve(filebase + "_verification.log").toString(); + addResultVerificationLoggingChannel(builder, verificationLogfilePath); + } + rootBuilder.add( builder.newAppenderRef("SCENARIO_APPENDER") .addAttribute("level", fileLevel) @@ -367,4 +379,23 @@ public class LoggerConfig extends ConfigurationFactory { this.loggerDir = logsDirectory; return this; } + + private void addResultVerificationLoggingChannel(ConfigurationBuilder builder, String verificationLogfilePath) { + var appenderName = "RESULTVERIFYLOG"; + var appender = builder + .newAppender(appenderName, FileAppender.PLUGIN_NAME) + .addAttribute("append", false) + .addAttribute("fileName", verificationLogfilePath) + .add(builder + .newLayout("PatternLayout") + .addAttribute("pattern", "%d %p %C{1.} [%t] %m%n") + ); + var logger = builder + .newLogger("VERIFY", Level.INFO) + .add(builder.newAppenderRef(appenderName)) + .addAttribute("additivity", false); + + builder.add(appender); + builder.add(logger); + } } diff --git a/nb-api/src/main/java/io/nosqlbench/api/errors/ExpectedResultVerificationError.java b/nb-api/src/main/java/io/nosqlbench/api/errors/ExpectedResultVerificationError.java new file mode 100644 index 000000000..44d70c594 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/api/errors/ExpectedResultVerificationError.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.api.errors; + +import java.io.Serializable; + +public class ExpectedResultVerificationError extends RuntimeException { + private final int triesLeft; + private final Serializable expectedResultExpression; + + public ExpectedResultVerificationError(int triesLeft, Serializable expectedResultExpression) { + this.triesLeft = triesLeft; + this.expectedResultExpression = expectedResultExpression; + } + + public int getTriesLeft() { + return triesLeft; + } + + public Serializable getExpectedResultExpression() { + return expectedResultExpression; + } + +} diff --git a/nb-api/src/main/java/io/nosqlbench/api/errors/MVELCompilationError.java b/nb-api/src/main/java/io/nosqlbench/api/errors/MVELCompilationError.java new file mode 100644 index 000000000..07ec6573d --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/api/errors/MVELCompilationError.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.api.errors; + +/** + * MVELCompilationErrors are those known to occur during the compilation of expected results expressions. + */ +public class MVELCompilationError extends RuntimeException { + public MVELCompilationError(String message, Throwable cause) { + super(message, cause); + } +}