Merge pull request #872 from nosqlbench/nosqlbench-797-callable

(see individual commits) merge concurrency fixes into main (nosqlbench-797-callable)

.github/workflows/build.yml

@@ -91,6 +91,8 @@ jobs:
           NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
           NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
         run: |
+          set -x
+          find . -ls
           rsync -av --delete -I --exclude '_index.md' drivers/ nosqlbench-build-docs/site/content/docs/drivers
           rsync -av --delete -I --exclude '_index.md' bindings/ nosqlbench-build-docs/site/content/docs/bindings
           echo "previewdocs.nosqlbench.io" > nosqlbench-build-docs/site/staticCNAME

adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagSpace.java

@@ -38,6 +38,7 @@ public class DiagSpace implements ActivityDefObserver, AutoCloseable {
     public DiagSpace(String name, NBConfiguration cfg) {
         this.cfg = cfg;
         this.name = name;
+        applyConfig(cfg);
         logger.trace("diag space initialized as '" + name + "'");
     }
 

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/ChainingOp.java

@@ -19,7 +19,7 @@ package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes;
 import java.util.function.Function;
 
 /**
- * <H2>ChainingOp<I,O>: f(I) -> O</I,O></H2>
+ * <H2>ChainingOp&lt;I,O&gt;: f(I) -> O&lt;I,O&gt;</H2>
  * <P>
  * Run a function on the current cached result in the current thread and replace it
  * with the result of the function. ChainingOps are one way of invoking

devdocs/concurrency/async_activity.puml

@@ -0,0 +1,105 @@
+@startuml
+'https://plantuml.com/sequence-diagram
+title: Lifecycle of an activity
+
+control caller as caller
+control ActivityExecutor as ae
+control "Activity\nException\nHandler" as aeh
+control "Activity\nThread\nFactory" as atf
+control ExecutorService as aes
+control Annotator as ann
+control Activity as activity
+
+== startup sequence ==
+caller -> ae**: create
+  ae -> aeh**: create
+  ae -> atf**: create(\w Exception Handler)
+  aeh -> atf: <injected\nvia ctor>
+ ae -> aes**: create(\w Thread Factory)
+  atf -> aes: <injected\nvia ctor>
+
+caller -> ae: startActivity()
+activate ae
+ ae -> ann: Annotate Activity Start
+
+ ae -> activity: initActivity()
+ activate activity
+ ae <- activity
+ deactivate activity
+
+ note over ae,aes: align threadcount as explained below
+
+caller <- ae
+deactivate ae
+
+== dynamic threadcount update ==
+note over ae, aes: threads can be changed dynamically
+
+caller -> ae: apply params
+activate ae
+ ae->ae: align motor count
+ ae->aes: stop extra motors
+ ae->aes: <start missing motors>
+ group for each new thread/motor
+  ae -> aes: execute(<motor>)
+ activate aes
+  aes -> atf: get()
+  atf -> thread**: create
+  activate atf
+  aes <- atf: <thread>
+  deactivate atf
+  aes --> thread: run()
+  note over ann, thread: At this point, the\nmotor thread starts running\nthe defined activity's action\nover cycles
+  ae->ae: await thread state update
+
+ ae<-aes:
+ deactivate aes
+ end group
+caller <- ae
+deactivate ae
+
+== shutdown sequence [after startup] ==
+
+caller -> ae: stopActivity()
+activate ae
+
+ ae -> ae: request stop motors
+ ae -> ae: await all stop
+
+ ae -> activity: shutdownActivity()
+ activate activity
+ ae <- activity
+ deactivate activity
+
+ ae -> ann: Annotate Activity Finish
+
+caller <- ae
+deactivate ae
+
+== on exception in motor thread ==
+thread -> aeh: catch(<thrown exception>)
+aeh -> ae: notifyException\n(<thread>,<throwable>)
+activate ae
+ ae -> ae: save exception
+ ae -> ae: forceStopActivity()
+ ae -> aes: shutdown();
+ activate aes
+ ae <- aes:
+ deactivate aes
+
+ group if needed [after timeout]]
+  ae -> aes: shutdownNow();
+  activate aes
+  ae <- aes
+  deactivate aes
+ end group
+
+ ae -> activity: shutdownActivity();
+ ae -> activity: closeAutoCloseables();
+
+ note over thread: action\nthread\nterminates
+ destroy thread
+deactivate ae
+
+
+@enduml

devdocs/concurrency/async_scenario.puml

@@ -0,0 +1,67 @@
+@startuml
+'https://plantuml.com/sequence-diagram
+
+title Lifecycle of a single scenario.call()
+control "caller" as c
+control "Scenario" as s
+control "Scenario\nController" as sc
+control "Scripting\nEngine" as engine
+control "Activity\nExecutor" as ae
+control "Java\nRuntime" as jrt
+control "Shutdown\nHook" as sh
+control "Annotations" as ann
+
+c -> s**: create
+
+c -> s: call()
+activate s
+
+ s -> sh**: create
+ s -> jrt: register(ShutdownHook)
+ s -> ann: Annotate Scenario Start
+
+ s -> sc**: create
+ s -> engine**: create
+
+ s -> engine: run(script)
+ activate engine
+ group async calls [javacript+Java]
+ engine <--> sc: scenario.(*)
+ engine <--> sc: activities.(*)
+ engine <--> sc: metrics.(*)
+ engine <--> sc: params.(*)
+ engine -> sc: start(<activity>)
+ activate sc
+ sc -> ae**: create
+ sc -> ae: startActivity()
+
+ deactivate sc
+ end group
+ s <- engine: result
+ deactivate engine
+
+ s -> sc: awaitCompletion()
+ activate sc
+  group for each activity
+   sc -> ae: awaitCompletion()
+   activate ae
+   sc <- ae
+   deactivate ae
+  end group
+
+ s <- sc
+ deactivate sc
+
+ s -> jrt: unregister(ShutdownHook)
+ s -> sh: run()
+ sh -> ann: Annotate Scenario Finish
+
+c <- s: Scenario\nResult
+deactivate s
+
+== on exception during call() ==
+ jrt -> sh: run()
+ sh -> ann: Annotate Scenario Finish
+
+
+@enduml

devdocs/concurrency/async_scenarios.puml

@@ -0,0 +1,62 @@
+@startuml
+'https://plantuml.com/sequence-diagram
+
+title Lifecycle of Scenarios
+
+control "NBCLI" as nbcli
+control "Scenario" as s
+control "Scenario\nController" as sc
+control "Scenarios\nExecutor" as se
+control "Exception\nHandler" as seh
+control "Thread\nFactory" as stf
+control "Executor\nService" as ses
+
+nbcli -> se** : create
+ se -> seh** : create
+ se -> stf** : create ThreadFactory\n(w/ ExceptionHandler)
+ se -> ses** : create ExecutorService\n(w/ ThreadFactory)
+
+nbcli -> s** : create
+ s -> sc** : create
+nbcli --> se : execute(Scenario)
+se --> ses: submit(<Callable> Scenario)
+activate ses
+ ses -> future**: create
+se <-- ses: <Future<ScenarioResult>>
+deactivate ses
+
+== [async] on thread from thread factory ==
+ses -> stf: get()
+ stf -> thread**: create
+ses <- stf: <thread>
+ses -> thread: run task
+activate thread
+thread -> s: call()
+activate s
+thread <- s: ScenarioResult
+deactivate s
+thread -> future: result
+deactivate thread
+
+== [async] on NBCLI thread ==
+
+nbcli -> se: awaitAllResults();
+activate se
+ se -> ses: shutdown
+ loop timeout
+ se -> ses: awaitTermination(timeout)
+ activate ses
+ se <- ses
+ deactivate ses
+ end loop
+ loop each future
+ se -> future: get()
+ activate future
+ se <- future: ScenarioResult
+ deactivate future
+ end loop
+
+nbcli <- se: <ScenariosResults>
+deactivate se
+
+@enduml

engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/Activity.java

@@ -28,6 +28,7 @@ import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
 import io.nosqlbench.api.engine.activityimpl.ActivityDef;
 import io.nosqlbench.api.engine.activityimpl.ParameterMap;
 import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
+import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
 
 import java.io.InputStream;
 import java.io.PrintWriter;
@@ -215,4 +216,6 @@ public interface Activity extends Comparable<Activity>, ActivityDefObserver, Pro
     default int getHdrDigits() {
         return getParams().getOptionalInteger("hdr_digits").orElse(4);
     }
+
+    RunStateTally getRunStateTally();
 }

engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/Motor.java

@@ -17,7 +17,7 @@
 package io.nosqlbench.engine.api.activityapi.core;
 
 import io.nosqlbench.engine.api.activityapi.input.Input;
-import io.nosqlbench.engine.api.activityimpl.SlotStateTracker;
+import io.nosqlbench.engine.api.activityimpl.MotorState;
 
 /**
  * The core threading harness within an activity.
@@ -54,6 +54,7 @@ public interface Motor<T> extends Runnable, Stoppable {
      * Get a description of the current slot run status.
      * @return - a value from the {@link RunState} enum
      */
-    SlotStateTracker getSlotStateTracker();
+    MotorState getState();
 
+    void removeState();
 }

engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/RunState.java

@@ -16,21 +16,69 @@
 
 package io.nosqlbench.engine.api.activityapi.core;
 
+import io.nosqlbench.engine.api.activityimpl.MotorState;
+
+/**
+ * <P>This enum indicates the state of a thread within an activity. The state is kept in an atomic
+ * register. Ownership of state changes is shared between a supervising thread and a managed thread.
+ * Both can make changes to the state.</P>
+ *
+ * <P>These states are ordered such that the highest ordinal state represents the most significant
+ * aggregate state of all motors. That is, if any has errored, then the other states do not matter.
+ * If any is finished, then stopped motors don't matter, and so on. This makes summarizing aggregate
+ * state simply a matter of ordering.</P>
+ *
+ */
 public enum RunState {
 
-    // Initial state after creation of this control
+    /**
+     * Initial state after creation of a motor. This is the initial state upon instantiation of a motor, before
+     * it is called on to do any active logic besides what occurs in the constructor.
+     */
     Uninitialized("i⌀"),
-    // This thread has been queued to run, but hasn't signaled yet that it is full started
-    // This must be set by the executor before executing the slot runnable
+
+    /**
+     * A thread has been invoked, but is initializing and preparing for its main control loop.
+     * This is signaled <EM>by the motor</EM> after {@link Runnable#run}, but before entering the main processing
+     * loop.
+     */
     Starting("s⏫"),
-    // This thread is running. This should only be set by the controlled thread
+
+    /**
+     * A thread is iterating within the main control loop.
+     * This is signaled <EM>by the motor</EM> once initialization in the main loop is complete and immediately
+     * before it enters it's main processing loop.
+     */
     Running("R\u23F5"),
-    // This thread has completed all of its activity, and will do no further work without new input
-    Finished("F⏯"),
-    // The thread has been requested to stop. This says nothing of the internal state.
+
+    /**
+     * <P>The thread has been requested to stop. This can be set by a managing thread which is not the
+     * motor thread, or by the motor thread. In either case, the motor thread is required to observe changes to this and initiate shutdown.</P>
+     */
     Stopping("s⏬"),
-    // The thread has stopped. This should only be set by the controlled thread
-    Stopped("_\u23F9");
+
+    /**
+     * The thread has stopped. This should only be set by the motor. This state will only be visible
+     * to signaling mechanisms so long as the motor is still managed.
+     *
+     * <P>NOTE: When a motor is stopped or finished, its state will remain visible in state tracking until
+     * {@link Motor#getState()}.{@link MotorState#removeState()} is called.</P>
+     */
+    Stopped("e\u23F9"),
+
+    /**
+     * <P>A thread has exhausted its supply of values on the input (AKA cycles), thus has completed its work.
+     * This is signaled upon a short read of the input <EM>by the motor</EM>.</P>
+     *
+     * <P>NOTE: When a motor is stopped or finished, its state will remain visible in state tracking until
+     * {@link Motor#getState()}.{@link MotorState#removeState()} is called.</P>
+     */
+    Finished("F⏯"),
+
+    /**
+     * If a motor has seen an exception, it goes into errored state before propagating the error.
+     */
+    Errored("E⚠");
 
     private final String runcode;
 
@@ -42,53 +90,26 @@ public enum RunState {
         return this.runcode;
     }
 
-    public boolean canTransitionTo(RunState to) {
-        switch (this) {
-            default:
-                return false;
-            case Uninitialized: // A motor was just created. This is its initial state.
-            case Stopped:
-                switch (to) {
-                    case Starting: // a motor has been reserved for an execution command
-                        return true;
-                    default:
-                        return false;
-                }
-            case Starting:
-                switch (to) {
-                    case Running: // a motor has indicated that is in the run() method
-                    case Finished: // a motor has exhausted its input, and has declined to go into started mode
-                        return true;
-                    default:
-                        return false;
-                }
-            case Running:
-                switch (to) {
-                    case Stopping: // A request was made to stop the motor before it finished
-                    case Finished: // A motor has exhausted its input, and is finished with its work
-                        return true;
-                    default:
-                        return false;
-                }
-            case Stopping:
-                switch (to) {
-                    case Stopped: // A motor was stopped by request before exhausting input
-                        return true;
-                    default:
-                        return false;
-                }// A motor was restarted after being stopped
-            case Finished:
-                switch (to) {
-                    case Running: // A motor was restarted?
-                        return true;
-                    // not useful as of yet.
-                    // Perhaps this will be allowed via explicit reset of input stream.
-                    // If the input isn't reset, then trying to start a finished motor
-                    // will cause it to short-circuit back to Finished state.
-                    default:
-                        return false;
-                }
-        }
+    /**
+     * @param target The target state
+     * @return true if the current state is allowed to transition to the target state
+     */
+    public boolean canTransitionTo(RunState target) {
+        return switch (this) {
+            default -> false; // A motor was just created. This is its initial state.
+            case Uninitialized, Stopped -> (target == Starting);
+            case Starting -> switch (target) { // a motor has indicated that is in the run() method
+                case Running, Finished, Errored -> true;// a motor has exhausted its input, and has declined to go into started mode
+                default -> false;
+            };
+            case Running -> switch (target) { // A request was made to stop the motor before it finished
+                case Stopping, Finished, Errored -> true;// A motor has exhausted its input, and is finished with its work
+                default -> false;
+            };
+            case Stopping -> (target == Stopped||target==Finished); // A motor was stopped by request before exhausting input
+            case Finished -> (target == Running); // A motor was restarted?
+            case Errored -> target==Errored;
+        };
 
     }
 

engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/MotorState.java

@@ -17,26 +17,31 @@
 package io.nosqlbench.engine.api.activityimpl;
 
 import io.nosqlbench.engine.api.activityapi.core.RunState;
-import org.apache.logging.log4j.Logger;
+import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
 
 /**
  * Holds the state of a slot, allows only valid transitions, and shares the
  * slot state as
  */
-public class SlotStateTracker {
-    private final AtomicReference<RunState> slotState = new AtomicReference<>(RunState.Uninitialized);
-    private final static Logger logger = LogManager.getLogger(SlotStateTracker.class);
+public class MotorState implements Supplier<RunState> {
+    private final static Logger logger = LogManager.getLogger("MOTORS");
+    private final AtomicReference<RunState> atomicState = new AtomicReference<>(RunState.Uninitialized);
     private final long slotId;
+    private final RunStateTally tally;
 
-    public SlotStateTracker(long slotId) {
+    public MotorState(long slotId, RunStateTally tally) {
         this.slotId = slotId;
+        this.tally = tally;
+        tally.add(atomicState.get());
     }
 
-    public RunState getSlotState() {
-        return slotState.get();
+    public RunState get() {
+        return atomicState.get();
     }
 
     /**
@@ -46,7 +51,7 @@ public class SlotStateTracker {
      * @return an atomic reference for SlotState
      */
     public AtomicReference<RunState> getAtomicSlotState() {
-        return slotState;
+        return atomicState;
     }
 
     /**
@@ -56,16 +61,19 @@ public class SlotStateTracker {
      * @param to The next SlotState for this thread/slot/motor
      */
     public synchronized void enterState(RunState to) {
-        RunState from = slotState.get();
+        RunState from = atomicState.get();
         if (!from.canTransitionTo(to)) {
             throw new RuntimeException("Invalid transition from " + from + " to " + to);
         }
-        while (!slotState.compareAndSet(from, to)) {
+        while (!atomicState.compareAndSet(from, to)) {
             logger.trace("retrying transition from:" + from + " to:" + to);
         }
+        tally.change(from,to);
         logger.trace("TRANSITION[" + slotId + "]: " + from + " ==> " + to);
-
     }
 
-
+    public void removeState() {
+        logger.trace(() -> "Removing motor state " + atomicState.get());
+        tally.remove(atomicState.get());
+    }
 }

engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java

@@ -40,6 +40,7 @@ import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec;
 import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
 import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
 import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
+import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
 import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
 import io.nosqlbench.engine.api.activityimpl.uniform.DryRunOpDispenserWrapper;
 import io.nosqlbench.engine.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider;
@@ -83,6 +84,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
     private NBErrorHandler errorHandler;
     private ActivityMetricProgressMeter progressMeter;
     private String workloadSource = "unspecified";
+    private final RunStateTally tally = new RunStateTally();
 
     public SimpleActivity(ActivityDef activityDef) {
         this.activityDef = activityDef;
@@ -96,7 +98,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
             } else {
                 activityDef.getParams().set("alias",
                     activityDef.getActivityType().toUpperCase(Locale.ROOT)
-                        + String.valueOf(nameEnumerator++));
+                        + nameEnumerator++);
             }
         }
     }
@@ -191,7 +193,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
     }
 
     public String toString() {
-        return getAlias();
+        return getAlias()+":"+getRunState()+":"+getRunStateTally().toString();
     }
 
     @Override
@@ -218,7 +220,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
     @Override
     public void closeAutoCloseables() {
         for (AutoCloseable closeable : closeables) {
-            logger.debug("CLOSING " + closeable.getClass().getCanonicalName() + ": " + closeable.toString());
+            logger.debug("CLOSING " + closeable.getClass().getCanonicalName() + ": " + closeable);
             try {
                 closeable.close();
             } catch (Exception e) {
@@ -393,7 +395,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
         if (threadSpec.isPresent()) {
             String spec = threadSpec.get();
             int processors = Runtime.getRuntime().availableProcessors();
-            if (spec.toLowerCase().equals("auto")) {
+            if (spec.equalsIgnoreCase("auto")) {
                 int threads = processors * 10;
                 if (threads > activityDef.getCycleCount()) {
                     threads = (int) activityDef.getCycleCount();
@@ -665,7 +667,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
             return stmtsDocList;
 
         } catch (Exception e) {
-            throw new OpConfigError("Error loading op templates: " + e.toString(), workloadSource, e);
+            throw new OpConfigError("Error loading op templates: " + e, workloadSource, e);
         }
 
     }
@@ -690,6 +692,11 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
         return getActivityDef().getParams().getOptionalInteger("maxtries").orElse(10);
     }
 
+    @Override
+    public RunStateTally getRunStateTally() {
+        return tally;
+    }
+
 
     @Override
     public String getName() {

engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java

@@ -29,12 +29,11 @@ import io.nosqlbench.engine.api.activityapi.input.Input;
 import io.nosqlbench.engine.api.activityapi.output.Output;
 import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
 import io.nosqlbench.api.engine.activityimpl.ActivityDef;
-import io.nosqlbench.engine.api.activityimpl.SlotStateTracker;
+import io.nosqlbench.engine.api.activityimpl.MotorState;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
 
 import static io.nosqlbench.engine.api.activityapi.core.RunState.*;
 
@@ -70,8 +69,8 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
     private final Activity activity;
     private Output output;
 
-    private final SlotStateTracker slotStateTracker;
-    private final AtomicReference<RunState> slotState;
+    private final MotorState motorState;
+    //    private final AtomicReference<RunState> slotState;
     private int stride = 1;
 
     private OpTracker<D> opTracker;
@@ -86,14 +85,13 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
      * @param input    A LongSupplier which provides the cycle number inputs.
      */
     public CoreMotor(
-            Activity activity,
-            long slotId,
-            Input input) {
+        Activity activity,
+        long slotId,
+        Input input) {
         this.activity = activity;
         this.slotId = slotId;
         setInput(input);
-        slotStateTracker = new SlotStateTracker(slotId);
-        slotState = slotStateTracker.getAtomicSlotState();
+        motorState = new MotorState(slotId, activity.getRunStateTally());
         onActivityDefUpdate(activity.getActivityDef());
     }
 
@@ -107,10 +105,10 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
      * @param action   An LongConsumer which is applied to the input for each cycle.
      */
     public CoreMotor(
-            Activity activity,
-            long slotId,
-            Input input,
-            Action action
+        Activity activity,
+        long slotId,
+        Input input,
+        Action action
     ) {
         this(activity, slotId, input);
         setAction(action);
@@ -126,11 +124,11 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
      * @param output   An optional opTracker.
      */
     public CoreMotor(
-            Activity activity,
-            long slotId,
-            Input input,
-            Action action,
-            Output output
+        Activity activity,
+        long slotId,
+        Input input,
+        Action action,
+        Output output
     ) {
         this(activity, slotId, input);
         setAction(action);
@@ -178,12 +176,18 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
     }
 
     @Override
-    public SlotStateTracker getSlotStateTracker() {
-        return slotStateTracker;
+    public MotorState getState() {
+        return motorState;
+    }
+
+    @Override
+    public void removeState() {
+        motorState.removeState();
     }
 
     @Override
     public void run() {
+        motorState.enterState(Starting);
 
         try {
             inputTimer = activity.getInstrumentation().getOrCreateInputTimer();
@@ -195,12 +199,10 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
             cycleRateLimiter = activity.getCycleLimiter();
 
 
-            if (slotState.get() == Finished) {
+            if (motorState.get() == Finished) {
                 logger.warn("Input was already exhausted for slot " + slotId + ", remaining in finished state.");
             }
 
-            slotStateTracker.enterState(Running);
-
             long cyclenum;
             action.init();
 
@@ -235,7 +237,8 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
                     strideconsumer = (StrideOutputConsumer<D>) async;
                 }
 
-                while (slotState.get() == Running) {
+                motorState.enterState(Running);
+                while (motorState.get() == Running) {
 
                     CycleSegment cycleSegment = null;
 
@@ -245,7 +248,7 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
 
                     if (cycleSegment == null) {
                         logger.trace("input exhausted (input " + input + ") via null segment, stopping motor thread " + slotId);
-                        slotStateTracker.enterState(Finished);
+                        motorState.enterState(Finished);
                         continue;
                     }
 
@@ -256,27 +259,27 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
 
                     StrideTracker<D> strideTracker = new StrideTracker<>(
                         strideServiceTimer,
-                            stridesResponseTimer,
-                            strideDelay,
-                            cycleSegment.peekNextCycle(),
-                            stride,
-                            output,
-                            strideconsumer);
+                        stridesResponseTimer,
+                        strideDelay,
+                        cycleSegment.peekNextCycle(),
+                        stride,
+                        output,
+                        strideconsumer);
                     strideTracker.start();
 
                     long strideStart = System.nanoTime();
 
-                    while (!cycleSegment.isExhausted() && slotState.get() == Running) {
+                    while (!cycleSegment.isExhausted() && motorState.get() == Running) {
                         cyclenum = cycleSegment.nextCycle();
                         if (cyclenum < 0) {
                             if (cycleSegment.isExhausted()) {
                                 logger.trace("input exhausted (input " + input + ") via negative read, stopping motor thread " + slotId);
-                                slotStateTracker.enterState(Finished);
+                                motorState.enterState(Finished);
                                 continue;
                             }
                         }
 
-                        if (slotState.get() != Running) {
+                        if (motorState.get() != Running) {
                             logger.trace("motor stopped in cycle " + cyclenum + ", stopping motor thread " + slotId);
                             continue;
                         }
@@ -287,7 +290,7 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
                         }
 
                         try {
-                            TrackedOp<D> op = opTracker.newOp(cyclenum,strideTracker);
+                            TrackedOp<D> op = opTracker.newOp(cyclenum, strideTracker);
                             op.setWaitTime(cycleDelay);
 
                             synchronized (opTracker) {
@@ -312,7 +315,7 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
 
                 }
 
-                if (slotState.get() == Finished) {
+                if (motorState.get() == Finished) {
                     boolean finished = opTracker.awaitCompletion(60000);
                     if (finished) {
                         logger.debug("slot " + this.slotId + " completed successfully");
@@ -321,12 +324,12 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
                     }
                 }
 
-                if (slotState.get() == Stopping) {
-                    slotStateTracker.enterState(Stopped);
+                if (motorState.get() == Stopping) {
+                    motorState.enterState(Stopped);
                 }
 
 
-            } else if (action instanceof SyncAction) {
+            } else if (action instanceof SyncAction sync) {
 
                 cycleServiceTimer = activity.getInstrumentation().getOrCreateCyclesServiceTimer();
                 strideServiceTimer = activity.getInstrumentation().getOrCreateStridesServiceTimer();
@@ -335,9 +338,8 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
                     throw new RuntimeException("The async parameter was given for this activity, but it does not seem to know how to do async.");
                 }
 
-                SyncAction sync = (SyncAction) action;
-
-                while (slotState.get() == Running) {
+                motorState.enterState(Running);
+                while (motorState.get() == Running) {
 
                     CycleSegment cycleSegment = null;
                     CycleResultSegmentBuffer segBuffer = new CycleResultSegmentBuffer(stride);
@@ -348,7 +350,7 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
 
                     if (cycleSegment == null) {
                         logger.trace("input exhausted (input " + input + ") via null segment, stopping motor thread " + slotId);
-                        slotStateTracker.enterState(Finished);
+                        motorState.enterState(Finished);
                         continue;
                     }
 
@@ -366,12 +368,12 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
                             if (cyclenum < 0) {
                                 if (cycleSegment.isExhausted()) {
                                     logger.trace("input exhausted (input " + input + ") via negative read, stopping motor thread " + slotId);
-                                    slotStateTracker.enterState(Finished);
+                                    motorState.enterState(Finished);
                                     continue;
                                 }
                             }
 
-                            if (slotState.get() != Running) {
+                            if (motorState.get() != Running) {
                                 logger.trace("motor stopped after input (input " + cyclenum + "), stopping motor thread " + slotId);
                                 continue;
                             }
@@ -391,6 +393,9 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
                                 result = sync.runCycle(cyclenum);
                                 long phaseEnd = System.nanoTime();
 
+                            } catch (Exception e) {
+                                motorState.enterState(Errored);
+                                throw e;
                             } finally {
                                 long cycleEnd = System.nanoTime();
                                 cycleServiceTimer.update((cycleEnd - cycleStart) + cycleDelay, TimeUnit.NANOSECONDS);
@@ -414,25 +419,29 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
                     }
                 }
 
-                if (slotState.get() == Stopping) {
-                    slotStateTracker.enterState(Stopped);
-                }
-
             } else {
                 throw new RuntimeException("Valid Action implementations must implement either the SyncAction or the AsyncAction sub-interface");
             }
 
+            if (motorState.get() == Stopping) {
+                motorState.enterState(Stopped);
+                logger.trace(() -> Thread.currentThread().getName() + " shutting down as " + motorState.get());
+            } else if (motorState.get() == Finished) {
+                logger.trace(() -> Thread.currentThread().getName() + " shutting down as " + motorState.get());
+            } else {
+                logger.warn("Unexpected motor state for CoreMotor shutdown: " + motorState.get());
+            }
 
         } catch (Throwable t) {
             logger.error("Error in core motor loop:" + t, t);
+            motorState.enterState(Errored);
             throw t;
         }
     }
 
-
     @Override
     public String toString() {
-        return "slot:" + this.slotId + "; state:" + slotState.get();
+        return "slot:" + this.slotId + "; state:" + motorState.get();
     }
 
     @Override
@@ -452,17 +461,17 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
 
     @Override
     public synchronized void requestStop() {
-        if (slotState.get() == Running) {
+        if (motorState.get() == Running) {
             if (input instanceof Stoppable) {
                 ((Stoppable) input).requestStop();
             }
             if (action instanceof Stoppable) {
                 ((Stoppable) action).requestStop();
             }
-            slotStateTracker.enterState(RunState.Stopping);
+            motorState.enterState(RunState.Stopping);
         } else {
-            if (slotState.get() != Stopped && slotState.get() != Stopping) {
-                logger.warn("attempted to stop motor " + this.getSlotId() + ": from non Running state:" + slotState.get());
+            if (motorState.get() != Stopped && motorState.get() != Stopping) {
+                logger.warn("attempted to stop motor " + this.getSlotId() + ": from non Running state:" + motorState.get());
             }
         }
     }

engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateImage.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.api.activityimpl.motor;
+
+import io.nosqlbench.engine.api.activityapi.core.RunState;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * A value type which encodes the atomic state of a RunState tally.
+ */
+public class RunStateImage {
+    private final static Logger logger = LogManager.getLogger("TALLY");
+
+    private final int[] counts = new int[RunState.values().length];
+    private final boolean timedout;
+
+    public RunStateImage(int[] counts, boolean timedout) {
+        System.arraycopy(counts, 0, this.counts, 0, counts.length);
+        this.timedout = timedout;
+    }
+
+    public boolean isTimeout() {
+        return this.timedout;
+    }
+
+    public boolean is(RunState runState) {
+        return counts[runState.ordinal()]>0;
+    }
+
+    public boolean isOnly(RunState runState) {
+        for (int i = 0; i < counts.length; i++) {
+            if (counts[i]>0 && i!=runState.ordinal()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public RunState getMaxState() {
+        for (int ord = counts.length-1; ord >= 0; ord--) {
+            if (counts[ord]>0) {
+                return RunState.values()[ord];
+            }
+        }
+        throw new RuntimeException("There were zero states, so max state is undefined");
+    }
+
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        for (RunState runState : RunState.values()) {
+            sb.append(runState.getCode()).append(" ").append(counts[runState.ordinal()]).append("  ");
+        }
+        return sb.toString();
+    }
+
+}

engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateTally.java

@@ -0,0 +1,266 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.api.activityimpl.motor;
+
+import io.nosqlbench.engine.api.activityapi.core.RunState;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Arrays;
+
+/**
+ * <H2>Synopsis</H2>
+ * <P>Event-oriented tally of the runtime states of all
+ * motor threads. This is meant to be used as the definitive
+ * aggregate scorecard for an activity's thread states.</P>
+ * <HR></HR>
+ * <H2>Purpose</H2>
+ * <P>This limits inter-thread signaling requirements by constraining
+ * the cases for which blockers are notified. This is because the
+ * meaningful scenarios for which a blocker would want to be notified
+ * do not include changes of positive values to other positive values.
+ * This allows an explicit optimization around scenarios for which a
+ * state count increments to 1 or decrements to 0, as in "some" or "none".
+ * This is an effective optimization for scenarios with many active
+ * threads.
+ * </P>
+ * <HR></HR>
+ * <H2>Calling Semantics</H2>
+ * <P>Callers of the await functions will block for the required condition or,
+ * if specified, the timeout to occur without the condition.
+ * These callers are unblocked atomically, after any state add, state remove,
+ * or state change events are fully completed.</P>
+ * <P>{@link RunStateImage} is returned from blocking methods so that
+ * callers can know consistently what the current run states were at
+ * the time their condition was met or timed out. Any callers of such
+ * methods <EM>must</EM> check for whether the condition was successfully
+ * met via {@link RunStateImage#isTimeout()}</P>
+ * <HR></HR>
+ * <H2>Invariants</H2>
+ * <UL>
+ *     <LI>Under consistent usage patterns, all counts should be zero or positive at all times.</LI>
+ *     <LI>The current count for each state should be equal to the visible {@link RunState} on each
+ *     motor thread, as all transitions are made through the motor threads directly.</LI>
+ * </UL>
+ */
+public class RunStateTally {
+    private final static Logger logger = LogManager.getLogger("TALLY");
+
+    /**
+     * If no timeout is given for any of the await methods, then the default is to wait for
+     * approximately many eons. Some tests run until told to stop.
+     */
+    public final long DEFAULT_TIMEOUT_MS=Long.MAX_VALUE;
+
+    private final int[] counts = new int[RunState.values().length];
+
+    /**
+     * @return the current count for the specified state
+     * @param runState The {@link RunState} to count
+     *
+     */
+    public synchronized int tallyFor(RunState runState) {
+        return counts[runState.ordinal()];
+    }
+
+    /**
+     * Signal that a motor thread has left one {@link RunState} and entered another,
+     * atomically. After both counts are updated, if any RunState counts changed to or from zero,
+     * then all potential observers are notified to re-evaluate their await conditions.
+     * @param from the prior RunState
+     * @param to the next RunState
+     */
+    public synchronized void change(RunState from, RunState to) {
+        counts[from.ordinal()]--;
+        counts[to.ordinal()]++;
+        logger.trace(() -> this +" -"+from+ String.format(":%04d",counts[from.ordinal()])+ ", +"+to+ String.format(":%04d",counts[to.ordinal()]));
+
+        if (counts[from.ordinal()]==0 || counts[to.ordinal()]==1) {
+            logger.debug(() -> "NOTIFYing on edge "+
+                "from " + from + String.format(":%04d",counts[from.ordinal()]) +
+                " to " + to + String.format(":%04d",counts[to.ordinal()]));
+            notifyAll();
+        }
+    }
+
+    /**
+     * Add a previously untracked motor thread to state tracking with the provided {@link RunState}.
+     * If the given state's count becomes non-zero, then all potential observers are notified to re-evaluate
+     * their await conditions.
+     * @param state The initial tracking state for the related motor thread
+     */
+    public synchronized void add(RunState state) {
+        counts[state.ordinal()]++;
+        logger.trace(() -> this +" +"+state+ String.format(":%04d",counts[state.ordinal()]));
+        if (counts[state.ordinal()]==1) {
+            logger.debug(() -> "NOTIFYing on ++-SOME edge for " + state + String.format(":%04d",counts[state.ordinal()]));
+            notifyAll();
+        }
+    }
+
+    /**
+     * Remove a previously tracked motor thread from state tracking with the provided {@link RunState}.
+     * If the given state's count becomes zero, then all potential observers are notified to re-evaluate
+     * their await conditions.
+     * @param state The final tracking state for the related motor thread
+     */
+    public synchronized void remove(RunState state) {
+        counts[state.ordinal()]--;
+        logger.trace(() -> this +" -"+state+ String.format(":%04d",counts[state.ordinal()]));
+        if (counts[state.ordinal()]==0) {
+            logger.debug(() -> "NOTIFYing on 00-NONE edge for " + state + String.format(":%04d",counts[state.ordinal()]));
+            notifyAll();
+        }
+    }
+
+    /**
+     * Await until all states but the provided {@link RunState}s have zero counts.
+     * This condition matches if the provided states have zero or positive counts <EM>if and only if</EM>
+     * all other states have zero counts. Thus, if there are no positive values at all, this condition will
+     * still match.
+     * @param runStates The states which <EM>may</EM> have zero counts and still match the condition
+     * @return A {@link RunStateImage}, indicating success or failure, and the view of states at the time of evaluation
+     */
+    public synchronized RunStateImage awaitNoneOther(RunState... runStates) {
+        return this.awaitNoneOther(DEFAULT_TIMEOUT_MS, runStates);
+    }
+    /**
+     * Exactly like {@link #awaitNoneOther(long, RunState...)}, except that it allows for a timeout,
+     * after which the method will unblock and signal an error if the await condition is still false.
+     * @param runStates RunStates which are the only valid states before unblocking
+     * @return A {@link RunStateImage}, indicating success or failure, and the view of states at the time of evaluation
+     */
+    public synchronized RunStateImage awaitNoneOther(long timeoutMillis, RunState... runStates) {
+        logger.debug(() -> "☐ Awaiting only " + Arrays.toString(runStates) + " for " + timeoutMillis+"ms");
+        long timeoutAt = timeoutAt(timeoutMillis);
+
+        int sum=0;
+        for (RunState runState: RunState.values()) {
+            sum+=counts[runState.ordinal()];
+        }
+        for (RunState runState : runStates) {
+            sum-=counts[runState.ordinal()];
+        }
+        while (sum>0 && System.currentTimeMillis()<timeoutAt) {
+            try {
+                wait(timeoutAt-System.currentTimeMillis());
+            } catch (InterruptedException e) {
+            }
+
+            sum=0;
+            for (RunState runState: RunState.values()) {
+                sum+=counts[runState.ordinal()];
+            }
+            for (RunState runState : runStates) {
+                sum-=counts[runState.ordinal()];
+            }
+        }
+
+        boolean timedout = (sum!=0);
+        logger.debug(() -> (timedout ? "✘ TIMED-OUT awaiting only " : "☑ Awaited only " ) + toString(runStates));
+        return new RunStateImage(this.counts,timedout);
+    }
+
+    private long timeoutAt(long timeoutMillis) {
+        long delayTill= System.currentTimeMillis() + timeoutMillis;
+        return (delayTill>0) ? delayTill : Long.MAX_VALUE;
+    }
+    /**
+     * Await until there are zero counts for all of the specified {@link RunState}s.
+     * @param runStates all RunStates which must be zeroed before unblocking
+     * @return A {@link RunStateImage}, indicating success or failure, and the view of states at the time of evaluation
+     */
+    public synchronized RunStateImage awaitNoneOf(RunState... runStates) {
+        return this.awaitNoneOf(DEFAULT_TIMEOUT_MS, runStates);
+    }
+    /**
+     * Exactly like {@link #awaitNoneOf(RunState...)}, except that it allows for a timeout, after which the
+     * method will unblock and signal a timeout if the await condition is still not met.
+     * @param runStates all RunStates which must be zeroed before unblocking
+     * @return A {@link RunStateImage}, indicating success or failure, and the view of states at the time of evaluation
+     */
+    public synchronized RunStateImage awaitNoneOf(long timeoutMillis, RunState... runStates) {
+        logger.debug(() -> "☐ Awaiting none of " + Arrays.toString(runStates)+ " for " + timeoutMillis+"ms");
+        long timeoutAt = timeoutAt(timeoutMillis);
+
+        int sum=0;
+        for (RunState runState : runStates) {
+            sum+=counts[runState.ordinal()];
+        }
+        while (sum>0 && System.currentTimeMillis()<timeoutAt) {
+            try {
+                wait(timeoutAt-System.currentTimeMillis());
+            } catch (InterruptedException e) {
+            }
+            sum=0;
+            for (RunState runState : runStates) {
+                sum+=counts[runState.ordinal()];
+            }
+        }
+        boolean timedout=sum==0;
+        logger.debug(() -> (timedout ? "✘ TIMED-OUT awaiting none of " : "☑ Awaited none of " ) + toString(runStates));
+        return new RunStateImage(this.counts,timedout);
+    }
+
+    /**
+     * Await until at least one of the provided runStates has a positive value.
+     * @param runStates RunStates any of which allow unblocking
+     * @return A {@link RunStateImage}, indicating success or failure, and the view of states at the time of evaluation
+     */
+    public synchronized RunStateImage awaitAny(RunState... runStates) {
+        return this.awaitAny(DEFAULT_TIMEOUT_MS,runStates);
+    }
+
+    /**
+     * Exactly like {@link #awaitAny(RunState...)}, except that it allows for a timeout, after which
+     * the method will unblock and signal a timeout if the await condition is still not met.
+     * @param runStates RunStates any of which allow unblocking
+     * @param timeoutMillis Milliseconds to wait for any of the runstates before giving up
+     * @return A {@link RunStateImage}, indicating success or failure, and the view of states at the time of evaluation
+     */
+    public synchronized RunStateImage awaitAny(long timeoutMillis, RunState... runStates) {
+        logger.debug(() -> "☐ Awaiting any " + Arrays.toString(runStates) + " for " + timeoutMillis + "ms");
+        long timeoutAt = timeoutAt(timeoutMillis);
+
+        while (System.currentTimeMillis()<timeoutAt) {
+            for (RunState runState : runStates) {
+                if (counts[runState.ordinal()]>0) {
+                    logger.debug("☑ Awaited any " + toString(runStates));
+                    return new RunStateImage(this.counts,false);
+                }
+            }
+            try {
+                wait(timeoutAt-System.currentTimeMillis());
+            } catch (InterruptedException e) {
+            }
+        }
+        logger.debug(() -> "✘ TIMED-OUT awaiting any of " + toString(runStates));
+        return new RunStateImage(this.counts,true);
+    }
+
+    public String toString(RunState... runStates) {
+        StringBuilder sb = new StringBuilder();
+        for (RunState runState : runStates) {
+            sb.append(runState.getCode()).append("(").append(counts[runState.ordinal()]).append(") ");
+        }
+        sb.setLength(sb.length()-1);
+        return sb.toString();
+    }
+    public String toString() {
+        return toString(RunState.values());
+    }
+}

engine-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateImageTest.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.api.activityimpl.motor;
+
+import io.nosqlbench.engine.api.activityapi.core.RunState;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class RunStateImageTest {
+
+    @Test
+    public void testMaxStateImage() {
+        int[] counts = new int[RunState.values().length];
+        counts[RunState.Running.ordinal()]=3;
+        RunStateImage image = new RunStateImage(counts, false);
+        assertThat(image.is(RunState.Running)).isTrue();
+        assertThat(image.isTimeout()).isFalse();
+        assertThat(image.is(RunState.Errored)).isFalse();
+        assertThat(image.isOnly(RunState.Running)).isTrue();
+        RunState maxState = image.getMaxState();
+        assertThat(maxState).isEqualTo(RunState.values()[2]);
+    }
+
+}

engine-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateTallyTest.java

@@ -0,0 +1,214 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.api.activityimpl.motor;
+
+import io.nosqlbench.engine.api.activityapi.core.RunState;
+import org.junit.jupiter.api.*;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+public class RunStateTallyTest {
+
+    volatile boolean awaited = false;
+    volatile RunStateImage event = null;
+
+    @BeforeEach
+    public void setup() {
+        awaited = false;
+        event = null;
+    }
+
+    @Test
+    @Order(1)
+    public void testAwaitAny() {
+        Thread.currentThread().setName("SETTER");
+
+        RunStateTally tally = new RunStateTally();
+        awaited = false;
+        Thread waiter = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                event = tally.awaitAny(RunState.Running);
+                awaited = true;
+            }
+        });
+        waiter.setName("WAITER");
+        waiter.setDaemon(true);
+        waiter.start();
+
+        try {
+            Thread.sleep(100);
+        } catch (Exception e) {
+        }
+
+        assertThat(awaited).isFalse();
+        tally.add(RunState.Running);
+
+        try {
+            Thread.sleep(100);
+        } catch (Exception e) {
+        }
+
+
+        assertThat(event.is(RunState.Running)).isTrue();
+        assertThat(event.isOnly(RunState.Running)).isTrue();
+
+        assertThat(awaited).isTrue();
+        assertThat(waiter.getState()).isNotEqualTo(Thread.State.RUNNABLE);
+    }
+
+    @Test
+    @Order(2)
+    public void testAwaitNoneOf() {
+        Thread.currentThread().setName("SETTER");
+
+        RunStateTally tally = new RunStateTally();
+        tally.add(RunState.Uninitialized);
+        tally.add(RunState.Stopped);
+        awaited = false;
+        Thread waiter = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                tally.awaitNoneOf(RunState.Stopped, RunState.Uninitialized);
+                awaited = true;
+            }
+        });
+        waiter.setName("WAITER");
+        waiter.setDaemon(true);
+        waiter.start();
+
+        try {
+            Thread.sleep(100);
+        } catch (Exception e) {
+        }
+
+        assertThat(awaited).isFalse();
+        tally.change(RunState.Stopped, RunState.Finished);
+
+        try {
+            Thread.sleep(100);
+        } catch (Exception e) {
+        }
+
+        assertThat(awaited).isFalse();
+        tally.change(RunState.Uninitialized, RunState.Finished);
+
+        try {
+            Thread.sleep(100);
+        } catch (Exception e) {
+        }
+
+        assertThat(awaited).isTrue();
+        assertThat(waiter.getState()).isNotEqualTo(Thread.State.RUNNABLE);
+
+    }
+
+    @Test
+    @Order(3)
+    public void testAwaitNoneOther() {
+        Thread.currentThread().setName("SETTER");
+
+        RunStateTally tally = new RunStateTally();
+        tally.add(RunState.Uninitialized);
+        tally.add(RunState.Running);
+        awaited = false;
+        Thread waiter = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                event = tally.awaitNoneOther(RunState.Stopped, RunState.Finished);
+                awaited = true;
+            }
+        });
+        waiter.setName("WAITER");
+        waiter.setDaemon(true);
+        waiter.start();
+
+        try {
+            Thread.sleep(100);
+        } catch (Exception e) {
+        }
+
+        assertThat(awaited).isFalse();
+        tally.change(RunState.Uninitialized, RunState.Finished);
+
+        try {
+            Thread.sleep(100);
+        } catch (Exception e) {
+        }
+
+        assertThat(awaited).isFalse();
+
+        // Note that neither Stopped or Finished are required to be positive,
+        // as long as all others are zero total.
+        tally.remove(RunState.Running);
+
+        try {
+            Thread.sleep(100);
+        } catch (Exception e) {
+        }
+
+        assertThat(awaited).isTrue();
+        assertThat(waiter.getState()).isNotEqualTo(Thread.State.RUNNABLE);
+
+    }
+
+    @Test
+    @Order(4)
+    public void testAwaitNoneOtherTimedOut() {
+        Thread.currentThread().setName("SETTER");
+
+        RunStateTally tally = new RunStateTally();
+        tally.add(RunState.Uninitialized);
+        tally.add(RunState.Running);
+        Thread waiter = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                event = tally.awaitNoneOther(1500, RunState.Stopped, RunState.Finished);
+                awaited = true;
+            }
+        });
+        waiter.setName("WAITER");
+        waiter.setDaemon(true);
+        waiter.start();
+
+        try {
+            Thread.sleep(100);
+        } catch (Exception e) {
+        }
+
+        assertThat(awaited).isFalse();
+        tally.change(RunState.Uninitialized, RunState.Finished);
+
+        try {
+            Thread.sleep(1500);
+        } catch (Exception e) {
+        }
+
+//        try {
+//            waiter.join();
+//        } catch (InterruptedException e) {
+//            throw new RuntimeException(e);
+//        }
+
+        assertThat(event.isOnly(RunState.Errored)).isFalse();
+        assertThat(waiter.getState()).isNotEqualTo(Thread.State.RUNNABLE);
+
+    }
+
+
+}

engine-api/src/test/java/io/nosqlbench/engine/api/activityimpl/tracking/LongTreeTrackerTest2.java

@@ -18,12 +18,15 @@ package io.nosqlbench.engine.api.activityimpl.tracking;
 
 import io.nosqlbench.engine.api.activityimpl.marker.longheap.LongTreeTracker;
 import io.nosqlbench.engine.api.activityimpl.marker.longheap.LongTreeTrackerAtomic;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class LongTreeTrackerTest2 {
+    private final static Logger logger = LogManager.getLogger(LongTreeTrackerTest2.class);
 
 //    @Test
 //    public void testCoMask() {
@@ -96,21 +99,21 @@ public class LongTreeTrackerTest2 {
     public void testApply() {
         LongTreeTracker t = new LongTreeTracker(0L);
         t.setCompleted(0);
-        System.out.println(t);
+        logger.debug(t);
         t.setCompleted(1);
-        System.out.println(t);
+        logger.debug(t);
         t.setCompleted(2);
-        System.out.println(t);
+        logger.debug(t);
         t.setCompleted(5);
-        System.out.println(t);
+        logger.debug(t);
         t.setCompleted(6);
-        System.out.println(t);
+        logger.debug(t);
         t.setCompleted(3);
-        System.out.println(t);
+        logger.debug(t);
         t.setCompleted(4);
-        System.out.println(t);
+        logger.debug(t);
         t.setCompleted(7);
-        System.out.println(t);
+        logger.debug(t);
     }
 
     @Test
@@ -119,7 +122,7 @@ public class LongTreeTrackerTest2 {
         for (int i = 0; i < 32 ; i++) {
             t.setCompleted(i);
         }
-        System.out.println(t);
+        logger.debug(t);
         assertThat(t.getImage()).isEqualTo(-2L);
     }
 
@@ -141,7 +144,7 @@ public class LongTreeTrackerTest2 {
     @Test
     public void testBitString() {
         LongTreeTracker t = new LongTreeTracker(2L);
-        System.out.println(t);
+        logger.debug(t);
     }
 
     /**

engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java

@@ -21,6 +21,8 @@ import com.google.gson.GsonBuilder;
 import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
 import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
 import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.Test;
 
 import java.util.Map;
@@ -28,6 +30,7 @@ import java.util.Map;
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class CommandTemplateTest {
+    private final static Logger logger = LogManager.getLogger(CommandTemplateTest.class);
 
     @Test
     public void testCommandTemplate() {
@@ -53,7 +56,7 @@ public class CommandTemplateTest {
         OpTemplate optpl = stmtsDocs.getStmts().get(0);
         CommandTemplate ct = new CommandTemplate(optpl);
         String format = gson.toJson(ct);
-        System.out.println(format);
+        logger.debug(format);
 
     }
 

engine-api/src/test/java/io/nosqlbench/nb/AggregateTests.java

@@ -18,6 +18,8 @@ package io.nosqlbench.nb;
 
 import org.HdrHistogram.DoubleHistogram;
 import org.HdrHistogram.DoubleRecorder;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.Test;
 
 import java.util.DoubleSummaryStatistics;
@@ -31,6 +33,7 @@ import static org.assertj.core.api.Assertions.assertThat;
  * a proof, so we can get on with testing!
  */
 public class AggregateTests {
+    private final static Logger logger = LogManager.getLogger(AggregateTests.class);
     double[][] data = new double[][]{
         {1, 1, 1, 1, 1, 1, 1, 1, 1, 91},
         {15, 15, 15, 15, 15, 5, 5, 5, 5, 5},
@@ -62,8 +65,8 @@ public class AggregateTests {
             aggstats.accept(series.getAverage());
         }
 
-        System.out.println("aggstats avg:" + aggstats.getAverage());
-        System.out.println("allstats avg:" + allstats.getAverage());
+        logger.debug("aggstats avg:" + aggstats.getAverage());
+        logger.debug("allstats avg:" + allstats.getAverage());
 
         assertThat(aggstats.getAverage()).isNotEqualTo(allstats.getAverage());
     }
@@ -92,21 +95,21 @@ public class AggregateTests {
                 all.recordValue(v);
             }
             snapshots[i]=recorder.getIntervalHistogram();
-            System.out.println(snapshot(snapshots[i],"ary[" + i + "]"));
+            logger.debug(snapshot(snapshots[i],"ary[" + i + "]"));
         }
 
         DoubleHistogram histoall = all.getIntervalHistogram();
-        System.out.println(snapshot(histoall, "all"));
+        logger.debug(snapshot(histoall, "all"));
 
         for (double pctile : new double[]{10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 99, 99.9, 99.99}) {
             DoubleSummaryStatistics avgOfInputs = new DoubleSummaryStatistics();
             for (DoubleHistogram snapshot : snapshots) {
                 avgOfInputs.accept(snapshot.getValueAtPercentile(pctile));
             }
-            System.out.println("avg of " + pctile + " => " + String.format("%.3f",avgOfInputs.getAverage()) + " (min,max)=("+String.format("%.3f",avgOfInputs.getMin()) + "," +
+            logger.debug("avg of " + pctile + " => " + String.format("%.3f",avgOfInputs.getAverage()) + " (min,max)=("+String.format("%.3f",avgOfInputs.getMin()) + "," +
                 String.format("%.3f",avgOfInputs.getMax())+ ")");
-            System.out.println("direct " + pctile + " => " + String.format("%.3f",histoall.getValueAtPercentile(pctile)));
-            System.out.println();
+            logger.debug("direct " + pctile + " => " + String.format("%.3f",histoall.getValueAtPercentile(pctile)));
+
 
         }
 
@@ -128,11 +131,10 @@ public class AggregateTests {
                 prototype[j]=r.nextDouble()*100;
             }
 
-            System.out.print("proto[" + i + "] = ");
+            logger.debug("proto[" + i + "] = ");
             for (double v : prototype) {
-                System.out.print(String.format("% 3.0f ",v));
+                logger.trace(String.format("% 3.0f ",v));
             }
-            System.out.println();
             series[i]=resampleCurve(prototype,100);
         }
 

engine-api/src/test/resources/log4j2-test.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<Configuration status="${project.test_logstatuslevel}" strict="true" name="XMLConfigTest"
+               packages="org.apache.logging.log4j.test">
+
+  <Filter type="ThresholdFilter" level="trace"/>
+
+  <Appenders>
+    <Appender type="Console" name="STDOUT">
+      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
+      </Filters>
+    </Appender>
+    <Appender type="Console" name="FLOW">
+      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
+      </Filters>
+    </Appender>
+    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
+      <Layout type="PatternLayout">
+        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
+      </Layout>
+    </Appender>
+  </Appenders>
+
+  <Loggers>
+
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
+      <AppenderRef ref="APPSLOG"/>
+    </Logger>
+
+    <Root level="${project.testlevel}">
+      <AppenderRef ref="STDOUT"/>
+    </Root>
+
+  </Loggers>
+
+</Configuration>

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -32,14 +32,17 @@ import io.nosqlbench.engine.api.activityapi.input.InputType;
 import io.nosqlbench.engine.api.activityapi.output.OutputType;
 import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
 import io.nosqlbench.engine.core.annotation.Annotators;
-import io.nosqlbench.engine.core.lifecycle.*;
+import io.nosqlbench.engine.core.lifecycle.process.NBCLIErrorHandler;
+import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader;
+import io.nosqlbench.engine.core.lifecycle.process.ShutdownManager;
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosResults;
 import io.nosqlbench.engine.core.logging.LoggerConfig;
-import io.nosqlbench.engine.core.metadata.MarkdownDocInfo;
+import io.nosqlbench.engine.core.metadata.MarkdownFinder;
 import io.nosqlbench.engine.core.metrics.MetricReporters;
-import io.nosqlbench.engine.core.script.MetricsMapper;
-import io.nosqlbench.engine.core.script.Scenario;
-import io.nosqlbench.engine.core.script.ScenariosExecutor;
-import io.nosqlbench.engine.core.script.ScriptParams;
+import io.nosqlbench.engine.core.lifecycle.scenario.script.MetricsMapper;
+import io.nosqlbench.engine.core.lifecycle.scenario.Scenario;
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosExecutor;
+import io.nosqlbench.engine.core.lifecycle.scenario.script.ScriptParams;
 import io.nosqlbench.engine.docker.DockerMetricsManager;
 import io.nosqlbench.nb.annotations.Maturity;
 import io.nosqlbench.nb.annotations.Service;
@@ -56,7 +59,6 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
-import java.util.concurrent.locks.LockSupport;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -353,7 +355,7 @@ public class NBCLI implements Function<String[], Integer> {
         }
 
         if (options.wantsTopicalHelp()) {
-            Optional<String> helpDoc = MarkdownDocInfo.forHelpTopic(options.wantsTopicalHelpFor());
+            Optional<String> helpDoc = MarkdownFinder.forHelpTopic(options.wantsTopicalHelpFor());
             System.out.println(helpDoc.orElseThrow(
                 () -> new RuntimeException("No help could be found for " + options.wantsTopicalHelpFor())
             ));
@@ -418,7 +420,7 @@ public class NBCLI implements Function<String[], Integer> {
         // intentionally not shown for warn-only
         logger.info("console logging level is " + options.getConsoleLogLevel());
 
-        ScenariosExecutor executor = new ScenariosExecutor("executor-" + sessionName, 1);
+        ScenariosExecutor scenariosExecutor = new ScenariosExecutor("executor-" + sessionName, 1);
         if (options.getConsoleLogLevel().isGreaterOrEqualTo(NBLogLevel.WARN)) {
             options.setWantsStackTraces(true);
             logger.debug("enabling stack traces since log level is " + options.getConsoleLogLevel());
@@ -466,7 +468,7 @@ public class NBCLI implements Function<String[], Integer> {
         scriptParams.putAll(buffer.getCombinedParams());
         scenario.addScenarioScriptParams(scriptParams);
 
-        executor.execute(scenario);
+        scenariosExecutor.execute(scenario);
 
 //        while (true) {
 //            Optional<ScenarioResult> pendingResult = executor.getPendingResult(scenario.getScenarioName());
@@ -476,7 +478,7 @@ public class NBCLI implements Function<String[], Integer> {
 //            LockSupport.parkNanos(100000000L);
 //        }
 
-        ScenariosResults scenariosResults = executor.awaitAllResults();
+        ScenariosResults scenariosResults = scenariosExecutor.awaitAllResults();
         logger.debug("Total of " + scenariosResults.getSize() + " result object returned from ScenariosExecutor");
 
         ActivityMetrics.closeMetrics(options.wantsEnableChart());
@@ -486,7 +488,7 @@ public class NBCLI implements Function<String[], Integer> {
         logger.info(scenariosResults.getExecutionSummary());
 
         if (scenariosResults.hasError()) {
-            Exception exception = scenariosResults.getOne().getException().get();
+            Exception exception = scenariosResults.getOne().getException();
             logger.warn(scenariosResults.getExecutionSummary());
             NBCLIErrorHandler.handle(exception, options.wantsStackTraces());
             System.err.println(exception.getMessage()); // TODO: make this consistent with ConsoleLogging sequencing

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java

@@ -18,7 +18,7 @@ package io.nosqlbench.engine.cli;
 
 import io.nosqlbench.engine.api.metrics.IndicatorMode;
 import io.nosqlbench.api.engine.util.Unit;
-import io.nosqlbench.engine.core.script.Scenario;
+import io.nosqlbench.engine.core.lifecycle.scenario.Scenario;
 import io.nosqlbench.nb.annotations.Maturity;
 import io.nosqlbench.api.system.NBEnvironment;
 import io.nosqlbench.api.errors.BasicError;

engine-cli/src/main/resources/log4j2.xml

@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
 <Configuration status="debug" strict="true" name="XMLConfigTest"
                packages="org.apache.logging.log4j.test">
 
@@ -28,7 +44,7 @@
 
   <Loggers>
 
-    <Logger name="io.nosqlbench.docsys" level="info" additivity="false" specificity="trace">
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
       <AppenderRef ref="APPSLOG"/>
     </Logger>
 
@@ -38,4 +54,4 @@
 
   </Loggers>
 
-</Configuration>
+</Configuration>

engine-cli/src/test/resources/log4j2-test.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<Configuration status="${project.test_logstatuslevel}" strict="true" name="XMLConfigTest"
+               packages="org.apache.logging.log4j.test">
+
+  <Filter type="ThresholdFilter" level="trace"/>
+
+  <Appenders>
+    <Appender type="Console" name="STDOUT">
+      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
+      </Filters>
+    </Appender>
+    <Appender type="Console" name="FLOW">
+      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
+      </Filters>
+    </Appender>
+    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
+      <Layout type="PatternLayout">
+        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
+      </Layout>
+    </Appender>
+  </Appenders>
+
+  <Loggers>
+
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
+      <AppenderRef ref="APPSLOG"/>
+    </Logger>
+
+    <Root level="${project.testlevel}">
+      <AppenderRef ref="STDOUT"/>
+    </Root>
+
+  </Loggers>
+
+</Configuration>

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ActivityFinisher.java

@@ -1,50 +0,0 @@
-/*
- * Copyright (c) 2022 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.nosqlbench.engine.core.lifecycle;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-public class ActivityFinisher extends Thread {
-    private final static Logger logger = LogManager.getLogger(ActivityFinisher.class);
-
-    private final ActivityExecutor executor;
-    private final int timeout;
-    private boolean result;
-
-    public ActivityFinisher(ActivityExecutor executor, int timeout) {
-        super(executor.getActivityDef().getAlias() + "_finisher");
-        this.executor = executor;
-        this.timeout = timeout;
-    }
-
-    @Override
-    public void run() {
-        logger.debug(this + " awaiting async completion of " + executor.getActivity().getAlias() + " on " + executor + " for timeout " + timeout);
-        result = executor.awaitCompletion(timeout);
-        logger.debug(this + " awaited async completion of " + executor.getActivity().getAlias());
-    }
-
-    public boolean getResult() {
-        return result;
-    }
-
-    @Override
-    public String toString() {
-        return this.getClass().getSimpleName()+"/" + executor.getActivity().getAlias();
-    }
-}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ExecutionMetricsResult.java

@@ -20,20 +20,16 @@ import com.codahale.metrics.*;
 import io.nosqlbench.api.engine.metrics.ActivityMetrics;
 import io.nosqlbench.engine.core.logging.Log4JMetricsReporter;
 import io.nosqlbench.engine.core.metrics.NBMetricsSummary;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 import java.nio.charset.StandardCharsets;
 import java.util.HashSet;
-import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
-public class ScenarioResult {
+public class ExecutionMetricsResult extends ExecutionResult {
 
-    private final static Logger logger = LogManager.getLogger(ScenarioResult.class);
     public static final Set<MetricAttribute> INTERVAL_ONLY_METRICS = Set.of(
         MetricAttribute.MIN,
         MetricAttribute.MAX,
@@ -51,32 +47,12 @@ public class ScenarioResult {
         MetricAttribute.M5_RATE,
         MetricAttribute.M15_RATE
     );
-    private final long startedAt;
-    private final long endedAt;
 
-    private final Exception exception;
-    private final String iolog;
-
-    public ScenarioResult(Exception e, String iolog, long startedAt, long endedAt) {
-        logger.debug("populating "+(e==null? "NORMAL" : "ERROR")+" scenario result");
-        if (logger.isDebugEnabled()) {
-            StackTraceElement[] st = Thread.currentThread().getStackTrace();
-            for (int i = 0; i < st.length; i++) {
-                logger.debug(":AT " + st[i].getFileName()+":"+st[i].getLineNumber()+":"+st[i].getMethodName());
-                if (i>10) break;
-            }
-        }
-        this.iolog = ((iolog!=null) ? iolog + "\n\n" : "") + (e!=null? e.getMessage() : "");
-        this.startedAt = startedAt;
-        this.endedAt = endedAt;
-        this.exception = e;
+    public ExecutionMetricsResult(long startedAt, long endedAt, String iolog, Exception error) {
+        super(startedAt, endedAt, iolog, error);
     }
 
-    public void reportElapsedMillis() {
-        logger.info("-- SCENARIO TOOK " + getElapsedMillis() + "ms --");
-    }
-
-    public String getSummaryReport() {
+    public String getMetricsSummary() {
         ByteArrayOutputStream os = new ByteArrayOutputStream();
         PrintStream ps = new PrintStream(os);
         ConsoleReporter.Builder builder = ConsoleReporter.forRegistry(ActivityMetrics.getMetricRegistry())
@@ -91,45 +67,23 @@ public class ScenarioResult {
         builder.disabledMetricAttributes(disabled);
         ConsoleReporter consoleReporter = builder.build();
         consoleReporter.report();
-
         ps.flush();
+        consoleReporter.close();
         String result = os.toString(StandardCharsets.UTF_8);
         return result;
     }
 
     public void reportToConsole() {
-        String summaryReport = getSummaryReport();
+        String summaryReport = getMetricsSummary();
         System.out.println(summaryReport);
     }
 
 
-    public Optional<Exception> getException() {
-        return Optional.ofNullable(exception);
+    public void reportMetricsSummaryTo(PrintStream out) {
+        out.println(getMetricsSummary());
     }
 
-    public void rethrowIfError() {
-        if (exception != null) {
-            if (exception instanceof RuntimeException) {
-                throw ((RuntimeException) exception);
-            } else {
-                throw new RuntimeException(exception);
-            }
-        }
-    }
-
-    public String getIOLog() {
-        return this.iolog;
-    }
-
-    public long getElapsedMillis() {
-        return endedAt - startedAt;
-    }
-
-    public void reportTo(PrintStream out) {
-        out.println(getSummaryReport());
-    }
-
-    public void reportToLog() {
+    public void reportMetricsSummaryToLog() {
         logger.debug("-- WARNING: Metrics which are taken per-interval (like histograms) will not have --");
         logger.debug("-- active data on this last report. (The workload has already stopped.) Record   --");
         logger.debug("-- metrics to an external format to see values for each reporting interval.      --");
@@ -142,10 +96,11 @@ public class ScenarioResult {
             .outputTo(logger)
             .build();
         reporter.report();
+        reporter.close();
         logger.debug("-- END METRICS DETAIL --");
     }
 
-    public void reportCountsTo(PrintStream printStream) {
+    public void reportMetricsCountsTo(PrintStream printStream) {
         StringBuilder sb = new StringBuilder();
 
         ActivityMetrics.getMetricRegistry().getMetrics().forEach((k, v) -> {

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ExecutionResult.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Provide a result type back to a caller, including the start and end times,
+ * any exception that occurred, and any content written to stdout or stderr equivalent
+ * IO streams. This is an <EM>execution result</EM>.
+ *
+ */
+public class ExecutionResult {
+    protected final static Logger logger = LogManager.getLogger(ExecutionMetricsResult.class);
+    protected final long startedAt;
+    protected final long endedAt;
+    protected final Exception exception;
+    protected final String iolog;
+
+    public ExecutionResult(long startedAt, long endedAt, String iolog, Exception error) {
+        this.startedAt = startedAt;
+        this.endedAt = endedAt;
+        this.exception = error;
+        this.iolog = ((iolog != null) ? iolog + "\n\n" : "") + exception;
+        logger.debug("populating "+(error==null ? "NORMAL" : "ERROR")+" scenario result");
+        if (logger.isTraceEnabled()) {
+            StackTraceElement[] st = Thread.currentThread().getStackTrace();
+            for (int i = 0; i < st.length; i++) {
+                logger.debug(":AT " + st[i].getFileName()+":"+st[i].getLineNumber()+":"+st[i].getMethodName());
+                if (i>10) break;
+            }
+        }
+
+    }
+
+    public void reportElapsedMillisToLog() {
+        logger.info(() -> String.format("-- SCENARIO TOOK %.3fS --",(getElapsedMillis()/1000.0f)));
+    }
+
+    public String getIOLog() {
+        return this.iolog;
+    }
+
+    public long getElapsedMillis() {
+        return endedAt - startedAt;
+    }
+
+    public Exception getException() {
+        return exception;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivitiesExceptionHandler.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.activity;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenarioController;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class ActivitiesExceptionHandler implements Thread.UncaughtExceptionHandler {
+
+    private static final Logger logger = LogManager.getLogger(ActivitiesExceptionHandler.class);
+
+    private final ScenarioController controller;
+
+    public ActivitiesExceptionHandler(ScenarioController controller) {
+        this.controller = controller;
+        logger.debug(() -> "Activity exception handler starting up for executor '" + this.controller + "'");
+    }
+
+
+    @Override
+    public void uncaughtException(Thread t, Throwable e) {
+        logger.error("Uncaught exception in thread '" + t.getName() + ", state[" + t.getState() + "], notifying executor '" + controller + "'");
+        controller.notifyException(t, e);
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExceptionHandler.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.lifecycle;
+package io.nosqlbench.engine.core.lifecycle.activity;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -33,7 +33,7 @@ public class ActivityExceptionHandler implements Thread.UncaughtExceptionHandler
 
     @Override
     public void uncaughtException(Thread t, Throwable e) {
-        logger.error("Uncaught exception in thread '" + t.getName() + ", state[" + t.getState() + "], notifying executor '" + executor + "'");
+        logger.error("Uncaught exception in thread '" + t.getName() + ", state[" + t.getState() + "], notifying executor '" + executor + "': " + e);
         executor.notifyException(t, e);
     }
 }

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.nosqlbench.engine.core.lifecycle;
+package io.nosqlbench.engine.core.lifecycle.activity;
 
 import io.nosqlbench.api.annotations.Annotation;
 import io.nosqlbench.api.annotations.Layer;
@@ -22,34 +22,38 @@ import io.nosqlbench.api.engine.activityimpl.ParameterMap;
 import io.nosqlbench.engine.api.activityapi.core.*;
 import io.nosqlbench.engine.api.activityapi.core.progress.ProgressCapable;
 import io.nosqlbench.engine.api.activityapi.core.progress.ProgressMeterDisplay;
+import io.nosqlbench.engine.api.activityimpl.motor.RunStateImage;
+import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
 import io.nosqlbench.engine.core.annotation.Annotators;
+import io.nosqlbench.engine.core.lifecycle.ExecutionResult;
+import io.nosqlbench.engine.core.lifecycle.IndexedThreadFactory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
 import java.util.stream.Collectors;
 
 /**
- * <p>An ActivityExecutor is a named instance of an execution harness for a single activity instance.
+ * <p>An ActivityExecutor is an execution harness for a single activity instance.
  * It is responsible for managing threads and activity settings which may be changed while the activity is running.</p>
  *
- * <p>An ActivityExecutor may be represent an activity that is defined and active in the running
- * scenario, but which is inactive. This can occur when an activity is paused by controlling logic, or when the threads
- * are set to zero.</p>
+ * <p>In order to allow for dynamic thread management, which is not easily supported as an explicit feature
+ * of most executor services, threads are started as long-running processes and managed via state signaling.
+ * The {@link RunState} enum, {@link io.nosqlbench.engine.api.activityimpl.MotorState} type, and {@link RunStateTally}
+ * state tracking class are used together to represent valid states and transitions, contain and transition state atomically,
+ * and provide blocking conditions for observers, respectively.</p>
+ *
+ * <P>Some basic rules and invariants must be observed for consistent concurrent behavior.
+ * Any state changes for a Motor must be made through {@link Motor#getState()}.
+ * This allows the state tracking to work consistently for all observers.</p>
  *
- * <p>
- * Invariants:
- * </p>
- * <ul>
- *     <li>Motors may not receive parameter updates before their owning activities are initialized.</li>
- * </ul>
  */
 
-public class ActivityExecutor implements ActivityController, ParameterMap.Listener, ProgressCapable {
+public class ActivityExecutor implements ActivityController, ParameterMap.Listener, ProgressCapable, Callable<ExecutionResult> {
+
+    // TODO Encapsulate valid state transitions to be only modifiable within the appropriate type view.
 
     private static final Logger logger = LogManager.getLogger(ActivityExecutor.class);
     private static final Logger activitylogger = LogManager.getLogger("ACTIVITY");
@@ -57,8 +61,9 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
     private final List<Motor<?>> motors = new ArrayList<>();
     private final Activity activity;
     private final ActivityDef activityDef;
-    private final ExecutorService executorService;
-    private RuntimeException stoppingException;
+    private final RunStateTally tally;
+    private ExecutorService executorService;
+    private Exception exception;
 
     private final static int waitTime = 10000;
     private String sessionId = "";
@@ -71,90 +76,43 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
     public ActivityExecutor(Activity activity, String sessionId) {
         this.activity = activity;
         this.activityDef = activity.getActivityDef();
-        executorService = new ThreadPoolExecutor(
-                0, Integer.MAX_VALUE,
-                0L, TimeUnit.SECONDS,
-                new SynchronousQueue<>(),
-                new IndexedThreadFactory(activity.getAlias(), new ActivityExceptionHandler(this))
-        );
         activity.getActivityDef().getParams().addListener(this);
         activity.setActivityController(this);
         this.sessionId = sessionId;
+        this.tally = activity.getRunStateTally();
     }
 
-    public void setSessionId(String sessionId) {
-        this.sessionId = sessionId;
-    }
-
-
     // TODO: Doc how uninitialized activities do not propagate parameter map changes and how
     // TODO: this is different from preventing modification to uninitialized activities
 
-    /**
-     * <p>True-up the number of motor instances known to the executor. Start all non-running motors.
-     * The protocol between the motors and the executor should be safe as long as each state change is owned by either
-     * the motor logic or the activity executor but not both, and strictly serialized as well. This is enforced by
-     * forcing start(...) to be serialized as well as using CAS on the motor states.</p>
-     * <p>The startActivity method may be called to true-up the number of active motors in an activity executor after
-     * changes to threads.</p>
-     */
-    public synchronized void startActivity() {
-        logger.info("starting activity " + activity.getAlias() + " for cycles " + activity.getCycleSummary());
-        Annotators.recordAnnotation(Annotation.newBuilder()
-                .session(sessionId)
-                .now()
-                .layer(Layer.Activity)
-                .label("alias", getActivityDef().getAlias())
-                .label("driver", getActivityDef().getActivityType())
-                .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none"))
-                .detail("params", getActivityDef().toString())
-                .build()
-        );
-
-        activitylogger.debug("START/before alias=(" + activity.getAlias() + ")");
-        try {
-            activity.setRunState(RunState.Starting);
-            this.startedAt = System.currentTimeMillis();
-            activity.initActivity();
-            //activity.onActivityDefUpdate(activityDef);
-        } catch (Exception e) {
-            this.stoppingException = new RuntimeException("Error initializing activity '" + activity.getAlias() + "':\n" + e.getMessage(), e);
-//            activitylogger.error("error initializing activity '" + activity.getAlias() + "': " + stoppingException);
-            throw stoppingException;
-        }
-        adjustToActivityDef(activity.getActivityDef());
-        activity.setRunState(RunState.Running);
-        activitylogger.debug("START/after alias=(" + activity.getAlias() + ")");
-    }
-
     /**
      * Simply stop the motors
      */
-    public synchronized void stopActivity() {
-        activitylogger.debug("STOP/before alias=(" + activity.getAlias() + ")");
-
-        activity.setRunState(RunState.Stopping);
+    public void stopActivity() {
         logger.info("stopping activity in progress: " + this.getActivityDef().getAlias());
+        activity.setRunState(RunState.Stopping);
         motors.forEach(Motor::requestStop);
-        motors.forEach(m -> awaitRequiredMotorState(m, 30000, 50, RunState.Stopped, RunState.Finished));
-        activity.shutdownActivity();
-        activity.closeAutoCloseables();
+        tally.awaitNoneOther(RunState.Stopped,RunState.Finished);
+
+        shutdownExecutorService(Integer.MAX_VALUE);
+        tally.awaitNoneOther(RunState.Stopped,RunState.Finished);
         activity.setRunState(RunState.Stopped);
+
         logger.info("stopped: " + this.getActivityDef().getAlias() + " with " + motors.size() + " slots");
-        activitylogger.debug("STOP/after alias=(" + activity.getAlias() + ")");
+
         Annotators.recordAnnotation(Annotation.newBuilder()
-                .session(sessionId)
-                .interval(this.startedAt, this.stoppedAt)
-                .layer(Layer.Activity)
-                .label("alias", getActivityDef().getAlias())
-                .label("driver", getActivityDef().getActivityType())
-                .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none"))
-                .detail("params", getActivityDef().toString())
-                .build()
+            .session(sessionId)
+            .interval(this.startedAt, this.stoppedAt)
+            .layer(Layer.Activity)
+            .label("alias", getActivityDef().getAlias())
+            .label("driver", getActivityDef().getActivityType())
+            .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none"))
+            .detail("params", getActivityDef().toString())
+            .build()
         );
     }
 
-    public synchronized RuntimeException forceStopScenario(int initialMillisToWait) {
+    public Exception forceStopActivity(int initialMillisToWait) {
 
         activitylogger.debug("FORCE STOP/before alias=(" + activity.getAlias() + ")");
         activity.setRunState(RunState.Stopped);
@@ -193,10 +151,10 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
         logger.debug("took " + (activityShutdownEndedAt - activityShutdownStartedAt) + " ms to shutdown activity threads");
         activitylogger.debug("FORCE STOP/after alias=(" + activity.getAlias() + ")");
 
-        if (stoppingException != null) {
+        if (exception != null) {
             activitylogger.debug("FORCE STOP/exception alias=(" + activity.getAlias() + ")");
         }
-        return stoppingException;
+        return exception;
 
     }
 
@@ -207,58 +165,18 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
      *                            everything to stop
      */
     public synchronized void forceStopScenarioAndThrow(int initialMillisToWait, boolean rethrow) {
-        RuntimeException exception = forceStopScenario(initialMillisToWait);
+        Exception exception = forceStopActivity(initialMillisToWait);
         if (exception != null && rethrow) {
-            throw exception;
+            throw new RuntimeException(exception);
         }
     }
 
-    public boolean finishAndShutdownExecutor(int secondsToWait) {
-
-        activitylogger.debug("REQUEST STOP/before alias=(" + activity.getAlias() + ")");
-        logger.debug("Stopping executor for " + activity.getAlias() + " when work completes.");
-
-        boolean wasStopped = false;
-        try {
-            executorService.shutdown();
-            logger.trace(() -> "awaiting termination with timeout of " + secondsToWait + " seconds");
-            wasStopped = executorService.awaitTermination(secondsToWait, TimeUnit.SECONDS);
-        } catch (InterruptedException ie) {
-            logger.trace("interrupted while awaiting termination");
-            wasStopped = false;
-            logger.warn("while waiting termination of shutdown " + activity.getAlias() + ", " + ie.getMessage());
-            activitylogger.debug("REQUEST STOP/exception alias=(" + activity.getAlias() + ") wasstopped=" + wasStopped);
-        } catch (RuntimeException e) {
-            logger.trace("Received exception while awaiting termination: " + e.getMessage());
-            wasStopped = true;
-            stoppingException = e;
-        } finally {
-
-            logger.trace(() -> "finally shutting down activity " + this.getActivity().getAlias());
-            activity.shutdownActivity();
-
-            logger.trace("closing auto-closeables");
-            activity.closeAutoCloseables();
-            activity.setRunState(RunState.Stopped);
-            this.stoppedAt = System.currentTimeMillis();
-        }
-
-        if (stoppingException != null) {
-            logger.trace(() -> "an exception caused the activity to stop:" + stoppingException.getMessage());
-            throw stoppingException;
-        }
-
-        activitylogger.debug("REQUEST STOP/after alias=(" + activity.getAlias() + ") wasstopped=" + wasStopped);
-
-        return wasStopped;
-    }
-
     /**
      * Listens for changes to parameter maps, maps them to the activity instance, and notifies all eligible listeners of
      * changes.
      */
     @Override
-    public synchronized void handleParameterMapUpdate(ParameterMap parameterMap) {
+    public void handleParameterMapUpdate(ParameterMap parameterMap) {
 
         activity.onActivityDefUpdate(activityDef);
 
@@ -267,13 +185,13 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
         // by the RunState.
         if (activity.getRunState() != RunState.Uninitialized) {
             if (activity.getRunState() == RunState.Running) {
-                adjustToActivityDef(activity.getActivityDef());
+                adjustMotorCountToThreadParam(activity.getActivityDef());
             }
             motors.stream()
-                    .filter(m -> (m instanceof ActivityDefObserver))
+                .filter(m -> (m instanceof ActivityDefObserver))
 //                    .filter(m -> m.getSlotStateTracker().getSlotState() != RunState.Uninitialized)
 //                    .filter(m -> m.getSlotStateTracker().getSlotState() != RunState.Starting)
-                    .forEach(m -> ((ActivityDefObserver) m).onActivityDefUpdate(activityDef));
+                .forEach(m -> ((ActivityDefObserver) m).onActivityDefUpdate(activityDef));
         }
     }
 
@@ -288,47 +206,32 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
      * <p>
      * TODO: move activity finisher thread to this class and remove separate implementation
      */
-    public boolean awaitCompletion(int waitTime) {
-        logger.debug(()-> "awaiting completion of '" + this.getActivity().getAlias() + "'");
-        boolean finished = finishAndShutdownExecutor(waitTime);
+    private boolean awaitCompletion(int waitTime) {
+        logger.debug(() -> "awaiting completion of '" + this.getActivity().getAlias() + "'");
+        boolean finished = shutdownExecutorService(waitTime);
 
         Annotators.recordAnnotation(Annotation.newBuilder()
-                .session(sessionId)
-                .interval(startedAt, this.stoppedAt)
-                .layer(Layer.Activity)
-                .label("alias", getActivityDef().getAlias())
-                .label("driver", getActivityDef().getActivityType())
-                .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none"))
-                .detail("params", getActivityDef().toString())
-                .build()
+            .session(sessionId)
+            .interval(startedAt, this.stoppedAt)
+            .layer(Layer.Activity)
+            .label("alias", getActivityDef().getAlias())
+            .label("driver", getActivityDef().getActivityType())
+            .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none"))
+            .detail("params", getActivityDef().toString())
+            .build()
         );
 
         return finished;
     }
 
-    public boolean awaitFinish(int timeout) {
-        activitylogger.debug("AWAIT-FINISH/before alias=(" + activity.getAlias() + ")");
-
-        boolean awaited = awaitAllRequiredMotorState(timeout, 50, RunState.Finished, RunState.Stopped);
-        if (awaited) {
-            awaited = awaitCompletion(timeout);
-        }
-        if (stoppingException != null) {
-            activitylogger.debug("AWAIT-FINISH/exception alias=(" + activity.getAlias() + ")");
-            throw stoppingException;
-        }
-        activitylogger.debug("AWAIT-FINISH/after alias=(" + activity.getAlias() + ")");
-        return awaited;
-    }
-
     public String toString() {
         return getClass().getSimpleName() + "~" + activityDef.getAlias();
     }
 
     private String getSlotStatus() {
         return motors.stream()
-                .map(m -> m.getSlotStateTracker().getSlotState().getCode())
-                .collect(Collectors.joining(",", "[", "]"));
+            .map(m -> m.getState().get().getCode())
+            .collect(Collectors.joining(",", "[", "]"));
     }
 
     /**
@@ -336,17 +239,19 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
      *
      * @param activityDef the activityDef for this activity instance
      */
-    private synchronized void adjustToActivityDef(ActivityDef activityDef) {
+    private void adjustMotorCountToThreadParam(ActivityDef activityDef) {
         logger.trace(() -> ">-pre-adjust->" + getSlotStatus());
 
-        // Stop and remove extra motor slots
-        while (motors.size() > activityDef.getThreads()) {
-            Motor motor = motors.get(motors.size() - 1);
-            logger.trace(() -> "Stopping cycle motor thread:" + motor);
-            motor.requestStop();
-            motors.remove(motors.size() - 1);
-        }
+        reduceActiveMotorCountDownToThreadParam(activityDef);
+        increaseActiveMotorCountUpToThreadParam(activityDef);
+        alignMotorStateToIntendedActivityState();
+        awaitAlignmentOfMotorStateToActivityState();
 
+        logger.trace(() -> ">post-adjust->" + getSlotStatus());
+
+    }
+
+    private void increaseActiveMotorCountUpToThreadParam(ActivityDef activityDef) {
         // Create motor slots
         while (motors.size() < activityDef.getThreads()) {
 
@@ -354,15 +259,27 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
             logger.trace(() -> "Starting cycle motor thread:" + motor);
             motors.add(motor);
         }
-
-        applyIntendedStateToDivergentMotors();
-        awaitActivityAndMotorStateAlignment();
-
-        logger.trace(() -> ">post-adjust->" + getSlotStatus());
-
     }
 
-    private void applyIntendedStateToDivergentMotors() {
+    private void reduceActiveMotorCountDownToThreadParam(ActivityDef activityDef) {
+        // Stop and remove extra motor slots
+        while (motors.size() > activityDef.getThreads()) {
+            Motor motor = motors.get(motors.size() - 1);
+            logger.trace(() -> "Stopping cycle motor thread:" + motor);
+            motor.requestStop();
+            motor.removeState();
+
+            /**
+             * NOTE: this leaves trailing, longer-running threads which might hold the executor open
+             * to potentially be cleaned up by {@link ExecutorService#shutdown()} or
+             * {@link ExecutorService#shutdownNow()}. At this point, the motor thread has
+             * been instructed to shutdown, and it is effectively thread-non-grata to the activity.
+             */
+            motors.remove(motors.size() - 1);
+        }
+    }
+
+    private void alignMotorStateToIntendedActivityState() {
         RunState intended = activity.getRunState();
         logger.trace(() -> "ADJUSTING to INTENDED " + intended);
         switch (intended) {
@@ -371,18 +288,17 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
             case Running:
             case Starting:
                 motors.stream()
-                        .filter(m -> m.getSlotStateTracker().getSlotState() != RunState.Running)
-                        .filter(m -> m.getSlotStateTracker().getSlotState() != RunState.Finished)
-                        .filter(m -> m.getSlotStateTracker().getSlotState() != RunState.Starting)
-                        .forEach(m -> {
-                            m.getSlotStateTracker().enterState(RunState.Starting);
-                            executorService.execute(m);
-                        });
+                    .filter(m -> m.getState().get() != RunState.Running)
+                    .filter(m -> m.getState().get() != RunState.Finished)
+                    .filter(m -> m.getState().get() != RunState.Starting)
+                    .forEach(m -> {
+                        executorService.execute(m);
+                    });
                 break;
             case Stopped:
                 motors.stream()
-                        .filter(m -> m.getSlotStateTracker().getSlotState() != RunState.Stopped)
-                        .forEach(Motor::requestStop);
+                    .filter(m -> m.getState().get() != RunState.Stopped)
+                    .forEach(Motor::requestStop);
                 break;
             case Finished:
             case Stopping:
@@ -393,20 +309,21 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
         }
     }
 
-    private void awaitActivityAndMotorStateAlignment() {
+    private void awaitAlignmentOfMotorStateToActivityState() {
 
+        logger.debug(()->"awaiting state alignment from " + activity.getRunState());
         switch (activity.getRunState()) {
             case Starting:
             case Running:
-                motors.forEach(m -> awaitRequiredMotorState(m, waitTime, 50, RunState.Running, RunState.Finished));
+                tally.awaitNoneOther(RunState.Running, RunState.Finished);
                 break;
             case Stopped:
-                motors.forEach(m -> awaitRequiredMotorState(m, waitTime, 50, RunState.Stopped, RunState.Finished));
+                tally.awaitNoneOther(RunState.Stopped, RunState.Finished);
                 break;
             case Uninitialized:
                 break;
             case Finished:
-                motors.forEach(m -> awaitRequiredMotorState(m, waitTime, 50, RunState.Finished));
+                tally.awaitNoneOther(RunState.Finished);
                 break;
             case Stopping:
                 throw new RuntimeException("Invalid requested state in activity executor:" + activity.getRunState());
@@ -417,136 +334,40 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
 
     }
 
-    /**
-     * Await a thread (aka motor/slot) entering a specific SlotState
-     *
-     * @param m                motor instance
-     * @param waitTime         milliseco`nds to wait, total
-     * @param pollTime         polling interval between state checks
-     * @param desiredRunStates any desired SlotState
-     * @return true, if the desired SlotState was detected
-     */
-    private boolean awaitMotorState(Motor m, int waitTime, int pollTime, RunState... desiredRunStates) {
-        long startedAt = System.currentTimeMillis();
-        while (System.currentTimeMillis() < (startedAt + waitTime)) {
-            Map<RunState, Integer> actualStates = new HashMap<>();
-            for (RunState state : desiredRunStates) {
-                actualStates.compute(state, (k, v) -> (v == null ? 0 : v) + 1);
-            }
-            for (RunState desiredRunState : desiredRunStates) {
-                actualStates.remove(desiredRunState);
-            }
-            logger.trace(() -> "state of remaining slots:" + actualStates);
-            if (actualStates.size() == 0) {
-                return true;
-            } else {
-                System.out.println("motor states:" + actualStates);
-                try {
-                    Thread.sleep(pollTime);
-                } catch (InterruptedException ignored) {
-                }
-            }
-        }
-        logger.trace(() -> activityDef.getAlias() + "/Motor[" + m.getSlotId() + "] is now in state " + m.getSlotStateTracker().getSlotState());
-        return false;
-    }
 
-
-    private boolean awaitAllRequiredMotorState(int waitTime, int pollTime, RunState... awaitingState) {
-        long startedAt = System.currentTimeMillis();
-        boolean awaited = false;
-        while (!awaited && (System.currentTimeMillis() < (startedAt + waitTime))) {
-            awaited = true;
-            for (Motor motor : motors) {
-                awaited = awaitMotorState(motor, waitTime, pollTime, awaitingState);
-                if (!awaited) {
-                    logger.trace(() -> "failed awaiting motor " + motor.getSlotId() + " for state in " +
-                            Arrays.asList(awaitingState));
-                    break;
-                }
-            }
-        }
-        return awaited;
-    }
-
-
-    private boolean awaitAnyRequiredMotorState(int waitTime, int pollTime, RunState... awaitingState) {
-        long startedAt = System.currentTimeMillis();
-        while (System.currentTimeMillis() < (startedAt + waitTime)) {
-            for (Motor motor : motors) {
-                for (RunState state : awaitingState) {
-                    if (motor.getSlotStateTracker().getSlotState() == state) {
-                        logger.trace(() -> "at least one 'any' of " + activityDef.getAlias() + "/Motor[" + motor.getSlotId() + "] is now in state " + motor.getSlotStateTracker().getSlotState());
-                        return true;
-                    }
-                }
-            }
-            try {
-                Thread.sleep(pollTime);
-            } catch (InterruptedException ignored) {
-            }
-        }
-        logger.trace(() -> "none of " + activityDef.getAlias() + "/Motor [" + motors.size() + "] is in states in " + Arrays.asList(awaitingState));
-        return false;
-    }
-
-
-    /**
-     * Await a required thread (aka motor/slot) entering a specific SlotState
-     *
-     * @param m             motor instance
-     * @param waitTime      milliseconds to wait, total
-     * @param pollTime      polling interval between state checks
-     * @param awaitingState desired SlotState
-     * @throws RuntimeException if the waitTime is used up and the desired state is not reached
-     */
-    private void awaitRequiredMotorState(Motor m, int waitTime, int pollTime, RunState... awaitingState) {
-        RunState startingState = m.getSlotStateTracker().getSlotState();
-        boolean awaitedRequiredState = awaitMotorState(m, waitTime, pollTime, awaitingState);
-        if (!awaitedRequiredState) {
-            String error = "Unable to await " + activityDef.getAlias() +
-                    "/Motor[" + m.getSlotId() + "]: from state " + startingState + " to " + m.getSlotStateTracker().getSlotState()
-                    + " after waiting for " + waitTime + "ms";
-            RuntimeException e = new RuntimeException(error);
-            logger.error(error);
-            throw e;
-        }
-        logger.trace(() -> "motor " + m + " entered awaited state: " + Arrays.asList(awaitingState));
-    }
-
-    private synchronized void requestStopMotors() {
+    private void requestStopMotors() {
         logger.info("stopping activity " + activity);
-        activity.setRunState(RunState.Stopped);
+        activity.setRunState(RunState.Stopping);
         motors.forEach(Motor::requestStop);
     }
 
 
     public boolean isRunning() {
-        return motors.stream().anyMatch(m -> m.getSlotStateTracker().getSlotState() == RunState.Running);
+        return motors.stream().anyMatch(m -> m.getState().get() == RunState.Running);
     }
 
     public Activity getActivity() {
         return activity;
     }
 
-    public synchronized void notifyException(Thread t, Throwable e) {
+    public void notifyException(Thread t, Throwable e) {
         logger.debug(() -> "Uncaught exception in activity thread forwarded to activity executor: " + e.getMessage());
-        this.stoppingException = new RuntimeException("Error in activity thread " + t.getName(), e);
-        forceStopScenario(10000);
+        this.exception = new RuntimeException("Error in activity thread " + t.getName(), e);
+        this.requestStopMotors();
     }
 
     @Override
     public synchronized void stopActivityWithReasonAsync(String reason) {
         logger.info("Stopping activity " + this.activityDef.getAlias() + ": " + reason);
-        this.stoppingException = new RuntimeException("Stopping activity " + this.activityDef.getAlias() + ": " + reason);
-        logger.error("stopping with reason: " + stoppingException);
+        this.exception = new RuntimeException("Stopping activity " + this.activityDef.getAlias() + ": " + reason);
+        logger.error("stopping with reason: " + exception);
         requestStopMotors();
     }
 
     @Override
     public synchronized void stopActivityWithErrorAsync(Throwable throwable) {
-        if (stoppingException == null) {
-            this.stoppingException = new RuntimeException(throwable);
+        if (exception == null) {
+            this.exception = new RuntimeException(throwable);
             logger.error("stopping on error: " + throwable.toString(), throwable);
         } else {
             if (activityDef.getParams().getOptionalBoolean("fullerrors").orElse(false)) {
@@ -564,4 +385,140 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
     }
 
 
+    @Override
+    public ExecutionResult call() throws Exception {
+
+        try {
+            // instantiate and configure fixtures that need to be present
+            // before threads start running such as metrics instruments
+            activity.initActivity();
+            awaitMotorsAtLeastRunning();
+            awaitActivityCompletion();
+            activity.shutdownActivity();
+            activity.closeAutoCloseables();
+        } catch (Exception e) {
+            this.exception = e;
+        }
+        ExecutionResult result = new ExecutionResult(startedAt, stoppedAt, "", exception);
+        return result;
+    }
+
+    /**
+     * This waits for at least one motor to be in running, finished or stopped state.
+     * A motor with enough cycles to read will go into a running state. A motor which has
+     * a short read immediately after being started will go into a finished state. A motor
+     * which has been stopped for some reason, like an error or a stop command will go into
+     * stopped state. All of these states are sufficient to signal that successful startup
+     * has been completed at least.
+     */
+    private void awaitMotorsAtLeastRunning() {
+        RunStateImage states = tally.awaitAny(RunState.Running, RunState.Stopped, RunState.Finished, RunState.Errored);
+        RunState maxState = states.getMaxState();
+        if (maxState==RunState.Errored) {
+            activity.setRunState(maxState);
+            throw new RuntimeException("Error in activity");
+        }
+    }
+
+    public void startActivity() {
+        // we need an executor service to run motor threads on
+        startMotorExecutorService();
+        startRunningActivityThreads();
+        awaitMotorsAtLeastRunning();
+    }
+
+    private boolean shutdownExecutorService(int secondsToWait) {
+
+        activitylogger.debug(() -> "Shutting down motor executor for (" + activity.getAlias() + ")");
+
+        boolean wasStopped = false;
+        try {
+            executorService.shutdown();
+            logger.trace(() -> "awaiting termination with timeout of " + secondsToWait + " seconds");
+            wasStopped = executorService.awaitTermination(secondsToWait, TimeUnit.SECONDS);
+        } catch (InterruptedException ie) {
+            logger.trace("interrupted while awaiting termination");
+            wasStopped = false;
+            logger.warn("while waiting termination of shutdown " + activity.getAlias() + ", " + ie.getMessage());
+            activitylogger.debug("REQUEST STOP/exception alias=(" + activity.getAlias() + ") wasstopped=" + wasStopped);
+        } catch (RuntimeException e) {
+            logger.trace("Received exception while awaiting termination: " + e.getMessage());
+            wasStopped = true;
+            exception = e;
+        } finally {
+            logger.trace(() -> "finally shutting down activity " + this.getActivity().getAlias());
+            this.stoppedAt = System.currentTimeMillis();
+            activity.setRunState(RunState.Stopped);
+        }
+
+        if (exception != null) {
+            logger.trace(() -> "an exception caused the activity to stop:" + exception.getMessage());
+            logger.warn("Setting ERROR on motor executor for activity '" + activity.getAlias() + "': " + exception.getMessage());
+            throw new RuntimeException(exception);
+        }
+
+        activitylogger.debug("motor executor for " + activity.getAlias() + ") wasstopped=" + wasStopped);
+
+        return wasStopped;
+    }
+
+    private void awaitActivityCompletion() {
+        RunStateImage state = tally.awaitNoneOther(RunState.Stopped, RunState.Finished, RunState.Errored);
+        RunState maxState = state.getMaxState();
+        activity.setRunState(maxState);
+        if (maxState==RunState.Errored) {
+            throw new RuntimeException("Error while waiting for activity completion:" + this.exception);
+        }
+    }
+
+    private void startMotorExecutorService() {
+        this.executorService = new ThreadPoolExecutor(
+            0, Integer.MAX_VALUE,
+            0L, TimeUnit.SECONDS,
+            new SynchronousQueue<>(),
+            new IndexedThreadFactory(activity.getAlias(), new ActivityExceptionHandler(this))
+        );
+    }
+
+
+    /**
+     * <p>True-up the number of motor instances known to the executor. Start all non-running motors.
+     * The protocol between the motors and the executor should be safe as long as each state change is owned by either
+     * the motor logic or the activity executor but not both, and strictly serialized as well. This is enforced by
+     * forcing start(...) to be serialized as well as using CAS on the motor states.</p>
+     * <p>The startActivity method may be called to true-up the number of active motors in an activity executor after
+     * changes to threads.</p>
+     */
+    private void startRunningActivityThreads() {
+
+        logger.info("starting activity " + activity.getAlias() + " for cycles " + activity.getCycleSummary());
+        Annotators.recordAnnotation(Annotation.newBuilder()
+            .session(sessionId)
+            .now()
+            .layer(Layer.Activity)
+            .label("alias", getActivityDef().getAlias())
+            .label("driver", getActivityDef().getActivityType())
+            .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none"))
+            .detail("params", getActivityDef().toString())
+            .build()
+        );
+
+        activitylogger.debug("START/before alias=(" + activity.getAlias() + ")");
+
+        try {
+            activity.setRunState(RunState.Starting);
+            this.startedAt = System.currentTimeMillis();
+            activity.onActivityDefUpdate(activityDef);
+        } catch (Exception e) {
+            this.exception = new RuntimeException("Error initializing activity '" + activity.getAlias() + "':\n" + e.getMessage(), e);
+            activitylogger.error(()->"error initializing activity '" + activity.getAlias() + "': " + exception);
+            throw new RuntimeException(exception);
+        }
+        adjustMotorCountToThreadParam(activity.getActivityDef());
+        tally.awaitAny(RunState.Running,RunState.Finished,RunState.Stopped);
+        activity.setRunState(RunState.Running);
+        activitylogger.debug("START/after alias=(" + activity.getAlias() + ")");
+    }
+
+
 }

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityLoader.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.activity;
+
+import io.nosqlbench.api.engine.activityimpl.ActivityDef;
+import io.nosqlbench.engine.api.activityapi.core.Activity;
+import io.nosqlbench.engine.api.activityimpl.uniform.StandardActivityType;
+import io.nosqlbench.engine.core.lifecycle.scenario.Scenario;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Consolidates the activity type and activity instantiation logic into one place
+ * per scope. Within the lifetime of this ActivityLoader, all activities may
+ * see each other by name.
+ */
+public class ActivityLoader {
+    private final static Logger logger = LogManager.getLogger("ACTIVITIES");
+    private final Map<String, Activity> activityMap = new ConcurrentHashMap<>();
+    private final Scenario scenario;
+
+    public ActivityLoader(Scenario scenario) {
+        this.scenario = scenario;
+    }
+
+    public synchronized Activity loadActivity(ActivityDef activityDef) {
+        Activity activity = new StandardActivityType(activityDef).getAssembledActivity(activityDef, activityMap);
+        activityMap.put(activity.getAlias(),activity);
+        logger.debug("Resolved activity for alias '" + activityDef.getAlias() + "'");
+        return activity;
+    }
+
+    public void purgeActivity(String activityAlias) {
+        this.activityMap.remove(activityAlias);
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityProgressIndicator.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.lifecycle;
+package io.nosqlbench.engine.core.lifecycle.activity;
 
 import io.nosqlbench.engine.api.activityapi.core.RunState;
 import io.nosqlbench.engine.api.activityapi.core.progress.ProgressMeterDisplay;
@@ -22,6 +22,7 @@ import io.nosqlbench.engine.api.activityapi.core.progress.StateCapable;
 import io.nosqlbench.engine.api.metrics.IndicatorMode;
 import io.nosqlbench.api.engine.metrics.PeriodicRunnable;
 import io.nosqlbench.api.engine.util.Unit;
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenarioController;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityRuntimeInfo.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.activity;
+
+import io.nosqlbench.engine.api.activityapi.core.Activity;
+import io.nosqlbench.engine.api.activityapi.core.RunState;
+import io.nosqlbench.engine.api.activityapi.core.progress.ProgressCapable;
+import io.nosqlbench.engine.api.activityapi.core.progress.ProgressMeterDisplay;
+import io.nosqlbench.engine.core.lifecycle.ExecutionResult;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class ActivityRuntimeInfo implements ProgressCapable {
+
+    private final static Logger logger = LogManager.getLogger(ActivityRuntimeInfo.class);
+
+    private final Activity activity;
+    private final Future<ExecutionResult> future;
+    private final ActivityExecutor executor;
+
+    private ExecutionResult result;
+
+    public ActivityRuntimeInfo(Activity activity, Future<ExecutionResult> result, ActivityExecutor executor) {
+
+        this.activity = activity;
+        this.future = result;
+        this.executor = executor;
+    }
+
+    @Override
+    public ProgressMeterDisplay getProgressMeter() {
+        return this.activity.getProgressMeter();
+    }
+
+
+    public Future<ExecutionResult> getFuture() {
+        return this.future;
+    }
+
+    /**
+     * Wait until the execution is complete and return the result.
+     * @param timeoutMillis
+     * @return null, or an ExecutionResult if the execution completed
+     */
+    public ExecutionResult awaitResult(long timeoutMillis) {
+        ExecutionResult result = null;
+        try {
+            result = future.get(timeoutMillis, TimeUnit.MILLISECONDS);
+        } catch (TimeoutException te) {
+        } catch (InterruptedException ie) {
+            logger.warn("interrupted waiting for execution to complete");
+        } catch (ExecutionException e) {
+            throw new RuntimeException(e);
+//            return new ExecutionResult(activity.getStartedAtMillis(),System.currentTimeMillis(),"",e);
+        }
+        return result;
+    }
+
+    public Activity getActivity() {
+        return this.activity;
+    }
+
+    public boolean isRunning() {
+        return executor.isRunning();
+    }
+
+    public RunState getRunState() {
+        return this.activity.getRunState();
+    }
+
+    public void stopActivity() {
+        this.executor.stopActivity();
+    }
+
+    public ActivityExecutor getActivityExecutor() {
+        return executor;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityStatus.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.activity;
+
+public class ActivityStatus {
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityTypeLoader.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.lifecycle;
+package io.nosqlbench.engine.core.lifecycle.activity;
 
 import io.nosqlbench.engine.api.activityapi.core.ActivityType;
 import io.nosqlbench.api.engine.activityimpl.ActivityDef;
@@ -150,18 +150,6 @@ public class ActivityTypeLoader {
         if (oda.isPresent()) {
             DriverAdapter<?, ?> driverAdapter = oda.get();
 
-//            activityDef.getParams().remove("driver");
-//            if (driverAdapter instanceof NBConfigurable) {
-//                NBConfigModel cfgModel = ((NBConfigurable) driverAdapter).getConfigModel();
-//                Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
-//                if (op_yaml_loc.isPresent()) {
-//                    Map<String,Object> disposable = new LinkedHashMap<>(activityDef.getParams());
-//                    StmtsDocList workload = StatementsLoader.loadPath(logger, op_yaml_loc.get(), disposable, "activities");
-//                    cfgModel=cfgModel.add(workload.getConfigModel());
-//                }
-//                NBConfiguration cfg = cfgModel.apply(activityDef.getParams());
-//                ((NBConfigurable) driverAdapter).applyConfig(cfg);
-//            }
             ActivityType activityType = new StandardActivityType<>(driverAdapter, activityDef);
             return Optional.of(activityType);
         } else {

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/process/NBCLIErrorHandler.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.lifecycle;
+package io.nosqlbench.engine.core.lifecycle.process;
 
 import io.nosqlbench.api.errors.BasicError;
 import org.apache.logging.log4j.LogManager;

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/process/ShutdownManager.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.nosqlbench.engine.core.lifecycle;
+package io.nosqlbench.engine.core.lifecycle.process;
 
 import io.nosqlbench.engine.api.activityapi.core.Shutdownable;
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/Scenario.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario;
 
 import com.codahale.metrics.MetricRegistry;
 import com.oracle.truffle.js.scriptengine.GraalJSScriptEngine;
@@ -26,11 +26,14 @@ import io.nosqlbench.api.metadata.SystemId;
 import io.nosqlbench.engine.api.extensions.ScriptingPluginInfo;
 import io.nosqlbench.engine.api.scripting.ScriptEnvBuffer;
 import io.nosqlbench.engine.core.annotation.Annotators;
-import io.nosqlbench.engine.core.lifecycle.ActivityProgressIndicator;
-import io.nosqlbench.engine.core.lifecycle.PolyglotScenarioController;
-import io.nosqlbench.engine.core.lifecycle.ScenarioController;
-import io.nosqlbench.engine.core.lifecycle.ScenarioResult;
-import io.nosqlbench.engine.core.metrics.PolyglotMetricRegistryBindings;
+import io.nosqlbench.engine.core.lifecycle.activity.ActivityProgressIndicator;
+import io.nosqlbench.engine.core.lifecycle.ExecutionMetricsResult;
+import io.nosqlbench.engine.core.lifecycle.scenario.script.bindings.PolyglotScenarioController;
+import io.nosqlbench.engine.core.lifecycle.scenario.script.bindings.ActivityBindings;
+import io.nosqlbench.engine.core.lifecycle.scenario.script.SandboxExtensionFinder;
+import io.nosqlbench.engine.core.lifecycle.scenario.script.ScenarioContext;
+import io.nosqlbench.engine.core.lifecycle.scenario.script.ScriptParams;
+import io.nosqlbench.engine.core.lifecycle.scenario.script.bindings.PolyglotMetricRegistryBindings;
 import io.nosqlbench.nb.annotations.Maturity;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -58,7 +61,7 @@ import java.util.Optional;
 import java.util.concurrent.Callable;
 import java.util.stream.Collectors;
 
-public class Scenario implements Callable<ScenarioResult> {
+public class Scenario implements Callable<ExecutionMetricsResult> {
 
     private final String commandLine;
     private final String reportSummaryTo;
@@ -71,9 +74,9 @@ public class Scenario implements Callable<ScenarioResult> {
     private Exception error;
     private ScenarioMetadata scenarioMetadata;
 
-    private ScenarioResult result;
+    private ExecutionMetricsResult result;
 
-    public Optional<ScenarioResult> getResultIfComplete() {
+    public Optional<ExecutionMetricsResult> getResultIfComplete() {
         return Optional.ofNullable(this.result);
     }
 
@@ -171,7 +174,7 @@ public class Scenario implements Callable<ScenarioResult> {
         return this;
     }
 
-    private void initializeScriptingEngine() {
+    private void initializeScriptingEngine(ScenarioController scenarioController) {
 
         logger.debug("Using engine " + engine.toString());
         MetricRegistry metricRegistry = ActivityMetrics.getMetricRegistry();
@@ -198,12 +201,10 @@ public class Scenario implements Callable<ScenarioResult> {
         this.scriptEngine = GraalJSScriptEngine.create(polyglotEngine, contextSettings);
 
 
-        scenarioController = new ScenarioController(this.scenarioName, minMaturity);
         if (!progressInterval.equals("disabled")) {
             activityProgressIndicator = new ActivityProgressIndicator(scenarioController, progressInterval);
         }
 
-
         scriptEnv = new ScenarioContext(scenarioController);
         scriptEngine.setContext(scriptEnv);
 
@@ -215,7 +216,7 @@ public class Scenario implements Callable<ScenarioResult> {
 
         scriptEngine.put("scenario", new PolyglotScenarioController(scenarioController));
         scriptEngine.put("metrics", new PolyglotMetricRegistryBindings(metricRegistry));
-        scriptEngine.put("activities", new NashornActivityBindings(scenarioController));
+        scriptEngine.put("activities", new ActivityBindings(scenarioController));
 
         for (ScriptingPluginInfo<?> extensionDescriptor : SandboxExtensionFinder.findAll()) {
             if (!extensionDescriptor.isAutoLoading()) {
@@ -264,9 +265,28 @@ public class Scenario implements Callable<ScenarioResult> {
                 .build()
         );
 
-        initializeScriptingEngine();
         logger.debug("Running control script for " + getScenarioName() + ".");
+        scenarioController = new ScenarioController(this,minMaturity);
+        try {
+            initializeScriptingEngine(scenarioController);
+            executeScenarioScripts();
+            long awaitCompletionTime = 86400 * 365 * 1000L;
+            logger.debug("Awaiting completion of scenario and activities for " + awaitCompletionTime + " millis.");
+            scenarioController.awaitCompletion(awaitCompletionTime);
+        } catch (Exception e) {
+            this.error=e;
+        } finally {
+            scenarioController.shutdown();
+        }
 
+        Runtime.getRuntime().removeShutdownHook(scenarioShutdownHook);
+        scenarioShutdownHook.run();
+    }
+
+    public void notifyException(Thread t, Throwable e) {
+        this.error=new RuntimeException("in thread " + t.getName() + ", " +e, e);
+    }
+    private void executeScenarioScripts() {
         for (String script : scripts) {
             try {
                 Object result = null;
@@ -304,6 +324,7 @@ public class Scenario implements Callable<ScenarioResult> {
                 System.err.flush();
                 System.out.flush();
             } catch (Exception e) {
+                this.error=e;
                 this.state = State.Errored;
                 logger.error("Error in scenario, shutting down. (" + e + ")");
                 try {
@@ -311,7 +332,6 @@ public class Scenario implements Callable<ScenarioResult> {
                 } catch (Exception eInner) {
                     logger.debug("Found inner exception while forcing stop with rethrow=false: " + eInner);
                 } finally {
-                    this.error = e;
                     throw new RuntimeException(e);
                 }
             } finally {
@@ -320,14 +340,6 @@ public class Scenario implements Callable<ScenarioResult> {
                 endedAtMillis = System.currentTimeMillis();
             }
         }
-        long awaitCompletionTime = 86400 * 365 * 1000L;
-        logger.debug("Awaiting completion of scenario and activities for " + awaitCompletionTime + " millis.");
-        scenarioController.awaitCompletion(awaitCompletionTime);
-        //TODO: Ensure control flow covers controller shutdown in event of internal error.
-
-        Runtime.getRuntime().removeShutdownHook(scenarioShutdownHook);
-        scenarioShutdownHook = null;
-        finish();
     }
 
     public void finish() {
@@ -370,31 +382,42 @@ public class Scenario implements Callable<ScenarioResult> {
     /**
      * This should be the only way to get a ScenarioResult for a Scenario.
      *
+     * The lifecycle of a scenario includes the lifecycles of all of the following:
+     * <OL>
+     * <LI>The scenario control script, executing within a graaljs context.</LI>
+     * <LI>The lifecycle of every activity which is started within the scenario.</LI>
+     * </OL>
+     *
+     * All of these run asynchronously within the scenario, however the same thread that calls
+     * the scenario is the one which executes the control script. A scenario ends when all
+     * of the following conditions are met:
+     * <UL>
+     * <LI>The scenario control script has run to completion, or experienced an exception.</LI>
+     * <LI>Each activity has run to completion, experienced an exception, or all</LI>
+     * </UL>
+     *
      * @return
      */
-    public synchronized ScenarioResult call() {
+    public synchronized ExecutionMetricsResult call() {
         if (result == null) {
             try {
                 runScenario();
             } catch (Exception e) {
-                if (this.error!=null) {
-                    logger.debug("OVERLAPPING ERRORS: prior" + this.error.getMessage() + ", current:" + e.getMessage());
-                }
-                this.error = e;
+                this.error=e;
             } finally {
-                logger.debug((this.error == null ? "NORMAL" : "ERRORED") + " scenario run");
+                logger.debug((this.error==null ? "NORMAL" : "ERRORED") + " scenario run");
             }
 
             String iolog = scriptEnv.getTimedLog();
-            this.result = new ScenarioResult(this.error, iolog, this.startedAtMillis, this.endedAtMillis);
-            result.reportToLog();
+            this.result = new ExecutionMetricsResult(this.startedAtMillis, this.endedAtMillis, iolog, error);
+            result.reportMetricsSummaryToLog();
             doReportSummaries(reportSummaryTo, result);
         }
 
         return result;
     }
 
-    private void doReportSummaries(String reportSummaryTo, ScenarioResult result) {
+    private void doReportSummaries(String reportSummaryTo, ExecutionMetricsResult result) {
         List<PrintStream> fullChannels = new ArrayList<>();
         List<PrintStream> briefChannels = new ArrayList<>();
 
@@ -437,7 +460,7 @@ public class Scenario implements Callable<ScenarioResult> {
                 }
             }
         }
-        fullChannels.forEach(result::reportTo);
+        fullChannels.forEach(result::reportMetricsSummaryTo);
 //        briefChannels.forEach(result::reportCountsTo);
     }
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/ScenarioController.java

@@ -13,46 +13,53 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.nosqlbench.engine.core.lifecycle;
+package io.nosqlbench.engine.core.lifecycle.scenario;
 
-import io.nosqlbench.engine.api.activityapi.core.Activity;
-import io.nosqlbench.engine.api.activityapi.core.ActivityType;
-import io.nosqlbench.engine.api.activityapi.core.RunState;
-import io.nosqlbench.engine.api.activityapi.core.progress.ProgressMeterDisplay;
-import io.nosqlbench.api.engine.activityimpl.ActivityDef;
-import io.nosqlbench.api.engine.activityimpl.ParameterMap;
-import io.nosqlbench.engine.api.activityimpl.uniform.StandardActivityType;
-import io.nosqlbench.api.engine.metrics.ActivityMetrics;
-import io.nosqlbench.engine.core.annotation.Annotators;
-import io.nosqlbench.nb.annotations.Maturity;
 import io.nosqlbench.api.annotations.Annotation;
 import io.nosqlbench.api.annotations.Layer;
-import io.nosqlbench.api.config.standard.ConfigSuggestions;
-import io.nosqlbench.api.errors.BasicError;
+import io.nosqlbench.api.engine.activityimpl.ActivityDef;
+import io.nosqlbench.api.engine.activityimpl.ParameterMap;
+import io.nosqlbench.api.engine.metrics.ActivityMetrics;
+import io.nosqlbench.engine.api.activityapi.core.Activity;
+import io.nosqlbench.engine.api.activityapi.core.progress.ProgressMeterDisplay;
+import io.nosqlbench.engine.core.annotation.Annotators;
+import io.nosqlbench.engine.core.lifecycle.ExecutionResult;
+import io.nosqlbench.engine.core.lifecycle.IndexedThreadFactory;
+import io.nosqlbench.engine.core.lifecycle.activity.*;
+import io.nosqlbench.nb.annotations.Maturity;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.security.InvalidParameterException;
 import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.*;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
- * A ScenarioController provides a way to start Activities, modify them while running, and forceStopMotors, pause or restart them.
+ * A ScenarioController provides a way to start Activities,
+ * modify them while running, and forceStopMotors, pause or restart them.
  */
 public class ScenarioController {
 
     private static final Logger logger = LogManager.getLogger(ScenarioController.class);
     private static final Logger scenariologger = LogManager.getLogger("SCENARIO");
 
-    private final Map<String, ActivityExecutor> activityExecutors = new ConcurrentHashMap<>();
-    private final String sessionId;
+    private final ActivityLoader activityLoader;
+
+    private final Map<String, ActivityRuntimeInfo> activityInfoMap = new ConcurrentHashMap<>();
+    private final Scenario scenario;
     private final Maturity minMaturity;
 
-    public ScenarioController(String sessionId, Maturity minMaturity) {
-        this.sessionId = sessionId;
+    private final ExecutorService activitiesExecutor;
+
+    public ScenarioController(Scenario scenario, Maturity minMaturity) {
+        this.scenario = scenario;
         this.minMaturity = minMaturity;
+        this.activityLoader = new ActivityLoader(scenario);
+
+        ActivitiesExceptionHandler exceptionHandler = new ActivitiesExceptionHandler(this);
+        IndexedThreadFactory indexedThreadFactory = new IndexedThreadFactory("ACTIVITY", exceptionHandler);
+        this.activitiesExecutor = Executors.newCachedThreadPool(indexedThreadFactory);
     }
 
     /**
@@ -63,7 +70,7 @@ public class ScenarioController {
      */
     public synchronized void start(ActivityDef activityDef) {
         Annotators.recordAnnotation(Annotation.newBuilder()
-            .session(sessionId)
+            .session(scenario.getScenarioName())
             .now()
             .layer(Layer.Activity)
             .label("alias", activityDef.getAlias())
@@ -71,11 +78,21 @@ public class ScenarioController {
             .detail("params", activityDef.toString())
             .build());
 
+        doStartActivity(activityDef);
+    }
 
-        ActivityExecutor activityExecutor = getActivityExecutor(activityDef, true);
-        scenariologger.debug("START " + activityDef.getAlias());
-        activityExecutor.startActivity();
 
+    private synchronized ActivityRuntimeInfo doStartActivity(ActivityDef activityDef) {
+        if (!this.activityInfoMap.containsKey(activityDef.getAlias())) {
+            Activity activity = this.activityLoader.loadActivity(activityDef);
+            ActivityExecutor executor = new ActivityExecutor(activity, this.scenario.getScenarioName());
+            Future<ExecutionResult> startedActivity = activitiesExecutor.submit(executor);
+            ActivityRuntimeInfo activityRuntimeInfo = new ActivityRuntimeInfo(activity, startedActivity, executor);
+            this.activityInfoMap.put(activity.getAlias(), activityRuntimeInfo);
+            executor.startActivity();
+            scenariologger.debug("STARTED " + activityDef.getAlias());
+        }
+        return this.activityInfoMap.get(activityDef.getAlias());
     }
 
     /**
@@ -101,18 +118,18 @@ public class ScenarioController {
 
     public synchronized void run(int timeout, Map<String, String> activityDefMap) {
         ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap));
-        run(timeout, ad);
+        run(ad, timeout);
     }
 
     /**
      * Synchronously run the defined activity with a timeout in seconds.
      *
-     * @param timeout     seconds to await completion of the activity.
+     * @param timeoutMs   seconds to await completion of the activity.
      * @param activityDef A definition for an activity to run
      */
-    public synchronized void run(int timeout, ActivityDef activityDef) {
+    public synchronized void run(ActivityDef activityDef, long timeoutMs) {
         Annotators.recordAnnotation(Annotation.newBuilder()
-            .session(sessionId)
+            .session(this.scenario.getScenarioName())
             .now()
             .layer(Layer.Activity)
             .label("alias", activityDef.getAlias())
@@ -120,18 +137,13 @@ public class ScenarioController {
             .detail("params", activityDef.toString())
             .build());
 
-        ActivityExecutor activityExecutor = getActivityExecutor(activityDef, true);
-        scenariologger.debug("RUN alias=" + activityDef.getAlias());
-        scenariologger.debug(" (RUN/START) alias=" + activityDef.getAlias());
-        activityExecutor.startActivity();
-        scenariologger.debug(" (RUN/AWAIT before) alias=" + activityDef.getAlias());
-        boolean completed = activityExecutor.awaitCompletion(timeout);
-        scenariologger.debug(" (RUN/AWAIT after) completed=" + activityDef.getAlias());
+        doStartActivity(activityDef);
+        awaitActivity(activityDef, timeoutMs);
     }
 
     public synchronized void run(int timeout, String activityDefString) {
         ActivityDef activityDef = ActivityDef.parseActivityDef(activityDefString);
-        run(timeout, activityDef);
+        run(activityDef, timeout);
     }
 
     public synchronized void run(Map<String, String> activityDefMap) {
@@ -144,7 +156,7 @@ public class ScenarioController {
 
 
     public synchronized void run(ActivityDef activityDef) {
-        run(Integer.MAX_VALUE, activityDef);
+        run(activityDef, Long.MAX_VALUE);
     }
 
 
@@ -153,9 +165,8 @@ public class ScenarioController {
     }
 
     public boolean isRunningActivity(ActivityDef activityDef) {
-
-        ActivityExecutor activityExecutor = getActivityExecutor(activityDef, false);
-        return activityExecutor != null && activityExecutor.isRunning();
+        ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(activityDef.getAlias());
+        return (runtimeInfo != null && runtimeInfo.isRunning());
     }
 
     public boolean isRunningActivity(Map<String, String> activityDefMap) {
@@ -172,7 +183,7 @@ public class ScenarioController {
      */
     public synchronized void stop(ActivityDef activityDef) {
         Annotators.recordAnnotation(Annotation.newBuilder()
-            .session(sessionId)
+            .session(this.scenario.getScenarioName())
             .now()
             .layer(Layer.Activity)
             .label("alias", activityDef.getAlias())
@@ -180,18 +191,14 @@ public class ScenarioController {
             .detail("params", activityDef.toString())
             .build());
 
-        ActivityExecutor activityExecutor = getActivityExecutor(activityDef, false);
-        if (activityExecutor == null) {
+        ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(activityDef.getAlias());
+        if (runtimeInfo == null) {
             throw new RuntimeException("could not stop missing activity:" + activityDef);
         }
-        RunState runstate = activityExecutor.getActivity().getRunState();
-        if (runstate != RunState.Running) {
-            logger.warn("NOT stopping activity '" + activityExecutor.getActivity().getAlias() + "' because it is in state '" + runstate + "'");
-            return;
-        }
 
         scenariologger.debug("STOP " + activityDef.getAlias());
-        activityExecutor.stopActivity();
+
+        runtimeInfo.stopActivity();
     }
 
     /**
@@ -228,76 +235,6 @@ public class ScenarioController {
         }
     }
 
-    /**
-     * Modify one of the parameters in a defined activity. Any observing activity components will be notified of the
-     * changes made to activity parameters.
-     *
-     * @param alias The name of an activity that is already known to the scenario.
-     * @param param The parameter name
-     * @param value a new parameter value
-     */
-    public synchronized void modify(String alias, String param, String value) {
-        if (param.equals("alias")) {
-            throw new InvalidParameterException("It is not allowed to change the name of an existing activity.");
-        }
-        ActivityExecutor activityExecutor = getActivityExecutor(alias);
-        ParameterMap params = activityExecutor.getActivityDef().getParams();
-        scenariologger.debug("SET (" + alias + "/" + param + ")=(" + value + ")");
-        params.set(param, value);
-    }
-
-    /**
-     * Apply any parameter changes to a defined activity, or start a new one.
-     * This method is syntactical sugar for scripting. Each of the parameters in the map
-     * is checked against existing values, and per-field modifications
-     * are applied one at a time, only if the values have changed.
-     *
-     * @param appliedParams Map of new values.
-     */
-    public synchronized void apply(Map<String, String> appliedParams) {
-        String alias = appliedParams.get("alias");
-
-        if (alias == null) {
-            throw new BasicError("alias must be provided");
-        }
-
-        ActivityExecutor executor = activityExecutors.get(alias);
-
-        if (executor == null) {
-            logger.info("started scenario from apply:" + alias);
-            start(appliedParams);
-            return;
-        }
-
-        ParameterMap previousMap = executor.getActivityDef().getParams();
-
-        for (String paramName : appliedParams.keySet()) {
-            String appliedVal = appliedParams.get(paramName);
-            Optional<String> prevVal = previousMap.getOptionalString(paramName);
-
-            if (!prevVal.isPresent() || !prevVal.get().equals(appliedVal)) {
-                logger.info("applying new value to activity '" + alias + "': '" + prevVal.get() + "' -> '" + appliedVal + "'");
-                previousMap.set(paramName, appliedVal);
-            }
-        }
-    }
-
-    /**
-     * Get the activity executor associated with the given alias. This should be used to find activitytypes
-     * which are presumed to be already defined.
-     *
-     * @param activityAlias The activity alias for the extant activity.
-     * @return the associated ActivityExecutor
-     * @throws RuntimeException a runtime exception if the named activity is not found
-     */
-    private ActivityExecutor getActivityExecutor(String activityAlias) {
-        Optional<ActivityExecutor> executor =
-            Optional.ofNullable(activityExecutors.get(activityAlias));
-        return executor.orElseThrow(
-            () -> new RuntimeException("ActivityExecutor for alias " + activityAlias + " not found.")
-        );
-
-    }
 
     private List<String> getMatchingAliases(String pattern) {
         Pattern matcher;
@@ -308,52 +245,13 @@ public class ScenarioController {
             matcher = Pattern.compile(pattern);
         }
 
-        List<String> matching = activityExecutors.keySet().stream()
+        List<String> matching = activityInfoMap.keySet().stream()
             .filter(a -> Pattern.matches(pattern, a))
             .peek(p -> logger.debug("MATCH " + pattern + " -> " + p))
             .collect(Collectors.toList());
         return matching;
     }
 
-    private ActivityExecutor getActivityExecutor(ActivityDef activityDef, boolean createIfMissing) {
-        synchronized (activityExecutors) {
-            ActivityExecutor executor = activityExecutors.get(activityDef.getAlias());
-
-            if (executor == null && createIfMissing) {
-                if (activityDef.getParams().containsKey("driver")) {
-                    ActivityType<?> activityType = new ActivityTypeLoader()
-                        .setMaturity(this.minMaturity)
-                        .load(activityDef)
-                        .orElseThrow(
-                            () -> new RuntimeException("Driver for '" + activityDef + "' was not found." +
-                                "\nYou can use --list-drivers to see what drivers are supported in this runtime." +
-                                ConfigSuggestions.suggestAlternates(
-                                        new ActivityTypeLoader().getAllSelectors(), activityDef.getActivityType(), 4)
-                                    .orElse("")
-                            )
-                        );
-
-                    executor = new ActivityExecutor(
-                        activityType.getAssembledActivity(
-                            activityDef,
-                            getActivityMap()
-                        ),
-                        this.sessionId
-                    );
-                    activityExecutors.put(activityDef.getAlias(), executor);
-                } else {
-                    executor = new ActivityExecutor(
-                        new StandardActivityType(activityDef).getAssembledActivity(
-                            activityDef, getActivityMap()
-                        ), this.sessionId
-                    );
-                }
-
-            }
-            return executor;
-        }
-    }
-
     /**
      * Wait for a bit. This is not the best approach, and will be replaced with a different system in the future.
      *
@@ -382,29 +280,7 @@ public class ScenarioController {
      * @return set of activity names
      */
     public Set<String> getAliases() {
-        return activityExecutors.keySet();
-    }
-
-    /**
-     * Return all the activity definitions that are known to this scenario.
-     *
-     * @return list of activity defs
-     */
-    public List<ActivityDef> getActivityDefs() {
-        return activityExecutors.values().stream()
-            .map(ActivityExecutor::getActivityDef)
-            .collect(Collectors.toList());
-    }
-
-    /**
-     * Get the named activity def, if it is known to this scenario.
-     *
-     * @param alias The name by which the activity is known to this scenario.
-     * @return an ActivityDef instance
-     * @throws RuntimeException if the alias is not known to the scenario
-     */
-    public ActivityDef getActivityDef(String alias) {
-        return getActivityExecutor(alias).getActivityDef();
+        return activityInfoMap.keySet();
     }
 
     /**
@@ -415,8 +291,9 @@ public class ScenarioController {
      * @param waitTimeMillis grace period during which an activity may cooperatively shut down
      */
     public synchronized void forceStopScenario(int waitTimeMillis, boolean rethrow) {
+        logger.debug("force stopping scenario " + this.scenario.getScenarioName());
+        activityInfoMap.values().forEach(a -> a.getActivityExecutor().forceStopActivity(10000));
         logger.debug("Scenario force stopped.");
-        activityExecutors.values().forEach(a -> a.forceStopScenarioAndThrow(waitTimeMillis, rethrow));
     }
 
 //    public synchronized void stopAll() {
@@ -425,7 +302,8 @@ public class ScenarioController {
 
     /**
      * Await completion of all running activities, but do not force shutdownActivity. This method is meant to provide
-     * the blocking point for calling logic. It waits.
+     * the blocking point for calling logic. It waits. If there is an error which should propagate into the scenario,
+     * then it should be thrown from this method.
      *
      * @param waitTimeMillis The time to wait, usually set very high
      * @return true, if all activities completed before the timer expired, false otherwise
@@ -433,31 +311,21 @@ public class ScenarioController {
     public boolean awaitCompletion(long waitTimeMillis) {
         logger.debug(() -> "awaiting completion");
         boolean completed = true;
-        long remaining = waitTimeMillis;
-
-        List<ActivityFinisher> finishers = new ArrayList<>();
-        for (ActivityExecutor ae : activityExecutors.values()) {
-            ActivityFinisher finisher = new ActivityFinisher(ae, (int) remaining);
-            finishers.add(finisher);
-            finisher.start();
-        }
-
-        for (ActivityFinisher finisher : finishers) {
-            try {
-                logger.debug("joining finisher " + finisher.getName());
-                finisher.join(waitTimeMillis);
-                logger.debug("joined finisher " + finisher.getName());
-            } catch (InterruptedException ignored) {
-            }
-        }
-
-        for (ActivityFinisher finisher : finishers) {
-            if (!finisher.getResult()) {
-                logger.debug("finisher for " + finisher.getName() + " did not signal TRUE");
+        for (ActivityRuntimeInfo activityRuntimeInfo : this.activityInfoMap.values()) {
+            ExecutionResult activityResult = activityRuntimeInfo.awaitResult(waitTimeMillis);
+            if (activityResult == null) {
+                logger.error("Unable to retrieve activity result for " + activityRuntimeInfo.getActivity().getAlias());
                 completed = false;
+            } else {
+                if (activityResult.getException()!=null) {
+                    if (activityResult.getException() instanceof RuntimeException e) {
+                        throw e;
+                    } else {
+                        throw new RuntimeException(activityResult.getException());
+                    }
+                }
             }
         }
-
         return completed;
     }
 
@@ -469,66 +337,98 @@ public class ScenarioController {
         }
     }
 
-    public boolean await(Map<String, String> activityDefMap) {
-        return this.awaitActivity(activityDefMap);
+    public void await(Map<String, String> activityDefMap) {
+        this.awaitActivity(activityDefMap);
     }
 
     public boolean awaitActivity(Map<String, String> activityDefMap) {
         ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap));
-        return awaitActivity(ad);
+        return awaitActivity(ad, Long.MAX_VALUE);
     }
 
     public boolean await(String alias) {
-        return this.awaitActivity(alias);
+        return this.awaitActivity(alias, Long.MAX_VALUE);
     }
 
-    public boolean awaitActivity(String alias) {
+    public boolean awaitActivity(String alias, long timeoutMs) {
         ActivityDef toAwait = aliasToDef(alias);
-        return awaitActivity(toAwait);
+        return awaitActivity(toAwait, Long.MAX_VALUE);
     }
 
-    public boolean await(ActivityDef activityDef) {
-        return this.awaitActivity(activityDef);
+    public void await(ActivityDef activityDef, long timeoutMs) {
+        this.awaitActivity(activityDef, timeoutMs);
     }
 
-    public boolean awaitActivity(ActivityDef activityDef) {
-        ActivityExecutor activityExecutor = getActivityExecutor(activityDef, false);
-        if (activityExecutor == null) {
-            throw new RuntimeException("Could not await missing activity: " + activityDef);
+    public boolean awaitActivity(ActivityDef activityDef, long timeoutMs) {
+        ActivityRuntimeInfo ari = this.activityInfoMap.get(activityDef.getAlias());
+        if (ari == null) {
+            throw new RuntimeException("Could not await missing activity: " + activityDef.getAlias());
         }
         scenariologger.debug("AWAIT/before alias=" + activityDef.getAlias());
-        boolean finished = activityExecutor.awaitFinish(Integer.MAX_VALUE);
-        scenariologger.debug("AWAIT/after  completed=" + finished);
-        return finished;
-
+        ExecutionResult result = null;
+        Future<ExecutionResult> future=null;
+        try {
+            future = ari.getFuture();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        try {
+            result = future.get(timeoutMs, TimeUnit.MILLISECONDS);
+        } catch (ExecutionException e) {
+            throw new RuntimeException(e);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        } catch (TimeoutException e) {
+            throw new RuntimeException(e);
+        }
+        return (result != null);
     }
 
     /**
      * @return an unmodifyable String to executor map of all activities known to this scenario
      */
-    public Map<String, ActivityExecutor> getActivityExecutorMap() {
-        return Collections.unmodifiableMap(activityExecutors);
+    public Map<String, ActivityRuntimeInfo> getActivityExecutorMap() {
+        return Collections.unmodifiableMap(activityInfoMap);
+    }
+
+    public List<ActivityDef> getActivityDefs() {
+        return activityInfoMap.values().stream().map(ari -> ari.getActivity().getActivityDef()).toList();
     }
 
     public void reportMetrics() {
         ActivityMetrics.reportTo(System.out);
     }
 
-    private Map<String, Activity> getActivityMap() {
-        Map<String, Activity> activityMap = new HashMap<String, Activity>();
-        for (Map.Entry<String, ActivityExecutor> entry : activityExecutors.entrySet()) {
-            activityMap.put(entry.getKey(), entry.getValue().getActivity());
-        }
-        return activityMap;
-    }
-
     public List<ProgressMeterDisplay> getProgressMeters() {
         List<ProgressMeterDisplay> indicators = new ArrayList<>();
-        for (ActivityExecutor ae : activityExecutors.values()) {
+        for (ActivityRuntimeInfo ae : activityInfoMap.values()) {
             indicators.add(ae.getProgressMeter());
         }
         indicators.sort(Comparator.comparing(ProgressMeterDisplay::getStartTime));
         return indicators;
     }
 
+    public void notifyException(Thread t, Throwable e) {
+        logger.error("Uncaught exception in activity lifecycle thread:" + e, e);
+        scenario.notifyException(t,e);
+        throw new RuntimeException(e);
+    }
+
+    public ActivityDef getActivityDef(String alias) {
+        return activityInfoMap.get(alias).getActivity().getActivityDef();
+    }
+
+    public void shutdown() {
+        this.activitiesExecutor.shutdown();
+        try {
+            if (!this.activitiesExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
+                this.activitiesExecutor.shutdownNow();
+                if (!this.activitiesExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
+                    throw new RuntimeException("Unable to shutdown activities executor");
+                }
+            }
+        } catch (Exception e) {
+
+        }
+    }
 }

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/ScenarioShutdownHook.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario;
 
 import org.apache.logging.log4j.Logger;
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/ScenariosExecutor.java

@@ -14,10 +14,12 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario;
 
-import io.nosqlbench.engine.core.lifecycle.*;
 import io.nosqlbench.api.errors.BasicError;
+import io.nosqlbench.engine.core.lifecycle.ExecutionMetricsResult;
+import io.nosqlbench.engine.core.lifecycle.IndexedThreadFactory;
+import io.nosqlbench.engine.core.lifecycle.scenario.script.ScenarioExceptionHandler;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -50,7 +52,7 @@ public class ScenariosExecutor {
         if (submitted.get(scenario.getScenarioName()) != null) {
             throw new BasicError("Scenario " + scenario.getScenarioName() + " is already defined. Remove it first to reuse the name.");
         }
-        Future<ScenarioResult> future = executor.submit(scenario);
+        Future<ExecutionMetricsResult> future = executor.submit(scenario);
         SubmittedScenario s = new SubmittedScenario(scenario, future);
         submitted.put(s.getName(), s);
     }
@@ -77,6 +79,7 @@ public class ScenariosExecutor {
      * @return the final scenario-result map
      */
     public ScenariosResults awaitAllResults(long timeout, long updateInterval) {
+        long waitFrom = System.currentTimeMillis();
         if (updateInterval > timeout) {
             throw new BasicError("timeout must be equal to or greater than updateInterval");
         }
@@ -96,6 +99,7 @@ public class ScenariosExecutor {
                     } catch (InterruptedException ignored) {
                     }
                 }
+                logger.trace("waited " + (System.currentTimeMillis()-waitFrom) + " millis for scenarios");
                 updateAt = Math.min(timeoutAt, System.currentTimeMillis() + updateInterval);
             }
 
@@ -106,7 +110,7 @@ public class ScenariosExecutor {
             throw new RuntimeException("executor still runningScenarios after awaiting all results for " + timeout
                 + "ms.  isTerminated:" + executor.isTerminated() + " isShutdown:" + executor.isShutdown());
         }
-        Map<Scenario, ScenarioResult> scenarioResultMap = new LinkedHashMap<>();
+        Map<Scenario, ExecutionMetricsResult> scenarioResultMap = new LinkedHashMap<>();
         getAsyncResultStatus()
             .entrySet()
             .forEach(
@@ -133,26 +137,26 @@ public class ScenariosExecutor {
      * All submitted scenarios are included. Those which are still pending
      * are returned with an empty option.</p>
      *
-     * <p>Results may be exceptional. If {@link ScenarioResult#getException()} is present,
+     * <p>Results may be exceptional. If {@link ExecutionMetricsResult#getException()} is present,
      * then the result did not complete normally.</p>
      *
      * @return map of async results, with incomplete results as Optional.empty()
      */
-    public Map<Scenario, Optional<ScenarioResult>> getAsyncResultStatus() {
+    public Map<Scenario, Optional<ExecutionMetricsResult>> getAsyncResultStatus() {
 
-        Map<Scenario, Optional<ScenarioResult>> optResults = new LinkedHashMap<>();
+        Map<Scenario, Optional<ExecutionMetricsResult>> optResults = new LinkedHashMap<>();
 
         for (SubmittedScenario submittedScenario : submitted.values()) {
-            Future<ScenarioResult> resultFuture = submittedScenario.getResultFuture();
+            Future<ExecutionMetricsResult> resultFuture = submittedScenario.getResultFuture();
 
-            Optional<ScenarioResult> oResult = Optional.empty();
+            Optional<ExecutionMetricsResult> oResult = Optional.empty();
             if (resultFuture.isDone()) {
                 try {
                     oResult = Optional.of(resultFuture.get());
                 } catch (Exception e) {
                     long now = System.currentTimeMillis();
                     logger.debug("creating exceptional scenario result from getAsyncResultStatus");
-                    oResult = Optional.of(new ScenarioResult(e, "errored output", now, now));
+                    oResult = Optional.of(new ExecutionMetricsResult(now, now, "errored output", e));
                 }
             }
 
@@ -179,7 +183,7 @@ public class ScenariosExecutor {
      * @param scenarioName the scenario name of interest
      * @return an optional result
      */
-    public Optional<Future<ScenarioResult>> getPendingResult(String scenarioName) {
+    public Optional<Future<ExecutionMetricsResult>> getPendingResult(String scenarioName) {
         return Optional.ofNullable(submitted.get(scenarioName)).map(s -> s.resultFuture);
     }
 
@@ -191,10 +195,7 @@ public class ScenariosExecutor {
         logger.debug("#stopScenario(name=" + scenarioName + ", rethrow="+ rethrow+")");
         Optional<Scenario> pendingScenario = getPendingScenario(scenarioName);
         if (pendingScenario.isPresent()) {
-            ScenarioController controller = pendingScenario.get().getScenarioController();
-            if (controller != null) {
-                controller.forceStopScenario(0, rethrow);
-            }
+            pendingScenario.get().getScenarioController().forceStopScenario(10000, true);
         } else {
             throw new RuntimeException("Unable to cancel scenario: " + scenarioName + ": not found");
         }
@@ -224,9 +225,9 @@ public class ScenariosExecutor {
 
     private static class SubmittedScenario {
         private final Scenario scenario;
-        private final Future<ScenarioResult> resultFuture;
+        private final Future<ExecutionMetricsResult> resultFuture;
 
-        SubmittedScenario(Scenario scenario, Future<ScenarioResult> resultFuture) {
+        SubmittedScenario(Scenario scenario, Future<ExecutionMetricsResult> resultFuture) {
             this.scenario = scenario;
             this.resultFuture = resultFuture;
         }
@@ -235,7 +236,7 @@ public class ScenariosExecutor {
             return scenario;
         }
 
-        Future<ScenarioResult> getResultFuture() {
+        Future<ExecutionMetricsResult> getResultFuture() {
             return resultFuture;
         }
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/ScenariosResults.java

@@ -14,10 +14,9 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.lifecycle;
+package io.nosqlbench.engine.core.lifecycle.scenario;
 
-import io.nosqlbench.engine.core.script.Scenario;
-import io.nosqlbench.engine.core.script.ScenariosExecutor;
+import io.nosqlbench.engine.core.lifecycle.ExecutionMetricsResult;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 
@@ -28,27 +27,26 @@ public class ScenariosResults {
 
     private static final Logger logger = LogManager.getLogger(ScenariosResults.class);
     private final String scenariosExecutorName;
-    private final Map<Scenario, ScenarioResult> scenarioResultMap = new LinkedHashMap<>();
+    private final Map<Scenario, ExecutionMetricsResult> scenarioResultMap = new LinkedHashMap<>();
 
 
     public ScenariosResults(ScenariosExecutor scenariosExecutor) {
         this.scenariosExecutorName = scenariosExecutor.getName();
     }
 
-    public ScenariosResults(ScenariosExecutor scenariosExecutor, Map<Scenario, ScenarioResult> map) {
+    public ScenariosResults(ScenariosExecutor scenariosExecutor, Map<Scenario, ExecutionMetricsResult> map) {
         this.scenariosExecutorName = scenariosExecutor.getName();
         scenarioResultMap.putAll(map);
     }
 
     public String getExecutionSummary() {
-        StringBuilder sb = new StringBuilder("executions: ");
-        sb.append(scenarioResultMap.size()).append(" scenarios, ");
-        sb.append(scenarioResultMap.values().stream().filter(r -> r.getException().isEmpty()).count()).append(" normal, ");
-        sb.append(scenarioResultMap.values().stream().filter(r -> r.getException().isPresent()).count()).append(" errored");
-        return sb.toString();
+        String sb = "executions: " + scenarioResultMap.size() + " scenarios, " +
+            scenarioResultMap.values().stream().filter(r -> r.getException()==null).count() + " normal, " +
+            scenarioResultMap.values().stream().filter(r -> r.getException()!=null).count() + " errored";
+        return sb;
     }
 
-    public ScenarioResult getOne() {
+    public ExecutionMetricsResult getOne() {
         if (this.scenarioResultMap.size() != 1) {
             throw new RuntimeException("getOne found " + this.scenarioResultMap.size() + " results instead of 1.");
         }
@@ -57,14 +55,14 @@ public class ScenariosResults {
     }
 
     public void reportToLog() {
-        for (Map.Entry<Scenario, ScenarioResult> entry : this.scenarioResultMap.entrySet()) {
+        for (Map.Entry<Scenario, ExecutionMetricsResult> entry : this.scenarioResultMap.entrySet()) {
             Scenario scenario = entry.getKey();
-            ScenarioResult oresult = entry.getValue();
+            ExecutionMetricsResult oresult = entry.getValue();
 
             logger.info("results for scenario: " + scenario);
 
             if (oresult != null) {
-                oresult.reportElapsedMillis();
+                oresult.reportElapsedMillisToLog();
             } else {
                 logger.error(scenario.getScenarioName() + ": incomplete (missing result)");
             }
@@ -74,7 +72,7 @@ public class ScenariosResults {
 
     public boolean hasError() {
         return this.scenarioResultMap.values().stream()
-                .anyMatch(r -> r.getException().isPresent());
+                .anyMatch(r -> r.getException()!=null);
     }
 
     public int getSize() {

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/MetricsMapper.java

@@ -13,15 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario.script;
 
 import com.codahale.metrics.*;
 import io.nosqlbench.engine.api.activityapi.core.Activity;
 import io.nosqlbench.engine.api.activityapi.core.ActivityType;
 import io.nosqlbench.api.engine.activityimpl.ActivityDef;
 import io.nosqlbench.api.engine.metrics.ActivityMetrics;
-import io.nosqlbench.engine.core.lifecycle.ActivityTypeLoader;
-import io.nosqlbench.engine.core.metrics.PolyglotMetricRegistryBindings;
+import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader;
+import io.nosqlbench.engine.core.lifecycle.scenario.script.bindings.PolyglotMetricRegistryBindings;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/SandboxExtensionFinder.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario.script;
 
 import io.nosqlbench.engine.api.extensions.ScriptingPluginInfo;
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/ScenarioContext.java

@@ -13,9 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario.script;
 
-import io.nosqlbench.engine.core.lifecycle.ScenarioController;
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenarioController;
 import io.nosqlbench.engine.api.scripting.ScriptEnvBuffer;
 
 public class ScenarioContext extends ScriptEnvBuffer {

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/ScenarioExceptionHandler.java

@@ -14,7 +14,9 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario.script;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosExecutor;
 
 public class ScenarioExceptionHandler implements Thread.UncaughtExceptionHandler {
     private final ScenariosExecutor scenariosExecutor;

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/ScriptExecutionError.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario.script;
 
 public class ScriptExecutionError extends RuntimeException {
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/ScriptParams.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario.script;
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
@@ -43,8 +43,7 @@ public class ScriptParams extends HashMap<String, String> implements ProxyObject
         Map<String, String> map;
         if (overrides instanceof Map) {
             map = (Map) overrides;
-        } else if (overrides instanceof Value) {
-            Value v = (Value) overrides;
+        } else if (overrides instanceof Value v) {
             map = v.as(Map.class);
         } else {
             throw new RuntimeException("Unrecognized overrides type: " + overrides.getClass().getCanonicalName());

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/bindings/ActivityBindings.java

@@ -13,10 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario.script.bindings;
 
 import io.nosqlbench.api.engine.activityimpl.ActivityDef;
-import io.nosqlbench.engine.core.lifecycle.ScenarioController;
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenarioController;
 import org.graalvm.polyglot.Value;
 import org.graalvm.polyglot.proxy.ProxyObject;
 
@@ -27,12 +27,12 @@ import java.util.stream.Collectors;
 /**
  * Provide a bindings wrapper around a ScenarioController,
  */
-public class NashornActivityBindings implements Bindings, ProxyObject {
+public class ActivityBindings implements Bindings, ProxyObject {
 
     private final ScenarioController scenario;
     private final Map<String, Bindings> elementMap = new HashMap<String, Bindings>();
 
-    public NashornActivityBindings(ScenarioController scenarioController) {
+    public ActivityBindings(ScenarioController scenarioController) {
         this.scenario = scenarioController;
     }
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/bindings/PolyglotMetricRegistryBindings.java

@@ -14,10 +14,11 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.metrics;
+package io.nosqlbench.engine.core.lifecycle.scenario.script.bindings;
 
 import com.codahale.metrics.*;
 import com.codahale.metrics.Timer;
+import io.nosqlbench.engine.core.metrics.MetricMap;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.graalvm.polyglot.Value;

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/bindings/PolyglotScenarioController.java

@@ -14,9 +14,10 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.lifecycle;
+package io.nosqlbench.engine.core.lifecycle.scenario.script.bindings;
 
 import io.nosqlbench.api.engine.activityimpl.ActivityDef;
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenarioController;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.graalvm.polyglot.Value;
@@ -80,7 +81,7 @@ public class PolyglotScenarioController {
     private synchronized void runValue(int timeout, Value spec) {
         logger.debug("run(Value) called with:" + spec);
         if (spec.isHostObject()) {
-            controller.run(timeout, (ActivityDef) spec.asHostObject());
+            controller.run(spec.asHostObject(),timeout);
         } else if (spec.isString()) {
             controller.run(timeout, spec.asString());
         } else if (spec.hasMembers()) {
@@ -88,7 +89,7 @@ public class PolyglotScenarioController {
         } else if (spec.isHostObject()) {
             Object o = spec.asHostObject();
             if (o instanceof ActivityDef) {
-                controller.run(timeout, (ActivityDef) o);
+                controller.run((ActivityDef) o, timeout);
             } else {
                 throw new RuntimeException("unrecognized polyglot host object type for run: " + spec);
             }
@@ -146,20 +147,6 @@ public class PolyglotScenarioController {
         }
     }
 
-    public synchronized void apply(Object o) {
-        if (o instanceof Value) {
-            applyValue((Value) o);
-        } else if (o instanceof Map) {
-            controller.apply((Map<String, String>) o);
-        } else {
-            throw new RuntimeException("unknown type: " + o.getClass().getCanonicalName());
-        }
-    }
-
-    private synchronized void applyValue(Value spec) {
-        Map<String, String> map = spec.as(Map.class);
-        controller.apply(map);
-    }
 
     public synchronized void awaitActivity(Object o) {
         this.await(o);
@@ -178,7 +165,7 @@ public class PolyglotScenarioController {
 
     private synchronized void awaitValue(Value spec) {
         if (spec.isHostObject()) {
-            controller.await((ActivityDef) spec.asHostObject());
+            controller.await(spec.asHostObject(), Long.MAX_VALUE);
         } else if (spec.hasMembers()) {
             controller.await(spec.as(Map.class));
         } else if (spec.isString()) {

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/bindings/ReadOnlyBindings.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario.script.bindings;
 
 import javax.script.Bindings;
 import java.util.Map;

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/bindings/ReadOnlyBindingsException.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.core.script;
+package io.nosqlbench.engine.core.lifecycle.scenario.script.bindings;
 
 public class ReadOnlyBindingsException extends RuntimeException {
     private final Object parent;

engine-core/src/main/java/io/nosqlbench/engine/core/metadata/MarkdownFinder.java

@@ -18,7 +18,7 @@ package io.nosqlbench.engine.core.metadata;
 
 import io.nosqlbench.engine.api.activityapi.core.ActivityType;
 import io.nosqlbench.api.engine.activityimpl.ActivityDef;
-import io.nosqlbench.engine.core.lifecycle.ActivityTypeLoader;
+import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader;
 import io.nosqlbench.nb.annotations.Service;
 import io.nosqlbench.api.content.Content;
 import io.nosqlbench.api.content.NBIO;
@@ -28,20 +28,20 @@ import org.apache.logging.log4j.Logger;
 
 import java.util.Optional;
 
-public class MarkdownDocInfo {
-    private final static Logger logger = LogManager.getLogger(MarkdownDocInfo.class);
+public class MarkdownFinder {
+    private final static Logger logger = LogManager.getLogger(MarkdownFinder.class);
 
     public static Optional<String> forHelpTopic(String topic) {
         String help = null;
         try {
-            help = new MarkdownDocInfo().forActivityInstance(topic);
+            help = new MarkdownFinder().forActivityInstance(topic);
             return Optional.ofNullable(help);
         } catch (Exception e) {
             logger.debug("Did not find help topic for activity instance: " + topic);
         }
 
         try {
-            help = new MarkdownDocInfo().forResourceMarkdown(topic, "docs/");
+            help = new MarkdownFinder().forResourceMarkdown(topic, "docs/");
             return Optional.ofNullable(help);
         } catch (Exception e) {
             logger.debug("Did not find help topic for generic markdown file: " + topic + "(.md)");

engine-core/src/main/java/io/nosqlbench/engine/core/metrics/MetricReporters.java

@@ -21,7 +21,7 @@ import com.codahale.metrics.graphite.Graphite;
 import com.codahale.metrics.graphite.GraphiteReporter;
 import io.nosqlbench.engine.api.activityapi.core.Shutdownable;
 import io.nosqlbench.api.engine.metrics.ActivityMetrics;
-import io.nosqlbench.engine.core.lifecycle.ShutdownManager;
+import io.nosqlbench.engine.core.lifecycle.process.ShutdownManager;
 import io.nosqlbench.engine.core.logging.Log4JMetricsReporter;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
@@ -54,7 +54,7 @@ public class MetricReporters implements Shutdownable {
     }
 
     public MetricReporters addGraphite(String dest, String prefix) {
-        logger.debug("Adding graphite reporter to " + dest + " with prefix " + prefix);
+        logger.debug(() -> "Adding graphite reporter to " + dest + " with prefix " + prefix);
         if (dest.indexOf(":")>=0) {
             String[] split = dest.split(":");
             addGraphite(split[0],Integer.valueOf(split[1]),prefix);
@@ -65,7 +65,7 @@ public class MetricReporters implements Shutdownable {
     }
 
     public void addCSVReporter(String directoryName, String prefix) {
-        logger.debug("Adding CSV reporter to " + directoryName + " with prefix " + prefix);
+        logger.debug(() -> "Adding CSV reporter to " + directoryName + " with prefix " + prefix);
 
         if (metricRegistries.isEmpty()) {
             throw new RuntimeException("There are no metric registries.");

engine-core/src/main/resources/log4j2.xml

@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
 <Configuration status="debug" strict="true" name="XMLConfigTest"
                packages="org.apache.logging.log4j.test">
 
@@ -28,7 +44,7 @@
 
   <Loggers>
 
-    <Logger name="io.nosqlbench.docsys" level="info" additivity="false" specificity="trace">
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
       <AppenderRef ref="APPSLOG"/>
     </Logger>
 
@@ -38,4 +54,4 @@
 
   </Loggers>
 
-</Configuration>
+</Configuration>

engine-core/src/test/java/io/nosqlbench/engine/core/ActivityExecutorTest.java

@@ -16,25 +16,30 @@
 
 package io.nosqlbench.engine.core;
 
+import io.nosqlbench.api.engine.activityimpl.ActivityDef;
 import io.nosqlbench.engine.api.activityapi.core.*;
-import io.nosqlbench.engine.api.activityapi.output.OutputDispenser;
 import io.nosqlbench.engine.api.activityapi.input.Input;
 import io.nosqlbench.engine.api.activityapi.input.InputDispenser;
-import io.nosqlbench.api.engine.activityimpl.ActivityDef;
+import io.nosqlbench.engine.api.activityapi.output.OutputDispenser;
 import io.nosqlbench.engine.api.activityimpl.CoreServices;
 import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
 import io.nosqlbench.engine.api.activityimpl.action.CoreActionDispenser;
-import io.nosqlbench.engine.api.activityimpl.input.CoreInputDispenser;
 import io.nosqlbench.engine.api.activityimpl.input.AtomicInput;
+import io.nosqlbench.engine.api.activityimpl.input.CoreInputDispenser;
 import io.nosqlbench.engine.api.activityimpl.motor.CoreMotor;
 import io.nosqlbench.engine.api.activityimpl.motor.CoreMotorDispenser;
-import io.nosqlbench.engine.core.lifecycle.ActivityExecutor;
-import io.nosqlbench.engine.core.lifecycle.ActivityTypeLoader;
-import org.apache.logging.log4j.Logger;
+import io.nosqlbench.engine.core.lifecycle.ExecutionResult;
+import io.nosqlbench.engine.core.lifecycle.activity.ActivityExecutor;
+import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.Test;
 
 import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -54,13 +59,22 @@ public class ActivityExecutorTest {
         a.setOutputDispenserDelegate(tdisp);
         a.setInputDispenserDelegate(idisp);
         a.setMotorDispenserDelegate(mdisp);
-
+        ExecutorService executor = Executors.newCachedThreadPool();
         ActivityExecutor ae = new ActivityExecutor(a, "test-restart");
-        ad.setThreads(1);
-        ae.startActivity();
-        ae.stopActivity();
-        ae.startActivity();
-        ae.awaitCompletion(15000);
+        Future<ExecutionResult> future = executor.submit(ae);
+        try {
+            ad.setThreads(1);
+            ae.startActivity();
+            ae.stopActivity();
+            ae.startActivity();
+            ae.startActivity();
+            ExecutionResult executionResult = future.get();
+            Thread.sleep(500L);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        System.out.print("ad.setThreads(1)");
+        executor.shutdown();
         assertThat(idisp.getInput(10).getInputSegment(3)).isNull();
 
     }
@@ -80,9 +94,19 @@ public class ActivityExecutorTest {
         a.setMotorDispenserDelegate(mdisp);
 
         ActivityExecutor ae = new ActivityExecutor(a, "test-delayed-start");
-        ad.setThreads(1);
-        ae.startActivity();
-        ae.awaitCompletion(15000);
+        ExecutorService testExecutor = Executors.newCachedThreadPool();
+        Future<ExecutionResult> future = testExecutor.submit(ae);
+
+        try {
+            ad.setThreads(1);
+            ae.startActivity();
+            ExecutionResult result = future.get();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        } catch (ExecutionException e) {
+            throw new RuntimeException(e);
+        }
+        testExecutor.shutdownNow();
         assertThat(idisp.getInput(10).getInputSegment(3)).isNull();
 
     }
@@ -93,7 +117,7 @@ public class ActivityExecutorTest {
         Optional<ActivityType> activityType = new ActivityTypeLoader().load(ad);
         Input longSupplier = new AtomicInput(ad);
         MotorDispenser<?> cmf = getActivityMotorFactory(
-                ad, motorActionDelay(999), longSupplier
+            ad, motorActionDelay(999), longSupplier
         );
         Activity a = new SimpleActivity(ad);
         InputDispenser idisp = new CoreInputDispenser(a);
@@ -108,11 +132,11 @@ public class ActivityExecutorTest {
         ad.setThreads(5);
         ae.startActivity();
 
-        int[] speeds = new int[]{1,2000,5,2000,2,2000};
-        for(int offset=0; offset<speeds.length; offset+=2) {
-            int threadTarget=speeds[offset];
-            int threadTime = speeds[offset+1];
-            logger.info("Setting thread level to " + threadTarget + " for " +threadTime + " seconds.");
+        int[] speeds = new int[]{1, 2000, 5, 2000, 2, 2000};
+        for (int offset = 0; offset < speeds.length; offset += 2) {
+            int threadTarget = speeds[offset];
+            int threadTime = speeds[offset + 1];
+            logger.info("Setting thread level to " + threadTarget + " for " + threadTime + " seconds.");
             ad.setThreads(threadTarget);
             try {
                 Thread.sleep(threadTime);

engine-core/src/test/java/io/nosqlbench/engine/core/CoreMotorTest.java

@@ -38,7 +38,6 @@ public class CoreMotorTest {
         Motor cm = new CoreMotor(activity, 5L, lockstepper);
         AtomicLong observableAction = new AtomicLong(-3L);
         cm.setAction(getTestConsumer(observableAction));
-        cm.getSlotStateTracker().enterState(RunState.Starting);
         Thread t = new Thread(cm);
         t.setName("TestMotor");
         t.start();
@@ -58,7 +57,6 @@ public class CoreMotorTest {
         AtomicLongArray ary = new AtomicLongArray(10);
         Action a1 = getTestArrayConsumer(ary);
         cm1.setAction(a1);
-        cm1.getSlotStateTracker().enterState(RunState.Starting);
 
         Thread t1 = new Thread(cm1);
         t1.setName("cm1");

engine-core/src/test/java/io/nosqlbench/engine/core/ScenarioTest.java

@@ -17,7 +17,7 @@
 package io.nosqlbench.engine.core;
 
 import io.nosqlbench.engine.api.scripting.ScriptEnvBuffer;
-import io.nosqlbench.engine.core.script.Scenario;
+import io.nosqlbench.engine.core.lifecycle.scenario.Scenario;
 import io.nosqlbench.nb.annotations.Maturity;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;

engine-core/src/test/java/io/nosqlbench/engine/core/script/ScenariosExecutorTest.java

@@ -16,7 +16,9 @@
 
 package io.nosqlbench.engine.core.script;
 
-import io.nosqlbench.engine.core.lifecycle.ScenariosResults;
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosResults;
+import io.nosqlbench.engine.core.lifecycle.scenario.Scenario;
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosExecutor;
 import io.nosqlbench.nb.annotations.Maturity;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;

engine-core/src/test/java/io/nosqlbench/engine/core/script/ScriptParamsTest.java

@@ -17,12 +17,12 @@
 package io.nosqlbench.engine.core.script;
 
 import io.nosqlbench.api.errors.BasicError;
+import io.nosqlbench.engine.core.lifecycle.scenario.script.ScriptParams;
 import org.junit.jupiter.api.Test;
 
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 
 public class ScriptParamsTest {

engine-core/src/test/resources/log4j-test.xml

@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="debug" strict="true" name="XMLConfigTest"
-               packages="org.apache.logging.log4j.test">
-
-  <Filter type="ThresholdFilter" level="trace"/>
-
-  <Appenders>
-    <Appender type="Console" name="STDOUT">
-      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
-      <Filters>
-        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
-        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
-      </Filters>
-    </Appender>
-    <Appender type="Console" name="FLOW">
-      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
-      <Filters>
-        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
-        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
-      </Filters>
-    </Appender>
-    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
-      <Layout type="PatternLayout">
-        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
-      </Layout>
-    </Appender>
-  </Appenders>
-
-  <Loggers>
-
-    <Logger name="io.nosqlbench.docsys" level="info" additivity="false" specificity="trace">
-      <AppenderRef ref="APPSLOG"/>
-    </Logger>
-
-    <Root level="trace">
-      <AppenderRef ref="STDOUT"/>
-    </Root>
-
-  </Loggers>
-
-</Configuration>

engine-core/src/test/resources/log4j2-test.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<Configuration status="${project.test_logstatuslevel}" strict="true" name="XMLConfigTest"
+               packages="org.apache.logging.log4j.test">
+
+  <Filter type="ThresholdFilter" level="trace"/>
+
+  <Appenders>
+    <Appender type="Console" name="STDOUT">
+      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
+      </Filters>
+    </Appender>
+    <Appender type="Console" name="FLOW">
+      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
+      </Filters>
+    </Appender>
+    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
+      <Layout type="PatternLayout">
+        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
+      </Layout>
+    </Appender>
+  </Appenders>
+
+  <Loggers>
+
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
+      <AppenderRef ref="APPSLOG"/>
+    </Logger>
+
+    <Root level="${project.testlevel}">
+      <AppenderRef ref="STDOUT"/>
+    </Root>
+
+  </Loggers>
+
+</Configuration>

engine-extensions/src/test/java/io/nosqlbench/optimizers/TestOptimoExperiments.java

@@ -21,6 +21,8 @@ import org.apache.commons.math3.optim.*;
 import org.apache.commons.math3.optim.nonlinear.scalar.GoalType;
 import org.apache.commons.math3.optim.nonlinear.scalar.ObjectiveFunction;
 import org.apache.commons.math3.optim.nonlinear.scalar.noderiv.BOBYQAOptimizer;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.Test;
 
 import java.util.Arrays;
@@ -52,6 +54,7 @@ import java.util.Random;
  */
 public class TestOptimoExperiments {
 
+    private final static Logger logger = LogManager.getLogger(TestOptimoExperiments.class);
     @Test
     public void testNewAlgo() {
 
@@ -81,7 +84,7 @@ public class TestOptimoExperiments {
         );
         PointValuePair result = mo.optimize(od.toArray(new OptimizationData[0]));
 
-        System.out.println(
+        logger.debug(
                 "point:" + Arrays.toString(result.getPoint()) +
                         " value=" + m.value(result.getPoint())
         );

engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorEndpoint.java

@@ -21,9 +21,9 @@ import io.nosqlbench.engine.cli.BasicScriptBuffer;
 import io.nosqlbench.engine.cli.Cmd;
 import io.nosqlbench.engine.cli.NBCLICommandParser;
 import io.nosqlbench.engine.cli.ScriptBuffer;
-import io.nosqlbench.engine.core.lifecycle.ScenarioResult;
-import io.nosqlbench.engine.core.script.Scenario;
-import io.nosqlbench.engine.core.script.ScenariosExecutor;
+import io.nosqlbench.engine.core.lifecycle.ExecutionMetricsResult;
+import io.nosqlbench.engine.core.lifecycle.scenario.Scenario;
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosExecutor;
 import io.nosqlbench.engine.rest.services.WorkSpace;
 import io.nosqlbench.engine.rest.services.WorkspaceFinder;
 import io.nosqlbench.engine.rest.transfertypes.LiveScenarioView;
@@ -234,8 +234,8 @@ public class ScenarioExecutorEndpoint implements WebServiceObject {
         Optional<Scenario> pendingScenario = executor.getPendingScenario(scenarioName);
 
         if (pendingScenario.isPresent()) {
-            Optional<Future<ScenarioResult>> pendingResult = executor.getPendingResult(scenarioName);
-            Future<ScenarioResult> scenarioResultFuture = pendingResult.get();
+            Optional<Future<ExecutionMetricsResult>> pendingResult = executor.getPendingResult(scenarioName);
+            Future<ExecutionMetricsResult> scenarioResultFuture = pendingResult.get();
             return new LiveScenarioView(pendingScenario.get());
         } else {
             throw new RuntimeException("Scenario name '" + scenarioName + "' not found.");

engine-rest/src/main/java/io/nosqlbench/engine/rest/services/openapi/OpenApiLoader.java

@@ -27,6 +27,8 @@ import io.swagger.v3.oas.models.PathItem;
 import io.swagger.v3.oas.models.Paths;
 import io.swagger.v3.parser.core.models.ParseOptions;
 import io.swagger.v3.parser.core.models.SwaggerParseResult;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.yaml.snakeyaml.DumperOptions;
 import org.yaml.snakeyaml.Yaml;
 import org.yaml.snakeyaml.introspector.BeanAccess;
@@ -39,6 +41,7 @@ import java.util.stream.Collectors;
  * .md#securityRequirementObject">OpenApi Spec 3.1.0</A>
  */
 public class OpenApiLoader {
+    private final static Logger logger = LogManager.getLogger(OpenApiLoader.class);
 
     private static final OpenAPIParser parser = new OpenAPIParser();
     private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
@@ -91,7 +94,7 @@ public class OpenApiLoader {
         yaml.setBeanAccess(BeanAccess.DEFAULT);
 
         for (PathOp activeOp : activeOps) {
-            System.out.println("yaml for op:" + yaml.dump(activeOp));
+            logger.debug("yaml for op:" + yaml.dump(activeOp));
             pathops.put(activeOp.getCall(), activeOp);
         }
 

engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/LiveScenarioView.java

@@ -19,7 +19,7 @@ package io.nosqlbench.engine.rest.transfertypes;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyDescription;
 import io.nosqlbench.engine.api.activityapi.core.progress.ProgressMeterDisplay;
-import io.nosqlbench.engine.core.script.Scenario;
+import io.nosqlbench.engine.core.lifecycle.scenario.Scenario;
 
 import java.util.ArrayList;
 import java.util.Collection;

engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ResultView.java

@@ -16,18 +16,18 @@
 
 package io.nosqlbench.engine.rest.transfertypes;
 
-import io.nosqlbench.engine.core.lifecycle.ScenarioResult;
+import io.nosqlbench.engine.core.lifecycle.ExecutionMetricsResult;
 
 public class ResultView {
 
-    private final ScenarioResult result;
+    private final ExecutionMetricsResult result;
 
-    public ResultView(ScenarioResult result) {
+    public ResultView(ExecutionMetricsResult result) {
         this.result = result;
     }
 
     public String getIOLog() {
-        if (result!=null) {
+        if (result != null) {
             return result.getIOLog();
         } else {
             return "";
@@ -35,8 +35,8 @@ public class ResultView {
     }
 
     public String getError() {
-        if (result!=null && result.getException().isPresent()) {
-            return result.getException().get().getMessage();
+        if (result != null && result.getException()!=null) {
+            return result.getException().getMessage();
         }
         return "";
     }

engine-rest/src/test/java/io/nosqlbench/engine/rest/services/openapi/OpenApiLoaderTest.java

@@ -16,10 +16,12 @@
 
 package io.nosqlbench.engine.rest.services.openapi;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.Test;
 
 public class OpenApiLoaderTest {
-
+    private final static Logger logger = LogManager.getLogger(OpenApiLoaderTest.class);
     @Test
     public void testYamlGenerator() {
         String openidpath = "stargate.yaml";
@@ -28,7 +30,7 @@ public class OpenApiLoaderTest {
             "}\n";
 
         String result = OpenApiLoader.generateWorkloadFromFilepath(openidpath, filterJson);
-        System.out.println(result);
+        logger.debug(result);
 
     }
 

engine-rest/src/test/resources/log4j2-test.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<Configuration status="${project.test_logstatuslevel}" strict="true" name="XMLConfigTest"
+               packages="org.apache.logging.log4j.test">
+
+  <Filter type="ThresholdFilter" level="info"/>
+
+  <Appenders>
+    <Appender type="Console" name="STDOUT">
+      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
+      </Filters>
+    </Appender>
+    <Appender type="Console" name="FLOW">
+      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
+      </Filters>
+    </Appender>
+    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
+      <Layout type="PatternLayout">
+        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
+      </Layout>
+    </Appender>
+  </Appenders>
+
+  <Loggers>
+
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
+      <AppenderRef ref="APPSLOG"/>
+    </Logger>
+
+    <Root level="${project.testlevel}">
+      <AppenderRef ref="STDOUT"/>
+    </Root>
+
+  </Loggers>
+
+</Configuration>

mvn-defaults/pom.xml

@@ -23,6 +23,11 @@
     <packaging>pom</packaging>
 
     <properties>
+        <!-- Set this level to override the logging level for tests during build -->
+        <project.testlevel>INFO</project.testlevel>
+        <!-- Set this level to override the logging level for tests logging configuration during build -->
+        <project.test_logstatuslevel>INFO</project.test_logstatuslevel>
+
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <javadoc.name>nosqlbench</javadoc.name>
@@ -463,6 +468,12 @@
     </dependencies>
 
     <build>
+        <testResources>
+            <testResource>
+                <directory>src/test/resources</directory>
+                <filtering>true</filtering>
+            </testResource>
+        </testResources>
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -596,7 +607,7 @@
             <plugin>
                 <groupId>org.apache.rat</groupId>
                 <artifactId>apache-rat-plugin</artifactId>
-                <version>0.13</version>
+                <version>0.15</version>
                 <executions>
                     <execution>
                         <phase>verify</phase>

nb-api/src/main/java/io/nosqlbench/api/config/standard/ConfigModel.java

@@ -131,10 +131,20 @@ public class ConfigModel implements NBConfigModel {
                     return (T) Double.valueOf(string);
                 } else if (type == BigDecimal.class) {
                     return (T) BigDecimal.valueOf(Double.parseDouble(string));
+                } else if (type == boolean.class || type == Boolean.class) {
+                    return (T) Boolean.valueOf(Boolean.parseBoolean(string));
                 } else {
                     throw new RuntimeException("CharSequence type " + type.getSimpleName() + " could " +
                         " not be converted from " + value.getClass().getSimpleName());
                 }
+            } else if (value instanceof Boolean bool) {
+                if (type == boolean.class) {
+                    return (T) bool;
+                } else {
+                    throw new RuntimeException("Boolean type " + type.getSimpleName() + " could " +
+                        " not be converted from " + value.getClass().getSimpleName());
+
+                }
             }
 
         } catch (Exception e) {
@@ -318,11 +328,10 @@ public class ConfigModel implements NBConfigModel {
 
     @Override
     public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("[").append(
-                params.stream().map(p -> p.getNames().get(0)).collect(Collectors.joining(",")))
-            .append("]");
+        String sb = "[" +
+            params.stream().map(p -> p.getNames().get(0)).collect(Collectors.joining(",")) +
+            "]";
 
-        return sb.toString();
+        return sb;
     }
 }

nbr-examples/src/test/java/io/nosqlbench/nbr/examples/ScriptExampleTests.java

@@ -16,10 +16,10 @@
 
 package io.nosqlbench.nbr.examples;
 
-import io.nosqlbench.engine.core.lifecycle.ScenarioResult;
-import io.nosqlbench.engine.core.lifecycle.ScenariosResults;
-import io.nosqlbench.engine.core.script.Scenario;
-import io.nosqlbench.engine.core.script.ScenariosExecutor;
+import io.nosqlbench.engine.core.lifecycle.ExecutionMetricsResult;
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosResults;
+import io.nosqlbench.engine.core.lifecycle.scenario.Scenario;
+import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosExecutor;
 import io.nosqlbench.nb.annotations.Maturity;
 import org.apache.commons.compress.utils.IOUtils;
 import org.assertj.core.data.Offset;
@@ -41,7 +41,7 @@ import java.util.stream.Collectors;
 import static org.assertj.core.api.Assertions.assertThat;
 public class ScriptExampleTests {
 
-    public static ScenarioResult runScenario(String scriptname, String... params) {
+    public static ExecutionMetricsResult runScenario(String scriptname, String... params) {
         if ((params.length % 2) != 0) {
             throw new RuntimeException("params must be pairwise key, value, ...");
         }
@@ -74,7 +74,7 @@ public class ScriptExampleTests {
 //        s.addScriptText("load('classpath:scripts/async/" + scriptname + ".js');");
         executor.execute(s);
         ScenariosResults scenariosResults = executor.awaitAllResults();
-        ScenarioResult scenarioResult = scenariosResults.getOne();
+        ExecutionMetricsResult scenarioResult = scenariosResults.getOne();
         executor.shutdownNow();
         return scenarioResult;
     }
@@ -86,7 +86,7 @@ public class ScriptExampleTests {
 
     @Test
     public void testLinkedInput() {
-        ScenarioResult scenarioResult = runScenario("linkedinput");
+        ExecutionMetricsResult scenarioResult = runScenario("linkedinput");
         Pattern p = Pattern.compile(".*started leader.*started follower.*stopped leader.*stopped follower.*",
             Pattern.DOTALL);
         assertThat(p.matcher(scenarioResult.getIOLog()).matches()).isTrue();
@@ -94,14 +94,14 @@ public class ScriptExampleTests {
 
     @Test
     public void testExceptionPropagationFromMotorThread() {
-        ScenarioResult scenarioResult = runScenario("activityerror");
-        assertThat(scenarioResult.getException()).isPresent();
-        assertThat(scenarioResult.getException().get().getMessage()).contains("For input string: \"unparsable\"");
+        ExecutionMetricsResult scenarioResult = runScenario("activityerror");
+        assertThat(scenarioResult.getException()).isNotNull();
+        assertThat(scenarioResult.getException().getMessage()).contains("For input string: \"unparsable\"");
     }
 
     @Test
     public void testCycleRate() {
-        ScenarioResult scenarioResult = runScenario("cycle_rate");
+        ExecutionMetricsResult scenarioResult = runScenario("cycle_rate");
         String iolog = scenarioResult.getIOLog();
         System.out.println("iolog\n" + iolog);
         Pattern p = Pattern.compile(".*mean cycle rate = (\\d[.\\d]+).*", Pattern.DOTALL);
@@ -116,13 +116,13 @@ public class ScriptExampleTests {
 
     @Test
     public void testExtensionPoint() {
-        ScenarioResult scenarioResult = runScenario("extensions");
+        ExecutionMetricsResult scenarioResult = runScenario("extensions");
         assertThat(scenarioResult.getIOLog()).contains("sum is 46");
     }
 
     @Test
     public void testOptimo() {
-        ScenarioResult scenarioResult = runScenario("optimo");
+        ExecutionMetricsResult scenarioResult = runScenario("optimo");
         String iolog = scenarioResult.getIOLog();
         System.out.println("iolog\n" + iolog);
         assertThat(iolog).contains("map of result was");
@@ -130,14 +130,14 @@ public class ScriptExampleTests {
 
     @Test
     public void testExtensionCsvLogger() {
-        ScenarioResult scenarioResult = runScenario("extension_csvmetrics");
+        ExecutionMetricsResult scenarioResult = runScenario("extension_csvmetrics");
         assertThat(scenarioResult.getIOLog()).contains("started new " +
             "csvlogger: logs/csvmetricstestdir");
     }
 
     @Test
     public void testScriptParamsVariable() {
-        ScenarioResult scenarioResult = runScenario("params_variable", "one", "two", "three", "four");
+        ExecutionMetricsResult scenarioResult = runScenario("params_variable", "one", "two", "three", "four");
         assertThat(scenarioResult.getIOLog()).contains("params[\"one\"]='two'");
         assertThat(scenarioResult.getIOLog()).contains("params[\"three\"]='four'");
         assertThat(scenarioResult.getIOLog()).contains("overridden[\"three\"] [overridden-three-five]='five'");
@@ -146,7 +146,7 @@ public class ScriptExampleTests {
 
     @Test
     public void testScriptParamsUndefVariableWithOverride() {
-        ScenarioResult scenarioResult = runScenario("undef_param", "one", "two", "three", "four");
+        ExecutionMetricsResult scenarioResult = runScenario("undef_param", "one", "two", "three", "four");
         assertThat(scenarioResult.getIOLog()).contains("before: params[\"three\"]:four");
         assertThat(scenarioResult.getIOLog()).contains("before: params.three:four");
         assertThat(scenarioResult.getIOLog()).contains("after: params[\"three\"]:undefined");
@@ -155,7 +155,7 @@ public class ScriptExampleTests {
 
     @Test
     public void testExtensionHistoStatsLogger() throws IOException {
-        ScenarioResult scenarioResult = runScenario("extension_histostatslogger");
+        ExecutionMetricsResult scenarioResult = runScenario("extension_histostatslogger");
         assertThat(scenarioResult.getIOLog()).contains("stdout started " +
             "logging to logs/histostats.csv");
         List<String> strings = Files.readAllLines(Paths.get(
@@ -167,7 +167,7 @@ public class ScriptExampleTests {
 
     @Test
     public void testExtensionCsvOutput() throws IOException {
-        ScenarioResult scenarioResult = runScenario("extension_csvoutput");
+        ExecutionMetricsResult scenarioResult = runScenario("extension_csvoutput");
         List<String> strings = Files.readAllLines(Paths.get(
             "logs/csvoutputtestfile.csv"));
         String logdata = strings.stream().collect(Collectors.joining("\n"));
@@ -177,7 +177,7 @@ public class ScriptExampleTests {
 
     @Test
     public void testExtensionHistogramLogger() throws IOException {
-        ScenarioResult scenarioResult = runScenario("extension_histologger");
+        ExecutionMetricsResult scenarioResult = runScenario("extension_histologger");
         assertThat(scenarioResult.getIOLog()).contains("stdout started logging to hdrhistodata.log");
         List<String> strings = Files.readAllLines(Paths.get("hdrhistodata.log"));
         String logdata = strings.stream().collect(Collectors.joining("\n"));
@@ -187,7 +187,7 @@ public class ScriptExampleTests {
 
     @Test
     public void testBlockingRun() {
-        ScenarioResult scenarioResult = runScenario("blockingrun");
+        ExecutionMetricsResult scenarioResult = runScenario("blockingrun");
         int a1end = scenarioResult.getIOLog().indexOf("blockingactivity1 finished");
         int a2start = scenarioResult.getIOLog().indexOf("running blockingactivity2");
         assertThat(a1end).isLessThan(a2start);
@@ -195,12 +195,12 @@ public class ScriptExampleTests {
 
     @Test
     public void testAwaitFinished() {
-        ScenarioResult scenarioResult = runScenario("awaitfinished");
+        ExecutionMetricsResult scenarioResult = runScenario("awaitfinished");
     }
 
     @Test
     public void testStartStop() {
-        ScenarioResult scenarioResult = runScenario("startstopdiag");
+        ExecutionMetricsResult scenarioResult = runScenario("startstopdiag");
         int startedAt = scenarioResult.getIOLog().indexOf("starting activity teststartstopdiag");
         int stoppedAt = scenarioResult.getIOLog().indexOf("stopped activity teststartstopdiag");
         assertThat(startedAt).isGreaterThan(0);
@@ -210,7 +210,7 @@ public class ScriptExampleTests {
     // TODO: find out why this causes a long delay after stop is called.
     @Test
     public void testThreadChange() {
-        ScenarioResult scenarioResult = runScenario("threadchange");
+        ExecutionMetricsResult scenarioResult = runScenario("threadchange");
         int changedTo1At = scenarioResult.getIOLog().indexOf("threads now 1");
         int changedTo5At = scenarioResult.getIOLog().indexOf("threads now 5");
         System.out.println("IOLOG:\n"+scenarioResult.getIOLog());
@@ -220,13 +220,13 @@ public class ScriptExampleTests {
 
     @Test
     public void testReadMetric() {
-        ScenarioResult scenarioResult = runScenario("readmetrics");
+        ExecutionMetricsResult scenarioResult = runScenario("readmetrics");
         assertThat(scenarioResult.getIOLog()).contains("count: ");
     }
 
     @Test
     public void testShutdownHook() {
-        ScenarioResult scenarioResult = runScenario("extension_shutdown_hook");
+        ExecutionMetricsResult scenarioResult = runScenario("extension_shutdown_hook");
         assertThat(scenarioResult.getIOLog()).doesNotContain("shutdown hook running").describedAs(
             "shutdown hooks should not run in the same IO context as the main scenario"
         );
@@ -234,15 +234,15 @@ public class ScriptExampleTests {
 
     @Test
     public void testExceptionPropagationFromActivityInit() {
-        ScenarioResult scenarioResult = runScenario("activityiniterror");
-        assertThat(scenarioResult.getException()).isPresent();
-        assertThat(scenarioResult.getException().get().getMessage()).contains("Unable to convert end cycle from invalid");
+        ExecutionMetricsResult scenarioResult = runScenario("activityiniterror");
+        assertThat(scenarioResult.getException()).isNotNull();
+        assertThat(scenarioResult.getException().getMessage()).contains("Unable to convert end cycle from invalid");
         assertThat(scenarioResult.getException()).isNotNull();
     }
 
     @Test
     public void testReportedCoDelayBursty() {
-        ScenarioResult scenarioResult = runScenario("cocycledelay_bursty");
+        ExecutionMetricsResult scenarioResult = runScenario("cocycledelay_bursty");
         assertThat(scenarioResult.getIOLog()).contains("step1 metrics.waittime=");
         assertThat(scenarioResult.getIOLog()).contains("step2 metrics.waittime=");
         String iolog = scenarioResult.getIOLog();
@@ -252,7 +252,7 @@ public class ScriptExampleTests {
 
     @Test
     public void testReportedCoDelayStrict() {
-        ScenarioResult scenarioResult = runScenario("cocycledelay_strict");
+        ExecutionMetricsResult scenarioResult = runScenario("cocycledelay_strict");
         assertThat(scenarioResult.getIOLog()).contains("step1 cycles.waittime=");
         assertThat(scenarioResult.getIOLog()).contains("step2 cycles.waittime=");
         String iolog = scenarioResult.getIOLog();
@@ -263,14 +263,14 @@ public class ScriptExampleTests {
 
     @Test
     public void testCycleRateChangeNewMetrics() {
-        ScenarioResult scenarioResult = runScenario("cycle_rate_change");
+        ExecutionMetricsResult scenarioResult = runScenario("cycle_rate_change");
         String ioLog = scenarioResult.getIOLog();
         assertThat(ioLog).contains("cycles adjusted, exiting on iteration");
     }
 
     @Test
     public void testExitLogic() {
-        ScenarioResult scenarioResult = runScenario(
+        ExecutionMetricsResult scenarioResult = runScenario(
             "basicdiag",
             "type", "diag", "cyclerate", "5", "erroroncycle", "10", "cycles", "2000"
         );

nbr-examples/src/test/java/io/nosqlbench/nbr/examples/SpeedCheckIntegrationTests.java

@@ -15,7 +15,7 @@
  */
 package io.nosqlbench.nbr.examples;
 
-import io.nosqlbench.engine.core.lifecycle.ScenarioResult;
+import io.nosqlbench.engine.core.lifecycle.ExecutionMetricsResult;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
@@ -30,14 +30,14 @@ public class SpeedCheckIntegrationTests {
     @Disabled
     // Verified as working
     public void testSpeedSanity() {
-        ScenarioResult scenarioResult = ScriptExampleTests.runScenario("speedcheck");
+        ExecutionMetricsResult scenarioResult = ScriptExampleTests.runScenario("speedcheck");
     }
 
     @Test
     @Disabled
     // This seems incomplete
     public void testThreadSpeeds() {
-        ScenarioResult scenarioResult = ScriptExampleTests.runScenario("threadspeeds");
+        ExecutionMetricsResult scenarioResult = ScriptExampleTests.runScenario("threadspeeds");
     }
 
 

nbr-examples/src/test/resources/log4j2-test.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<Configuration status="${project.test_logstatuslevel}" strict="true" name="XMLConfigTest"
+               packages="org.apache.logging.log4j.test">
+
+  <Filter type="ThresholdFilter" level="trace"/>
+
+  <Appenders>
+    <Appender type="Console" name="STDOUT">
+      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
+      </Filters>
+    </Appender>
+    <Appender type="Console" name="FLOW">
+      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
+      </Filters>
+    </Appender>
+    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
+      <Layout type="PatternLayout">
+        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
+      </Layout>
+    </Appender>
+  </Appenders>
+
+  <Loggers>
+
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
+      <AppenderRef ref="APPSLOG"/>
+    </Logger>
+
+    <Root level="${project.testlevel}">
+      <AppenderRef ref="STDOUT"/>
+    </Root>
+
+  </Loggers>
+
+</Configuration>

nbr-examples/src/test/resources/scripts/examples/activityerror.js

@@ -19,7 +19,7 @@ activitydef1 = {
     "driver" : "diag",
     "cycles" : "0..1500000",
     "threads" : "1",
-    "targetrate" : "500",
+    "targetrate" : "10",
     "op" : {
         "log": "type=log modulo=1"
     }

nbr-examples/src/test/resources/scripts/examples/startstopdiag.js

@@ -18,17 +18,18 @@ activitydef = {
     "alias" : "teststartstopdiag",
     "driver" : "diag",
     "cycles" : "0..1000000000",
-    "threads" : "25",
+    "threads" : "5",
     "interval" : "2000",
-    "op" : "noop"
+    "op" : "noop",
+    "rate" : "5"
 };
 
 print('starting activity teststartstopdiag');
 scenario.start(activitydef);
 
 print('waiting 500 ms');
-
 scenario.waitMillis(500);
+
 print('waited, stopping activity teststartstopdiag');
 scenario.stop(activitydef);
 

nbr/src/main/resources/log4j.xml

@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="debug" strict="true" name="XMLConfigTest"
-               packages="org.apache.logging.log4j.test">
-
-  <Filter type="ThresholdFilter" level="trace"/>
-
-  <Appenders>
-    <Appender type="Console" name="STDOUT">
-      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
-      <Filters>
-        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
-        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
-      </Filters>
-    </Appender>
-    <Appender type="Console" name="FLOW">
-      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
-      <Filters>
-        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
-        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
-      </Filters>
-    </Appender>
-    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
-      <Layout type="PatternLayout">
-        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
-      </Layout>
-    </Appender>
-  </Appenders>
-
-  <Loggers>
-
-    <Logger name="io.nosqlbench.docsys" level="info" additivity="false" specificity="trace">
-      <AppenderRef ref="APPSLOG"/>
-    </Logger>
-
-    <Root level="trace">
-      <AppenderRef ref="STDOUT"/>
-    </Root>
-
-  </Loggers>
-
-</Configuration>

nbr/src/main/resources/log4j2.xml

@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
 <Configuration status="debug" strict="true" name="XMLConfigTest"
                packages="org.apache.logging.log4j.test">
 
@@ -28,7 +44,7 @@
 
   <Loggers>
 
-    <Logger name="io.nosqlbench.docsys" level="info" additivity="false" specificity="trace">
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
       <AppenderRef ref="APPSLOG"/>
     </Logger>
 
@@ -38,4 +54,4 @@
 
   </Loggers>
 
-</Configuration>
+</Configuration>

nbr/src/test/java/io/nosqlbench/cli/testing/ExitStatusIntegrationTests.java

@@ -86,19 +86,18 @@ class ExitStatusIntegrationTests {
         assertThat(result.exitStatus).isEqualTo(2);
     }
 
-//  This will not work reliablyl until the activity shutdown bug is fixed.
-//    @Test
-//    public void testCloseErrorHandlerOnSpace() {
-//        ProcessInvoker invoker = new ProcessInvoker();
-//        invoker.setLogDir("logs/test");
-//        ProcessResult result = invoker.run("exitstatus_erroronclose", 30,
-//            java, "-jar", JARNAME, "--logs-dir", "logs/test/error_on_close", "run",
-//            "driver=diag", "threads=2", "rate=5", "op=noop", "cycles=10", "erroronclose=true", "-vvv"
-//        );
-//        String stdout = String.join("\n", result.getStdoutData());
-//        String stderr = String.join("\n", result.getStderrData());
-//        assertThat(result.exception).isNotNull();
-//        assertThat(result.exception.getMessage()).contains("diag space was configured to throw");
-//    }
+    @Test
+    public void testCloseErrorHandlerOnSpace() {
+        ProcessInvoker invoker = new ProcessInvoker();
+        invoker.setLogDir("logs/test");
+        ProcessResult result = invoker.run("exitstatus_erroronclose", 30,
+            java, "-jar", JARNAME, "--logs-dir", "logs/test/error_on_close", "run",
+            "driver=diag", "threads=2", "rate=5", "op=noop", "cycles=10", "erroronclose=true", "-vvv"
+        );
+        String stdout = String.join("\n", result.getStdoutData());
+        String stderr = String.join("\n", result.getStderrData());
+        assertThat(result.exitStatus).isEqualTo(2);
+        assertThat(stderr).contains("diag space was configured to throw");
+    }
 
 }

nbr/src/test/java/io/nosqlbench/engine/core/script/MetricsMapperIntegrationTest.java

@@ -16,6 +16,7 @@
 
 package io.nosqlbench.engine.core.script;
 
+import io.nosqlbench.engine.core.lifecycle.scenario.script.MetricsMapper;
 import org.junit.jupiter.api.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;

nbr/src/test/resources/log4j2-test.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<Configuration status="${project.test_logstatuslevel}" strict="true" name="XMLConfigTest"
+               packages="org.apache.logging.log4j.test">
+
+  <Filter type="ThresholdFilter" level="trace"/>
+
+  <Appenders>
+    <Appender type="Console" name="STDOUT">
+      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
+      </Filters>
+    </Appender>
+    <Appender type="Console" name="FLOW">
+      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
+      </Filters>
+    </Appender>
+    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
+      <Layout type="PatternLayout">
+        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
+      </Layout>
+    </Appender>
+  </Appenders>
+
+  <Loggers>
+
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
+      <AppenderRef ref="APPSLOG"/>
+    </Logger>
+
+    <Root level="${project.testlevel}">
+      <AppenderRef ref="STDOUT"/>
+    </Root>
+
+  </Loggers>
+
+</Configuration>

virtdata-lang/src/test/resources/log4j-test.xml

@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="debug" strict="true" name="XMLConfigTest"
-               packages="org.apache.logging.log4j.test">
-
-  <Filter type="ThresholdFilter" level="trace"/>
-
-  <Appenders>
-    <Appender type="Console" name="STDOUT">
-      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
-      <Filters>
-        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
-        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
-      </Filters>
-    </Appender>
-    <Appender type="Console" name="FLOW">
-      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
-      <Filters>
-        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
-        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
-      </Filters>
-    </Appender>
-    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
-      <Layout type="PatternLayout">
-        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
-      </Layout>
-    </Appender>
-  </Appenders>
-
-  <Loggers>
-
-    <Logger name="io.nosqlbench.docsys" level="info" additivity="false" specificity="trace">
-      <AppenderRef ref="APPSLOG"/>
-    </Logger>
-
-    <Root level="trace">
-      <AppenderRef ref="STDOUT"/>
-    </Root>
-
-  </Loggers>
-
-</Configuration>

virtdata-lang/src/test/resources/log4j2-test.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<Configuration status="${project.test_logstatuslevel}" strict="true" name="XMLConfigTest"
+               packages="org.apache.logging.log4j.test">
+
+  <Filter type="ThresholdFilter" level="trace"/>
+
+  <Appenders>
+    <Appender type="Console" name="STDOUT">
+      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
+      </Filters>
+    </Appender>
+    <Appender type="Console" name="FLOW">
+      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
+      </Filters>
+    </Appender>
+    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
+      <Layout type="PatternLayout">
+        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
+      </Layout>
+    </Appender>
+  </Appenders>
+
+  <Loggers>
+
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
+      <AppenderRef ref="APPSLOG"/>
+    </Logger>
+
+    <Root level="${project.testlevel}">
+      <AppenderRef ref="STDOUT"/>
+    </Root>
+
+  </Loggers>
+
+</Configuration>

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStep.java

@@ -16,6 +16,8 @@
 
 package io.nosqlbench.virtdata.library.basics.shared.from_long.to_long;
 
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
 import io.nosqlbench.virtdata.api.annotations.Example;
 import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
 
@@ -38,6 +40,7 @@ import java.util.function.LongUnaryOperator;
  * join the project or let us know.</p>
  */
 @ThreadSafeMapper
+@Categories({Category.experimental})
 public class TriangularStep implements LongUnaryOperator {
 
     private final Hash hasher = new Hash();

virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepTest.java

@@ -34,7 +34,7 @@ public class TriangularStepTest {
     public void testStepExample1() {
         TriangularStep e1 = new TriangularStep(100, 20);
         int[] runLengths = this.rleStatsFor(e1, 0, 10000);
-        System.out.println(Arrays.toString(runLengths));
+//        System.out.println(Arrays.toString(runLengths));
         assertThat(IntStream.of(runLengths).min().orElseThrow()).isEqualTo(80L);
         assertThat(IntStream.of(runLengths).max().orElseThrow()).isEqualTo(120L);
     }
@@ -43,7 +43,7 @@ public class TriangularStepTest {
     public void testStepExample2() {
         TriangularStep e1 = new TriangularStep(80, 10);
         int[] runLengths = this.rleStatsFor(e1, 0, 10000);
-        System.out.println(Arrays.toString(runLengths));
+//        System.out.println(Arrays.toString(runLengths));
         assertThat(IntStream.of(runLengths).min().orElseThrow()).isEqualTo(70L);
         assertThat(IntStream.of(runLengths).max().orElseThrow()).isEqualTo(90L);
     }
@@ -101,7 +101,7 @@ public class TriangularStepTest {
         }
 
         for (int count : counts) {
-            System.out.println(count);
+//            System.out.println(count);
             histo[count-minval][FREQUENCY]++;
         }
         return histo;

virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/tests/long_long/SignedHashTest.java

@@ -17,11 +17,14 @@
 package io.nosqlbench.virtdata.library.basics.tests.long_long;
 
 import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.SignedHash;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class SignedHashTest {
+    private final static Logger logger = LogManager.getLogger(SignedHashTest.class);
 
     @Test
     public void testFunctionalResult() {
@@ -44,12 +47,13 @@ public class SignedHashTest {
         SignedHash hash = new SignedHash();
         for (int i = 0; i < 10; i++) {
             long l = hash.applyAsLong(i) % 50L;
-            System.out.println("i=" + i + " result=" + l);
+            logger.debug("i=" + i + " result=" + l);
+
         }
 
         for (int i = 0; i < 10; i++) {
             long l = hash.applyAsLong(i+1000000L) % 50L;
-            System.out.println("i=" + i + " result=" + l);
+            logger.debug("i=" + i + " result=" + l);
         }
 
     }

virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/tests/long_string/CombinationsTest.java

@@ -17,12 +17,15 @@
 package io.nosqlbench.virtdata.library.basics.tests.long_string;
 
 import io.nosqlbench.virtdata.library.basics.shared.from_long.to_string.Combinations;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 
 public class CombinationsTest {
+    private final static Logger logger = LogManager.getLogger(CombinationsTest.class);
 
     @Test
     public void testSimplePrimes() {
@@ -38,7 +41,7 @@ public class CombinationsTest {
         Combinations binaryByte = new Combinations("0-1;0-1;0-1;0-1;0-1;0-1;0-1;0-1;");
         assertThat(binaryByte.apply(37)).isEqualTo("00100101");
         for (int i = 0; i < 512; i++) {
-            System.out.println("i:" + i + " = b:" + binaryByte.apply(i));
+            logger.debug("i:" + i + " = b:" + binaryByte.apply(i));
         }
     }
 

virtdata-lib-basics/src/test/resources/log4j-test.xml

@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="debug" strict="true" name="XMLConfigTest"
-               packages="org.apache.logging.log4j.test">
-
-  <Filter type="ThresholdFilter" level="trace"/>
-
-  <Appenders>
-    <Appender type="Console" name="STDOUT">
-      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
-      <Filters>
-        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
-        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
-      </Filters>
-    </Appender>
-    <Appender type="Console" name="FLOW">
-      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
-      <Filters>
-        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
-        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
-      </Filters>
-    </Appender>
-    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
-      <Layout type="PatternLayout">
-        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
-      </Layout>
-    </Appender>
-  </Appenders>
-
-  <Loggers>
-
-    <Logger name="io.nosqlbench.docsys" level="info" additivity="false" specificity="trace">
-      <AppenderRef ref="APPSLOG"/>
-    </Logger>
-
-    <Root level="trace">
-      <AppenderRef ref="STDOUT"/>
-    </Root>
-
-  </Loggers>
-
-</Configuration>

virtdata-lib-basics/src/test/resources/log4j2-test.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<Configuration status="${project.test_logstatuslevel}" strict="true" name="XMLConfigTest"
+               packages="org.apache.logging.log4j.test">
+
+  <Filter type="ThresholdFilter" level="trace"/>
+
+  <Appenders>
+    <Appender type="Console" name="STDOUT">
+      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
+      </Filters>
+    </Appender>
+    <Appender type="Console" name="FLOW">
+      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
+      <Filters>
+        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
+      </Filters>
+    </Appender>
+    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
+      <Layout type="PatternLayout">
+        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
+      </Layout>
+    </Appender>
+  </Appenders>
+
+  <Loggers>
+
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
+      <AppenderRef ref="APPSLOG"/>
+    </Logger>
+
+    <Root level="${project.testlevel}">
+      <AppenderRef ref="STDOUT"/>
+    </Root>
+
+  </Loggers>
+
+</Configuration>

virtdata-lib-curves4/src/test/java/io/nosqlbench/virtdata/library/curves4/continuous/RealDistributionsConcurrencyTests.java

@@ -18,6 +18,8 @@ package io.nosqlbench.virtdata.library.curves4.continuous;
 
 import io.nosqlbench.virtdata.core.bindings.DataMapper;
 import io.nosqlbench.virtdata.core.bindings.VirtData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
@@ -30,6 +32,7 @@ import java.util.concurrent.Future;
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class RealDistributionsConcurrencyTests {
+    private final static Logger logger = LogManager.getLogger(RealDistributionsConcurrencyTests.class);
 
     @Test
     public void testConcurrentBinomialHashValues() {
@@ -73,8 +76,7 @@ public class RealDistributionsConcurrencyTests {
         for (int i = 0; i < futures.size(); i++) {
             try {
                 results.add(futures.get(i).get());
-//                System.out.println(description + ": got results for thread " + i);
-//                System.out.flush();
+                logger.trace(description + ": got results for thread " + i);
             } catch (Exception e) {
                 throw new RuntimeException(e);
             }
@@ -83,7 +85,7 @@ public class RealDistributionsConcurrencyTests {
 
         for (int vthread = 0; vthread < threads; vthread++) {
             assertThat(results.get(vthread)).isEqualTo(values);
-            System.out.println(description + ": verified values for thread " + vthread);
+            logger.debug(description + ": verified values for thread " + vthread);
         }
 
 
@@ -107,8 +109,7 @@ public class RealDistributionsConcurrencyTests {
         public double[] call() throws Exception {
             double[] output = new double[size];
             DataMapper<Double> mapper = VirtData.getMapper(mapperSpec, double.class);
-//            System.out.println("resolved:" + mapper);
-//            System.out.flush();
+            logger.trace("resolved:" + mapper);
 
             synchronized (signal) {
                 signal.wait(10000);
@@ -116,9 +117,9 @@ public class RealDistributionsConcurrencyTests {
 
             for (int i = 0; i < output.length; i++) {
                 output[i] = mapper.get(i);
-//                if ((i % 100) == 0) {
-//                    System.out.println("wrote t:" + slot + ", iter:" + i + ", val:" + output[i]);
-//                }
+                if ((i % 100) == 0) {
+                    logger.trace("wrote t:" + slot + ", iter:" + i + ", val:" + output[i]);
+                }
             }
             return output;
         }

virtdata-lib-curves4/src/test/java/io/nosqlbench/virtdata/library/curves4/continuous/RealDistributionsValuesTest.java

@@ -19,6 +19,8 @@ package io.nosqlbench.virtdata.library.curves4.continuous;
 import io.nosqlbench.virtdata.library.curves4.continuous.long_double.Normal;
 import io.nosqlbench.virtdata.library.curves4.continuous.long_double.Uniform;
 import org.apache.commons.math4.stat.descriptive.DescriptiveStatistics;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.assertj.core.data.Offset;
 import org.junit.jupiter.api.Test;
 
@@ -29,12 +31,13 @@ import java.util.function.LongToDoubleFunction;
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class RealDistributionsValuesTest {
+    private final static Logger logger = LogManager.getLogger(RealDistributionsValuesTest.class);
 
 
     @Test
     public void testComputedNormal() {
         RunData runData = iterateMapperDouble(new Normal(10.0,2.0,"compute"), 1000000,1);
-        System.out.println(runData);
+        logger.debug(runData);
         assertThat(runData.getFractionalPercentile(0.5D))
                 .isCloseTo(10.0D, Offset.offset(0.01D));
         assertThat(runData.getFractionalPercentile(0.4D))
@@ -46,7 +49,7 @@ public class RealDistributionsValuesTest {
     @Test
     public void testInterpolatedNormal() {
         RunData runData = iterateMapperDouble(new Normal(10.0,2.0,"interpolate"), 1000000,1);
-        System.out.println(runData);
+        logger.debug(runData);
         assertThat(runData.getFractionalPercentile(0.5D))
                 .isCloseTo(10.0D, Offset.offset(0.01D));
         assertThat(runData.getFractionalPercentile(0.4D))
@@ -64,7 +67,7 @@ public class RealDistributionsValuesTest {
                 .isCloseTo(50.0D, Offset.offset(1.0D));
         assertThat(runData.getFractionalPercentile(0.78D))
                 .isCloseTo(78.0D, Offset.offset(1.0D));
-        System.out.println(runData);
+        logger.debug(runData);
     }
 
     @Test
@@ -76,7 +79,7 @@ public class RealDistributionsValuesTest {
                 .isCloseTo(50.0D, Offset.offset(1.0D));
         assertThat(runData.getFractionalPercentile(0.78D))
                 .isCloseTo(78.0D, Offset.offset(1.0D));
-        System.out.println(runData);
+        logger.debug(runData);
     }
 
     @Test
@@ -92,7 +95,7 @@ public class RealDistributionsValuesTest {
 
         assertThat(mapper.applyAsDouble(Long.MAX_VALUE)).isCloseTo(100.0D, Offset.offset(1.0D));
 
-        System.out.println(runData);
+        logger.debug(runData);
     }
 
     private RunData iterateMapperDouble(LongToDoubleFunction mapper, int iterations, long funcstep) {

virtdata-lib-curves4/src/test/java/io/nosqlbench/virtdata/library/curves4/discrete/IntegerDistributionsConcurrencyTest.java

@@ -19,6 +19,8 @@ package io.nosqlbench.virtdata.library.curves4.discrete;
 import io.nosqlbench.virtdata.core.bindings.DataMapper;
 import io.nosqlbench.virtdata.core.bindings.VirtData;
 import org.apache.commons.statistics.distribution.BinomialDistribution;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.assertj.core.data.Offset;
 import org.junit.jupiter.api.Test;
 
@@ -33,6 +35,7 @@ import java.util.concurrent.Future;
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class IntegerDistributionsConcurrencyTest {
+    private final static Logger logger = LogManager.getLogger(IntegerDistributionsConcurrencyTest.class);
 
     @Test
     public void testBinomialICDR() {
@@ -109,7 +112,6 @@ public class IntegerDistributionsConcurrencyTest {
             try {
                 results[i] = futures.get(i).get();
                 System.out.println(description + ": got results for thread " + i);
-                System.out.flush();
             } catch (Exception e) {
                 throw new RuntimeException(e);
             }
@@ -125,15 +127,14 @@ public class IntegerDistributionsConcurrencyTest {
                     for (int ithread = 0; ithread < threads; ithread++) {
                         System.out.print(results[ithread][i] + ",");
                     }
-                    System.out.println();
                 }
             }
             boolean equal = Arrays.equals(results[vthread],values);
             if (!equal) {
-                System.out.println("not equal!");
+                logger.debug("not equal!");
             }
             assertThat(results[vthread]).isEqualTo(values);
-            System.out.println(description + ": verified values for thread " + vthread);
+            logger.debug(description + ": verified values for thread " + vthread);
         }
 
 
@@ -157,8 +158,7 @@ public class IntegerDistributionsConcurrencyTest {
         public long[] call() throws Exception {
             long[] output = new long[size];
             DataMapper<Long> mapper = VirtData.getMapper(mapperSpec, long.class);
-//            System.out.println("resolved:" + mapper);
-//            System.out.flush();
+//            logger.debug("resolved:" + mapper);
 
             synchronized (signal) {
                 signal.wait(10000);
@@ -167,7 +167,7 @@ public class IntegerDistributionsConcurrencyTest {
             for (int i = 0; i < output.length; i++) {
                 output[i] = mapper.get(i);
 //                if ((i % 100) == 0) {
-//                    System.out.println("wrote t:" + slot + ", iter:" + i + ", val:" + output[i]);
+//                    logger.debug("wrote t:" + slot + ", iter:" + i + ", val:" + output[i]);
 //                }
             }
             return output;

virtdata-lib-curves4/src/test/java/io/nosqlbench/virtdata/library/curves4/discrete/IntegerDistributionsValuesTest.java

@@ -19,6 +19,8 @@ package io.nosqlbench.virtdata.library.curves4.discrete;
 import io.nosqlbench.virtdata.library.curves4.continuous.long_double.Uniform;
 import io.nosqlbench.virtdata.library.curves4.discrete.long_long.Zipf;
 import org.apache.commons.math4.stat.descriptive.DescriptiveStatistics;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.assertj.core.data.Offset;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -31,13 +33,14 @@ import java.util.function.LongUnaryOperator;
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class IntegerDistributionsValuesTest {
+    private final static Logger logger = LogManager.getLogger(IntegerDistributionsValuesTest.class);
 
 
     @Disabled
     @Test
     public void testComputedZipf() {
         RunData runData = iterateMapperLong(new Zipf(10000,2.0), 10000);
-        System.out.println(runData);
+        logger.debug(runData);
         assertThat(runData.getFractionalPercentile(0.6D))
                 .isCloseTo(1.0D, Offset.offset(0.0001D));
         assertThat(runData.getFractionalPercentile(0.7D))
@@ -51,7 +54,7 @@ public class IntegerDistributionsValuesTest {
     @Test
     public void testInterpolatedZipf() {
         RunData runData = iterateMapperLong(new Zipf(10000,2.0), 10000);
-        System.out.println(runData);
+        logger.debug(runData);
         assertThat(runData.getFractionalPercentile(0.6D))
                 .isCloseTo(1.0D, Offset.offset(0.0001D));
         assertThat(runData.getFractionalPercentile(0.7D))
@@ -72,7 +75,7 @@ public class IntegerDistributionsValuesTest {
                 .isCloseTo(50.0D, Offset.offset(1.0D));
         assertThat(runData.getFractionalPercentile(0.78D))
                 .isCloseTo(78.0D, Offset.offset(1.0D));
-        System.out.println(runData);
+        logger.debug(runData);
     }
 
     @Test
@@ -84,7 +87,7 @@ public class IntegerDistributionsValuesTest {
                 .isCloseTo(50.0D, Offset.offset(1.0D));
         assertThat(runData.getFractionalPercentile(0.78D))
                 .isCloseTo(78.0D, Offset.offset(1.0D));
-        System.out.println(runData);
+        logger.debug(runData);
     }
 
     @Test
@@ -127,7 +130,7 @@ public class IntegerDistributionsValuesTest {
         int readout = iterations/10;
         for (int i = 0; i < iterations; i++) {
             if ((i%readout)==0) {
-                System.out.println("i="+i+"/"+iterations);
+                logger.debug("i="+i+"/"+iterations);
             }
             samples[i] = mapper.applyAsDouble(i);
         }

virtdata-userlibs/src/main/resources/log4j.xml

@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="debug" strict="true" name="XMLConfigTest"
-               packages="org.apache.logging.log4j.test">
-
-  <Filter type="ThresholdFilter" level="trace"/>
-
-  <Appenders>
-    <Appender type="Console" name="STDOUT">
-      <Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
-      <Filters>
-        <Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
-        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
-      </Filters>
-    </Appender>
-    <Appender type="Console" name="FLOW">
-      <Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
-      <Filters>
-        <Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
-        <Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
-      </Filters>
-    </Appender>
-    <Appender type="File" name="APPSLOG" fileName="docs/apps.log">
-      <Layout type="PatternLayout">
-        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
-      </Layout>
-    </Appender>
-  </Appenders>
-
-  <Loggers>
-
-    <Logger name="io.nosqlbench.docsys" level="info" additivity="false" specificity="trace">
-      <AppenderRef ref="APPSLOG"/>
-    </Logger>
-
-    <Root level="trace">
-      <AppenderRef ref="STDOUT"/>
-    </Root>
-
-  </Loggers>
-
-</Configuration>

virtdata-userlibs/src/main/resources/log4j2.xml

@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2022 nosqlbench
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
 <Configuration status="debug" strict="true" name="XMLConfigTest"
                packages="org.apache.logging.log4j.test">
 
@@ -28,7 +44,7 @@
 
   <Loggers>
 
-    <Logger name="io.nosqlbench.docsys" level="info" additivity="false" specificity="trace">
+    <Logger name="io.nosqlbench.docsys" level="info" additivity="false">
       <AppenderRef ref="APPSLOG"/>
     </Logger>
 
@@ -38,4 +54,4 @@
 
   </Loggers>
 
-</Configuration>
+</Configuration>