adapter API improvements

adapters-api/pom.xml

@@ -1,37 +1,40 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
 
 
-  <parent>
-    <artifactId>mvn-defaults</artifactId>
-    <groupId>io.nosqlbench</groupId>
-    <version>4.15.52-SNAPSHOT</version>
-    <relativePath>../mvn-defaults</relativePath>
-  </parent>
+    <parent>
+        <artifactId>mvn-defaults</artifactId>
+        <groupId>io.nosqlbench</groupId>
+        <version>4.15.52-SNAPSHOT</version>
+        <relativePath>../mvn-defaults</relativePath>
+    </parent>
 
-  <artifactId>adapters-api</artifactId>
-  <packaging>jar</packaging>
-  <name>${project.artifactId}</name>
-  <description>
-    The DriverAdapter API for nosqlbench;
-    Provides the interfaces needed to build drivers that can be loaded
-    by nosqlbench core, using a minimal and direct API for op mapping.
-  </description>
+    <artifactId>adapters-api</artifactId>
+    <packaging>jar</packaging>
+    <name>${project.artifactId}</name>
+    <description>
+        The DriverAdapter API for nosqlbench;
+        Provides the interfaces needed to build drivers that can be loaded
+        by nosqlbench core, using a minimal and direct API for op mapping.
+    </description>
 
-  <dependencies>
+    <dependencies>
 
-    <dependency>
-      <groupId>io.nosqlbench</groupId>
-      <artifactId>nb-api</artifactId>
-      <version>4.15.52-SNAPSHOT</version>
-    </dependency>
+        <dependency>
+            <groupId>io.nosqlbench</groupId>
+            <artifactId>nb-api</artifactId>
+            <version>4.15.52-SNAPSHOT</version>
+        </dependency>
 
-    <dependency>
-      <groupId>io.nosqlbench</groupId>
-      <artifactId>virtdata-userlibs</artifactId>
-      <version>4.15.52-SNAPSHOT</version>
-    </dependency>
+        <dependency>
+            <groupId>io.nosqlbench</groupId>
+            <artifactId>virtdata-userlibs</artifactId>
+            <version>4.15.52-SNAPSHOT</version>
+        </dependency>
 
-  </dependencies>
+
+    </dependencies>
 
 </project>

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/OpDispenser.java

@@ -7,8 +7,7 @@ import java.util.function.LongFunction;
  * <H2>Synopsis</H2>
  * An OpDispenser is responsible for mapping a cycle number into
  * an executable operation. This is where <i>Op Synthesis</i> occurs
- * in NoSQLBench -- The process of building executable operations from
- * templates.</p>
+ * in NoSQLBench.</p>
  * <hr/>
  * <p>
  * <H2>Concepts</H2>

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/OpMapper.java

@@ -1,5 +1,6 @@
 package io.nosqlbench.engine.api.activityimpl;
 
+import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op;
 import io.nosqlbench.engine.api.templating.ParsedCommand;
 
 import java.util.function.Function;
@@ -61,7 +62,7 @@ import java.util.function.Function;
  *            to hold all the details for executing an operation,
  *            generally something that implements {@link Runnable}.
  */
-public interface OpMapper<T extends Runnable> extends Function<ParsedCommand, OpDispenser<T>> {
+public interface OpMapper<T extends Op> extends Function<ParsedCommand, OpDispenser<T>> {
 
     /**
      * Interrogate the parsed command, and provide a new

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java

@@ -1,7 +1,11 @@
 package io.nosqlbench.engine.api.activityimpl.uniform;
 
 import io.nosqlbench.engine.api.activityimpl.uniform.fieldmappers.FieldDestructuringMapper;
-import io.nosqlbench.nb.api.config.standard.*;
+import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op;
+import io.nosqlbench.nb.api.config.standard.ConfigModel;
+import io.nosqlbench.nb.api.config.standard.NBConfigModel;
+import io.nosqlbench.nb.api.config.standard.NBConfigurable;
+import io.nosqlbench.nb.api.config.standard.NBConfiguration;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -10,7 +14,7 @@ import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-public abstract class BaseDriverAdapter<R extends Runnable,S>
+public abstract class BaseDriverAdapter<R extends Op,S>
     implements DriverAdapter<R,S>, NBConfigurable {
 
     private final DriverSpaceCache<? extends S> spaceCache;
@@ -93,6 +97,7 @@ public abstract class BaseDriverAdapter<R extends Runnable,S>
         return ConfigModel.of(this.getClass());
     }
 
+    @Override
     public NBConfiguration getConfiguration() {
         return cfg;
     }

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverAdapter.java

@@ -2,7 +2,9 @@ package io.nosqlbench.engine.api.activityimpl.uniform;
 
 import io.nosqlbench.engine.api.activityimpl.OpDispenser;
 import io.nosqlbench.engine.api.activityimpl.OpMapper;
+import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op;
 import io.nosqlbench.engine.api.templating.ParsedCommand;
+import io.nosqlbench.nb.api.config.standard.NBConfiguration;
 
 import java.util.Map;
 import java.util.function.Function;
@@ -27,7 +29,7 @@ import java.util.function.Function;
  *            during construction of R type operations, or even for individual
  *            operations.
  */
-public interface DriverAdapter<R extends Runnable, S> {
+public interface DriverAdapter<R extends Op, S> {
 
 
     /**
@@ -125,4 +127,5 @@ public interface DriverAdapter<R extends Runnable, S> {
     }
 
 
+    NBConfiguration getConfiguration();
 }

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/ResultProcessor.java

@@ -0,0 +1,31 @@
+package io.nosqlbench.engine.api.activityimpl.uniform;
+
+/**
+ * A result processor can consume data from a result which is contains of a set of
+ * ordered elements.
+ * @param <C> The result type which acts as the container of elements
+ * @param <E> The element type
+ */
+public interface ResultProcessor<C,E> {
+
+    /**
+     * Call the start method before any buffering of results.
+     * @param cycle The cycle that the result is associated with. This is for presentation only.
+     * @param container The result object which holds individual result elements.
+     */
+    void start(long cycle, C container);
+
+    /**
+     * For each element in the container, buffer it. The effect of buffering is contextual. For
+     * ResultProcessors which need to see all the result data before applying its effect,
+     * simply buffer the elements to an internal container type.
+     * @param element
+     */
+    void buffer(E element);
+
+    /**
+     * Once all the elements of the result have been buffered, flush must be called.
+     * ResultProcessors which need to see all the data can finish processing here.
+     */
+    void flush();
+}

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/ChainingOp.java

@@ -0,0 +1,18 @@
+package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes;
+
+import java.util.function.Function;
+
+/**
+ * Run a function on the current cached result and replace it
+ * with the result of the function. Functions are one way of invoking
+ * logic within a cycle. However, they are not intended to stand alone.
+ * A CycleFunction must always have an input to work on. This input is
+ * provided by a Supplier as optionally implemented by an Op
+ *
+ * @param <I> Some input type.
+ * @param <O> Some output type.
+ */
+public interface ChainingOp<I,O> extends Op, Function<I,O> {
+    @Override
+    O apply(I i);
+}

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/CycleOp.java

@@ -0,0 +1,37 @@
+package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes;
+
+import java.util.function.LongFunction;
+
+/**
+ * A CycleRunnable is simply a variation of a Runnable type.
+ * The main difference is that it is supplied with the cycle
+ * as input.
+ */
+public interface CycleOp<T> extends Op, LongFunction<T> {
+//    /**
+//     * <p>Run an action for the given cycle. The cycle is provided for anecdotal
+//     * usage such as logging and debugging. It is valid to use the cycle value in these places,
+//     * but you should not use it to determine the logic of what is run. The mechanism
+//     * for doing this is provided in {@link io.nosqlbench.engine.api.activityimpl.OpMapper}
+//     * and {@link io.nosqlbench.engine.api.activityimpl.OpDispenser} types.</p>
+//     *
+//     *
+//     * @param cycle The cycle value for which an operation is run
+//     */
+////     * This method should do the same thing that {@link #apply(long)} does, except that
+////     * there is no need to prepare or return a result. This is the form that will be called
+////     * if there is no chaining operation to consume the result of this operation.
+//    void accept(long cycle);
+
+    /**
+     * <p>Run an action for the given cycle. The cycle
+     * value is only to be used for anecdotal presentation. This form is called
+     * when there is a chaining operation which will do something with this result.</p>
+     * @param value The cycle value for which an operation is run
+     * @return A result which is the native result type for the underlying driver.
+     */
+    @Override
+    T apply(long value);
+
+
+}

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/Op.java

@@ -0,0 +1,21 @@
+package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes;
+
+/**
+ * This is the root type of any operation which is used in a NoSQLBench
+ * DriverAdapter. It is a tagging interface for incremental type validation
+ * in the NB runtime. You probably don't want to use it directly.
+ *
+ * Instead, use these:
+ * <ul>
+ *  <li>{@link CycleOp}</li> - An interface that will called if there is nothing to consume
+ *  the result type from your operation. In some cases preparing a result body to
+ *  hand down the chain is more costly, so implementing this interface allows
+ * </ul>
+ *
+ * either {@link CycleOp} or {@link ChainingOp} (but not both!)
+ *
+ * In the standard flow of an activity, either of the above interfaces is called
+ * so long as an Op implements one of them.
+ */
+public interface Op {
+}

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/OpGenerator.java

@@ -0,0 +1,19 @@
+package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes;
+
+/**
+ * <p>If an Op implements OpGenerator, then it will be asked for chained
+ * operations that are secondary unless or until {@link #getNextOp()}}
+ * returns null.</p>
+ *
+ * <p>If an Op *might* generate a secondary operation, then it should implement
+ * this interface and simply return null in the case that there is none.</p>
+ *
+ * <p>If you need to run many operations after this, then you can keep the
+ * state of these in the same Op implementation and simply keep returning
+ * it until the work list is done. The same applies for structured op
+ * generation, such as lists of lists or similar.</p>
+ *
+ */
+public interface OpGenerator {
+    Op getNextOp();
+}

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/VariableCapture.java

@@ -0,0 +1,18 @@
+package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes;
+
+import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
+
+import java.util.Map;
+
+/**
+ * If an op implements VariableCapture, then it is known to be able to
+ * extract variables from its result. Generally speaking, this should
+ * be implemented within an Op according to the standard format
+ * of {@link ParsedTemplate#getCaptures()}. Any op implementing
+ * this should use the interface below to support interop between adapters
+ * and to allow for auto documentation tha the feature is supported for
+ * a given adapter.
+ */
+public interface VariableCapture {
+    Map<String,?> capture();
+}

devdocs/devguide/drivers/driver_adapter_standards.md

@@ -0,0 +1,321 @@
+# NoSQLBench Driver Adapter Standards
+
+This document is intended to replace the earlier driver standard guide, as the APIs in the upcoming
+release are both more streamlined and prescriptive. If you have built a driver in NoSQLBench before,
+you will find that there is not much to it with the latest API updates.
+
+---
+
+This is the document to read if you want to know if your NoSQLBench Driver Adapter is complete.
+Within this document, the phrase `conformant` will be taken to mean that a DriverAdapter is
+implemented according to the design intent and standards of this NoSQLBench guide and the API that
+it describes.
+
+While it may be possible to partially implement a Driver Adapter for basic use, following the
+guidelines in this document will ensure that contributed drivers for NoSQLBench work in a familiar
+and reliable way for users from one driver to another.
+
+Over time, the standards in this guide will be programmatically enforced by the NoSQLBench Driver
+Adapter API.
+
+## Terms
+
+- Driver Adapter - The NoSQLBench level driver adapter, the code that this document refers to.
+- Native driver - An underlying driver which is provided by a vendor or project.
+
+## Overview
+
+The NoSQLBench runtime supports the usage of multiple native drivers via a runtime layer called the
+Driver Adapter. Each DriverAdapter implementation serves as a bridge between users and a native
+driver API. For each operation specified by a user (as an op template), a Driver Adapter does the
+following:
+
+1) Examine the op fields and values to determine what type of operation a user intends. This process
+   is called *op mapping*. (TODO: link to OpMapper JavaDoc)
+2) Construct a dispenser object which can create instances of the specific type of operation
+   determined above. This step is called *op synthesis*. (TODO: Link to OpDispenser JavaDoc)
+
+The first (op mapping) is done at initialization time, and is responsible for doing all the
+pre-vetting of what users put into their op templates (via yaml, json, or whatever). As a mechanism
+that determines user intent, clarity is paramount.
+
+The second (op synthesis) is done for each and every cycle of an activity that is run, and must be
+done efficiently to allow NoSQLBench to operate as an effective testing instrument. If the op
+mapping done correctly, there is no need for an op dispenser object to try to sort out what kind of
+operation the user intended. An op dispenser synthesizes the fields together.
+
+These two steps are closely related although they have very distinct responsibilities. They are
+connected directly in the API -- The OpMapper dispenses OpDispensers (which dispense Ops). Although
+they are connected in this way, you have *exact* control of when each happens in the lifecycle of an
+activity: Everything in an op mapper and it's _apply_ method *and* in the constructors of op
+dispensers that it creates is done before an activity starts (before the first cycle of that
+activity). Everything after that (in the body of the OpDispenser apply method, for example) occurs
+within a cycle of an activity.
+
+## Op Templates
+
+Users create op templates by writing YAML or JSON or data structures via the scripting API. These
+are simply nested collections in their normative form, with string-based keys and values as in any
+data structure. NoSQLBench recognizes a standard format (TODO: LINK THIS)
+which is the same across all driver types that can be used with NoSQLBench.
+
+The Op Templates which are provided by users are normalized by NoSQLBench into a standard
+representation that is used within the op mapping and synthesis steps. This representation is
+provided by the ParsedCommand and ParsedTemplate APIs. The User-Facing construct is _Op Template_,
+while the developer building driver adapters only sees _Parsed Commands_ and a fully-normalized API.
+
+## Effective Op Mapping
+
+### Setting the Stage
+
+Op Mapping has specific inputs and specific outputs. On the input side, an op mapper can see as much
+as the user specifies in the op template, and possibly more as provided to the op mapper's
+constructor. Op Mappers may need access to the activity's parameters or the activity's space cache.
+These can be provided from the DriverAdapter base type when needed.
+
+Assuming you provide the activity params and the space cache to an OpMapper implementation, when
+it's `apply(ParsedCommand cmd)` method is called, you have access to a few levels of information:
+
+1. The ParsedCommand -- representing the specific details of an operation to be performed:
+    * op field names
+    * static field values - literal values or any non-string collection type (map, set, list)
+    * dynamic field values - Any type which contains a string template or a single binding
+2. Op params, specified outside the layer of the op payload above. These are static fields which
+   users can specify outside
+3. The Activity params. Sometimes you want to provide an activity-wide default for how a type of
+   operations works. When this applies, be sure to favor the op-specific parameters over any
+   activity params.
+4. The state cache for the DriverAdapter instance, AKA the space cache. (TODO: Link to javadocs
+   for this)
+
+Since op mapping logic is responsible for creating the op dispenser, an op mapper must pass along
+any state, config, or other runtime details needed to create a native operation. You have control of
+this since you design and instantiate op mapper types directly from your DriverAdapter
+implementation.
+
+### Distinguishing Op Types
+
+There are multiple methods a driver adapter may use to determine what kind of operation a user
+intends. Of these mentioned here, the best guidance is to choose the one that most closely mimics
+the semantics and extant APIs for the specific native driver in question:
+
+* Use a type designator field like `type: put` or something similar.
+* Infer the op type by which field names are present in the template. (be careful with this one!)
+* Model the op exactly like the payload of the native driver, and hand the op data directly to the
+  native driver as such (only one "type" of op here at the NB level)
+
+### Show Users How
+
+In any case, the method that you choose to use needs to be clearly documented, unambiguous, and
+unaffected by the addition of new op types added to your driver in the future. You should
+provide examples of each op type in your (driver adapter) documentation. Ideally, your
+documentation is based on testable examples that are kept in the source tree and used for both
+unti testing *and* user examples.
+
+## Effective Op Synthesis
+
+1. Pre-compute as much as you can in the constructor of the OpDispenser. These objects are retained
+   for the life of an activity.
+2. Store re-usable elements of an operation in thread-safe form and re-use it wherever possible.
+
+
+# Congruent Behavior
+
+In order to ensure fairness and equity in how drivers work across systems and vendors, it is
+necessary to standardize on what each driver does with its operations. A compliant driver
+adapter will do the following:
+
+1. Provide an Op implementation which can be retried without resynthesis
+2. Fully read all the data in every result by default. Deviations from this default can only be
+   allowed when users explicitly specify something else, and should be accompanied by a
+   documentation or logging level warning that it is not normal behavior for a client.
+3. Provide metrics about the quantity of elements read in a result.
+
+# Config Sources
+
+Activites have configuration at various levels:
+
+1. Activity-wide parameters, called _activity params_.
+2. (within the workload template, like a YAML doc) doc level params
+3. (with a workload template, such as a YAML doc) block level params
+4. op level params
+5. op template fields
+
+Op template fields (seen by the NB driver developer through the
+ParsedCommand API) are properly meant to specify a distinct type of operation
+by its defined properties, no less or more. However, users will sometimes
+put op params into the op template alongside the op fields. This is *OK*.
+
+*The rule of thumb is to ensure that a named field can only be used as an
+op field or an op param but not both.* Each ParsedCommand has access to
+all of the layers above, and should be used to extract out the fields
+which are properly configuration level data before the fields are used
+for op mapping. By using this technique, op fields can be configured from any convenient
+level.
+
+
+# Enhancements
+
+* Configuration params that govern op behavior can be specified at any level, including within the op template itself.
+* Binding functions can be expressed as named anchors or inline as direct definitions
+* Variable capture syntax in parsed op formats is standardized.
+...
+
+# Revamp Below!
+
+
+## Result Validation
+
+TBD
+
+## Diagnostic Mode
+
+TBD
+
+## Naming Conventions
+
+TBD
+
+### Parameter naming
+
+Parameters should be formatted as snake_case by default. Hyphens or camel case often cause issues
+when using mixed media such as command lines and yaml formats. Snake case is a simple common
+denominator which works across all these forms with little risk of ambiguity when parsing or
+documenting how parameters are set apart from other syntax.
+
+## Documentation
+
+Each activity is required to have a set of markdown documentation in its resource directory. The
+name of the driver should also be used as the name of the documentation for that driver.
+
+Additional documentation can be added beyond this file. However, all documentation for a given
+driver must start with the drivers name and a hyphen.
+
+If a driver wants to include topics, the convention is to mention these other topics within the
+driver's main help. Any markdown file which is included in the resources of a driver module will be
+viewable by users with the help command `nb help <name>`. For example, if a driver module
+contains `../src/main/resources/mydriver-specials.md`, then a user would be able to find this help
+by running `nb help mydriver-specials`.
+
+These sources of documentation can be wired into the main NoSQLBench documentation system with a set
+of content descriptors.
+
+## Named Scenarios
+
+Conformant driver implementations should come with one or more examples of a workload under the
+activities directory path. Useful driver implementations should come with one or more examples of a
+workloads under the activities directory path. These examples should employ the "named scenarios"
+format as described in the main docs. By including named scenarios in the yaml format, these named
+scenarios then become available to users when they look for scenarios to call with the
+`--list-scenarios` command.
+
+To include such scenario, simply add a working yaml with a scenarios section to the root of your
+module under the
+`src/main/resources/activities` directory.
+
+## Included Examples
+
+Useful driver implementations should come with a set of examples under the examples directory path
+which demonstrate useful patterns, bindings, or statement forms.
+
+Users can find these examples in the same way as they can find the named scenarios above with the
+only difference being their location. By convention the directory `src/main/resources/examples`
+directory is where these are located.
+
+The format is the same as for named scenarios, because the examples *are*
+named scenarios. Users can find these by using the `--include=examples`
+option in addition to the `--list-scenarios` command.
+
+## Testing and Docs
+
+Complete driver implementations should also come with a set of examples under the examples directory
+path.
+
+Unit testing within the NB code base is necessary in many places, but not in others. Use your
+judgement about when to *not* add unit testing, but default to adding it when it seems subjective. A
+treatise on when and how to choose appropriate unit testing won't fit here, but suffice it to say
+that you can always ask the project maintainers for help on this if you need.
+
+Non-trivial code in pull requests without any form of quality checks or testing will not be merged
+until or unless the project maintainers are satisfied that there is little risk of user impact.
+Experimental features clearly labeled as such will be given more wiggle room here, but the label
+will not be removable unless/until a degree of robustness is proven in some testing layer.
+
+### Testing Futures
+
+In the future, the integration testing and the docs system are intended to become part of one whole.
+Particularly, docs should provide executable examples which can also be used to explain how NB or
+drivers work. Until this is done, use the guidelines above.
+
+## Handling secrets
+
+Reading passwords ...
+
+## Parameter Use
+
+Activity parameters *and* statement parameters must combine in intuitive ways.
+
+### ActivityType Parameters
+
+The documentation for an activity type should have an explanation of all the activity parameters
+that are unique to it. Examples of each of these should be given. The default values for these
+parameters should be given. Further, if there are some common settings that may be useful to users,
+these should be included in the examples.
+
+### Statement Parameters
+
+The documentation for an activity type should have an explanation of all the statement parameters
+that are unique to it. Examples of each of these should be given. The default values for these
+parameters should be given.
+
+### Additive Configuration
+
+If there is a configuration element in the activity type which can be modified in multiple ways that
+are not mutually exclusive, each time that configuration element is modified, it should be done
+additively. This means that users should not be surprised when they use multiple parameters that
+modify the configuration element with only the last one being applied. An example of this would be
+adding a load-balancing policy to a cql driver and then, separately adding another. The second one
+should wrap the first, as this is expected to be additive by nature of the native driver's API.
+
+### Parameter Conflicts
+
+If it is possible for parameters to conflict with each other in a way that would provide an invalid
+configuration when both are applied, or in a way that the underlying API would not strictly allow,
+then these conditions must be detected by the activity type, with an error thrown to the user
+explaining the conflict.
+
+### Parameter Diagnostics
+
+Each and every activity parameter that is set on an activity *must* be logged at DEBUG level with
+the pattern `ACTIVITY PARAMETER: <activity alias>` included in the log line, so that the user may
+verify applied parameter settings. Further, an explanation for what this parameter does to the
+specific activity *should*
+be included in a following log line.
+
+Each and every statement parameter that is set on a statement *must* be logged at DEBUG level with
+the pattern `STATEMENT PARAMETER: <statement name>: ` included in the log line, so that the user may
+verify applied statement settings. Further, an explanation for what this parameter does to the
+specific statement *
+should* be included in a following log line.
+
+### Environment Variables
+
+Environment variable may be hoisted into a driver's configuration, but only using explicit
+mechanisms. By default, environment variables are not injected into any NoSQLBench usage context
+where it is not explicitly enabled by the user. The mechanism of enabling environment variables is
+simple indirection, using a symbolic variable reference where they would normally use a value.
+
+Further, the variable must be explicitly enabled for env interpolation by the developer, and
+documented as such. Having variables which often use
+`$...` formats for other purposes besides environment variables is a nuisance. Conversely, not
+supporting env vars in `$...` values which are historically enabled for such is also a nuisance.
+
+#### format
+
+such as `myparam=$ENV_VAR_FOO`, where the env var name must follow this pattern:
+
+1. A `$` literal dollar sign.
+2. Any alphabetic or underscore character (`[a-zA-Z_]`)
+3. Zero or more trailing characters to include optional dots and digits. (`[a-zA-Z0-9_]*`)
+
+Alternately, the `${...}` form is less strict, and allows any characters which are not `}`.

driver-direct/src/main/java/io/nosqlbench/driver/direct/DirectCall.java

@@ -1,8 +1,10 @@
 package io.nosqlbench.driver.direct;
 
+import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op;
+
 import java.lang.reflect.Method;
 
-public class DirectCall implements Runnable {
+public class DirectCall implements Op,Runnable {
     private final Method method;
     private final Object[] args;
     private final Object instance;

driver-jmx/src/main/java/io/nosqlbench/driver/jmx/ops/JmxOp.java

@@ -1,7 +1,8 @@
 package io.nosqlbench.driver.jmx.ops;
 
-import org.apache.logging.log4j.Logger;
+import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import javax.management.MBeanServerConnection;
 import javax.management.ObjectName;
@@ -10,7 +11,7 @@ import javax.management.remote.JMXConnector;
 /**
  * All JMX Operations should built on this base type.
  */
-public abstract class JmxOp implements Runnable {
+public abstract class JmxOp implements Op,Runnable {
 
     protected final static Logger logger = LogManager.getLogger(JmxOp.class);
 

drivers-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedCommandTest.java

@@ -1,6 +1,8 @@
 package io.nosqlbench.engine.api.templating;
 
 import io.nosqlbench.engine.api.activityconfig.yaml.OpData;
+import io.nosqlbench.nb.api.config.standard.ConfigModel;
+import io.nosqlbench.nb.api.config.standard.Param;
 import org.junit.jupiter.api.Test;
 
 import java.util.List;
@@ -24,7 +26,11 @@ public class ParsedCommandTest {
                     "dyna1", "NumberNameToString()"
                 )
             )
-        )
+        ),
+        ConfigModel.of(ParsedCommandTest.class)
+            .add(Param.defaultTo("testcfg","testval"))
+            .asReadOnly()
+            .apply(Map.of())
     );
 
     @Test

engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/planning/OpSource.java

@@ -1,6 +1,7 @@
 package io.nosqlbench.engine.api.activityapi.planning;
 
 import io.nosqlbench.engine.api.activityimpl.OpDispenser;
+import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op;
 
 import java.util.function.LongFunction;
 
@@ -12,7 +13,7 @@ import java.util.function.LongFunction;
  */
 public interface OpSource<T> extends LongFunction<T> {
 
-    static <O extends Runnable> OpSource<O> of(OpSequence<OpDispenser<O>> seq) {
+    static <O extends Op> OpSource<O> of(OpSequence<OpDispenser<O>> seq) {
         return (long l) -> seq.apply(l).apply(l);
     }
 

engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java

@@ -102,6 +102,7 @@ public class OpDef extends OpTemplate {
     private LinkedHashMap<String, String> composeTags() {
         LinkedHashMap<String, String> tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawStmtDef.getTags(), block.getTags()));
         tagsWithName.put("name",getName());
+        tagsWithName.put("block",block.getName());
         return tagsWithName;
     }
 

engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java

@@ -18,10 +18,12 @@ import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
 import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
 import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
 import io.nosqlbench.engine.api.activityimpl.input.ProgressCapable;
+import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op;
 import io.nosqlbench.engine.api.metrics.ActivityMetrics;
 import io.nosqlbench.engine.api.templating.CommandTemplate;
 import io.nosqlbench.engine.api.templating.ParsedCommand;
 import io.nosqlbench.engine.api.templating.StrInterpolator;
+import io.nosqlbench.nb.api.config.standard.NBConfiguration;
 import io.nosqlbench.nb.api.errors.BasicError;
 import io.nosqlbench.nb.api.errors.OpConfigError;
 import org.apache.logging.log4j.LogManager;
@@ -417,18 +419,19 @@ public class SimpleActivity implements Activity, ProgressCapable {
      * @param <O>
      * @return
      */
-    protected <O extends Runnable> OpSequence<OpDispenser<O>> createOpSequenceFromCommands(Function<CommandTemplate, OpDispenser<O>> opinit) {
+    protected <O extends Op> OpSequence<OpDispenser<O>> createOpSequenceFromCommands(Function<CommandTemplate, OpDispenser<O>> opinit) {
         Function<OpTemplate, CommandTemplate> f = CommandTemplate::new;
         Function<OpTemplate, OpDispenser<O>> opTemplateOFunction = f.andThen(opinit);
 
         return createOpSequence(opTemplateOFunction);
     }
 
-    protected <O extends Runnable> OpSequence<OpDispenser<O>> createOpSourceFromCommands(
+    protected <O extends Op> OpSequence<OpDispenser<O>> createOpSourceFromCommands(
         Function<ParsedCommand, OpDispenser<O>> opinit,
+        NBConfiguration cfg,
         List<Function<Map<String, Object>, Map<String, Object>>> parsers
     ) {
-        Function<OpTemplate, ParsedCommand> f = t -> new ParsedCommand(t, parsers);
+        Function<OpTemplate, ParsedCommand> f = t -> new ParsedCommand(t, cfg, parsers);
         Function<OpTemplate, OpDispenser<O>> opTemplateOFunction = f.andThen(opinit);
         return createOpSequence(opTemplateOFunction);
     }

engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java

@@ -6,6 +6,7 @@ import io.nosqlbench.engine.api.activityapi.planning.OpSource;
 import io.nosqlbench.engine.api.activityimpl.ActivityDef;
 import io.nosqlbench.engine.api.activityimpl.OpDispenser;
 import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
+import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op;
 import io.nosqlbench.engine.api.templating.ParsedCommand;
 import io.nosqlbench.nb.api.errors.OpConfigError;
 
@@ -20,7 +21,7 @@ import java.util.function.Function;
  *
  * @param <R> A type of runnable which wraps the operations for this type of driver.
  */
-public class StandardActivity<R extends Runnable,S> extends SimpleActivity {
+public class StandardActivity<R extends Op,S> extends SimpleActivity {
 
     private final DriverAdapter<R,S> adapter;
     private final OpSource<R> opsource;
@@ -34,7 +35,7 @@ public class StandardActivity<R extends Runnable,S> extends SimpleActivity {
         try {
             Function<ParsedCommand, OpDispenser<R>> opmapper = adapter.getOpMapper();
             Function<Map<String, Object>, Map<String, Object>> preprocessor = adapter.getPreprocessor();
-            sequence = createOpSourceFromCommands(opmapper,List.of(preprocessor));
+            sequence = createOpSourceFromCommands(opmapper, adapter.getConfiguration(), List.of(preprocessor));
             opsource= OpSource.of(sequence);
         } catch (Exception e) {
             if (e instanceof OpConfigError) {

nb-api/src/main/java/io/nosqlbench/nb/api/config/params/ParamsParser.java

@@ -20,10 +20,7 @@ package io.nosqlbench.nb.api.config.params;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.regex.Pattern;
 
 
@@ -272,6 +269,21 @@ public class ParamsParser {
         return parms;
     }
 
+    public static Map<String, String> parseToMap(Object src, String mainField) {
+        if (src instanceof Map) {
+            return (Map) src;
+        } else if (src instanceof CharSequence) {
+            String input = ((CharSequence) src).toString();
+            if (hasValues(input)) {
+                return parse(input, false);
+            } else {
+                return new HashMap<>(Map.of(mainField, input));
+            }
+        } else {
+            throw new RuntimeException("can't parseToMap(...) on an object that is neither Map nor CharSequence");
+        }
+    }
+
     private enum ParseState {
         expectingName,
         readingName,
@@ -280,4 +292,6 @@ public class ParamsParser {
         readingSquotedVal,
         readingDquotedVal
     }
+
+
 }

nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBTypeConverter.java

@@ -38,7 +38,8 @@ public class NBTypeConverter {
         if (outc.isAssignableFrom(input.getClass())) return true; // assignable
         if (ClassUtils.isAssignable(input.getClass(), outc, true)) return true; // assignable with boxing
         if (String.class.isAssignableFrom(outc)) return true; // all things can be strings
-        if (outc.isPrimitive() && outc != boolean.class && outc != void.class && (input instanceof Number)) return true; // via Number conversions
+        if (outc.isPrimitive() && outc != boolean.class && outc != void.class && (input instanceof Number))
+            return true; // via Number conversions
         return (lookup(input.getClass(), outc) != null); // fall-through to helper method lookup
     }
 
@@ -80,6 +81,13 @@ public class NBTypeConverter {
         return Optional.ofNullable(converted);
     }
 
+    public static <T> T convertOr(Object input, T defaultValue) {
+        if (input == null) {
+            return defaultValue;
+        }
+        return convert(input, (Class<T>) defaultValue.getClass());
+    }
+
     public static <T> T convert(Object input, Class<T> outType) {
 
         T converted = do_convert(input, outType);
@@ -155,7 +163,7 @@ public class NBTypeConverter {
             Object result = converter.invoke(null, input);
             return (T) result;
         } catch (Exception e) {
-            throw new RuntimeException("Unable to convert (" + input + ") to " + outType.getSimpleName() + ": " + e,e);
+            throw new RuntimeException("Unable to convert (" + input + ") to " + outType.getSimpleName() + ": " + e, e);
         }
     }