Merge branch 'main' into mwolters/UpdateComputeFunctions

.github/workflows/build.yml

@@ -14,24 +14,20 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         name: checkout nosqlbench
-        with:
-          node-version: '16'
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         name: setup java
         with:
-          node-version: '16'
-          java-version: '21'
-          java-package: jdk
           architecture: x64
           distribution: 'oracle'
+          java-package: jdk
+          java-version: '21'
 
       - name: Cache Maven packages
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
-          node-version: '16'
           path: ~/.m2
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
           restore-keys: ${{ runner.os }}-m2
@@ -71,9 +67,8 @@ jobs:
 
       - name: upload docs artifact
         if: success()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          node-version: '16'
           name: exported-docs
           path: exported_docs.zip
 
@@ -90,9 +85,8 @@ jobs:
         run: git config --global user.name "${{ secrets.NBDROID_NAME }}"
 
       - name: download exported_docs
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          node-version: '16'
           name: exported-docs
 
       - run: ls -la

adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/util/AmqpAdapterMetrics.java

@@ -20,6 +20,7 @@ import com.codahale.metrics.Counter;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
 import io.nosqlbench.adapter.amqp.dispensers.AmqpBaseOpDispenser;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.labels.NBLabeledElement;
 import io.nosqlbench.nb.api.labels.NBLabels;
 import org.apache.logging.log4j.LogManager;
@@ -55,17 +56,45 @@ public class AmqpAdapterMetrics {
 
     public void initS4JAdapterInstrumentation() {
         // Histogram metrics
-        messageSizeHistogram = amqpBaseOpDispenser.create().histogram("message_size");
+        messageSizeHistogram = amqpBaseOpDispenser.create().histogram(
+            "amqp_message_size",
+            MetricCategory.Driver,
+            "AMQP message size"
+        );
         // Timer metrics
-        bindTimer = amqpBaseOpDispenser.create().timer("bind");
-        executeTimer = amqpBaseOpDispenser.create().timer("execute");
+        bindTimer = amqpBaseOpDispenser.create().timer(
+            "amqp_bind",
+            MetricCategory.Driver,
+            "AMQP bind timer"
+        );
+        executeTimer = amqpBaseOpDispenser.create().timer(
+            "amqp_execute",
+            MetricCategory.Driver,
+            "AMQP execute timer"
+        );
         // End-to-end metrics
         // Latency
-        e2eMsgProcLatencyHistogram = amqpBaseOpDispenser.create().histogram("e2e_msg_latency");
+        e2eMsgProcLatencyHistogram = amqpBaseOpDispenser.create().histogram(
+            "amqp_e2e_msg_latency",
+            MetricCategory.Driver,
+            "AMQP end-to-end message processing latency"
+        );
         // Error metrics
-        msgErrOutOfSeqCounter = amqpBaseOpDispenser.create().counter("err_msg_oos");
-        msgErrLossCounter = amqpBaseOpDispenser.create().counter("err_msg_loss");
-        msgErrDuplicateCounter = amqpBaseOpDispenser.create().counter("err_msg_dup");
+        msgErrOutOfSeqCounter = amqpBaseOpDispenser.create().counter(
+            "amqp_err_msg_oos",
+            MetricCategory.Driver,
+            "AMQP out-of-sequence error count"
+        );
+        msgErrLossCounter = amqpBaseOpDispenser.create().counter(
+            "amqp_err_msg_loss",
+            MetricCategory.Driver,
+            "AMQP lost message error count"
+        );
+        msgErrDuplicateCounter = amqpBaseOpDispenser.create().counter(
+            "err_msg_dup",
+            MetricCategory.Driver,
+            "AMQP duplicate message count"
+        );
     }
 
     public Timer getBindTimer() { return bindTimer; }

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java

@@ -37,6 +37,7 @@ import org.apache.logging.log4j.Logger;
 
 import javax.net.ssl.SSLContext;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.InetSocketAddress;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -97,8 +98,18 @@ public class Cqld4Space implements AutoCloseable {
 //        int port = cfg.getOptional(int.class, "port").orElse(9042);
 
         Optional<String> scb = cfg.getOptional(String.class, "secureconnectbundle", "scb");
-        scb.flatMap(s -> NBIO.all().pathname(s).first().map(Content::getInputStream))
-            .map(builder::withCloudSecureConnectBundle);
+
+        if (scb.isPresent()) {
+            Optional<InputStream> stream =
+                scb.flatMap(s -> NBIO.all().pathname(s).first().map(Content::getInputStream));
+            if (stream.isPresent()) {
+                stream.map(builder::withCloudSecureConnectBundle);
+            } else {
+                String error = String.format("Unable to load Secure Connect Bundle from path %s", scb.get());
+                logger.error(error);
+                throw new RuntimeException(error);
+            }
+        }
 
         Optional<List<InetSocketAddress>> contactPointsOption = cfg
             .getOptional("host", "hosts")

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java

@@ -31,6 +31,7 @@ import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser;
 import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
 import io.nosqlbench.adapters.api.templating.ParsedOp;
 
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -57,9 +58,24 @@ public abstract class Cqld4BaseOpDispenser extends BaseOpDispenser<Cqld4CqlOp, C
         this.maxpages = op.getStaticConfigOr("maxpages", 1);
         this.isRetryReplace = op.getStaticConfigOr("retryreplace", false);
         this.maxLwtRetries = op.getStaticConfigOr("maxlwtretries", 1);
-        this.rowsHistogram = create().histogram("rows", op.getStaticConfigOr("hdr_digits", 3));
-        this.pagesHistogram = create().histogram("pages", op.getStaticConfigOr("hdr_digits", 3));
-        this.payloadBytesHistogram = create().histogram("payload_bytes", op.getStaticConfigOr("hdr_digits", 3));
+        this.rowsHistogram = create().histogram(
+            "rows",
+            op.getStaticConfigOr("hdr_digits", 3),
+            MetricCategory.Payload,
+            "The number of rows returned in the CQL result"
+        );
+        this.pagesHistogram = create().histogram(
+            "pages",
+            op.getStaticConfigOr("hdr_digits", 3),
+            MetricCategory.Payload,
+            "The number of pages returned in the CQL result"
+        );
+        this.payloadBytesHistogram = create().histogram(
+            "payload_bytes",
+            op.getStaticConfigOr("hdr_digits", 3),
+            MetricCategory.Payload,
+            "The number of bytes returned in the CQL result"
+        );
     }
 
     public int getMaxPages() {

adapter-cqld4/src/test/resources/testschemas/cql_alltypes.yaml

@@ -23,9 +23,9 @@ bindings:
   boolean: ToBoolean();
   date: EpochMillisToJavaLocalDate();
   decimal: ToBigDecimal();
-  double: ToDouble()
+  double: HashRange(0.0,1.0);
   duration: ToCqlDurationNanos();
-  float: ToFloat()
+  float: HashRange(0.0,1.0); DoubleToFloat();
   frozen<list<int>>: ListSizedHashed(HashRange(3,7),ToInt()));
   list<text>: ListStepped(NumberNameToString(),NumberNameToString())
   map<text,text>: MapSized(3, Combinations('A-Z;0-9'), NumberNameToString(), ToString());
@@ -47,11 +47,7 @@ blocks:
     ops:
       create-keyspace-ks_00001:
         simple: |
-          create keyspace ks_00001
-          with replication = {'class': 'SimpleStrategy',
-          'replication_factor': 'TEMPLATE(rf:1)'
-          }
-           and durable writes = false;
+          create keyspace IF NOT EXISTS ks_00001 with replication = {'class': 'SimpleStrategy','replication_factor': 'TEMPLATE(rf,1)'};
   schema-tables:
     params:
       timeout: 60.0
@@ -150,7 +146,7 @@ blocks:
       timeout: 30.0
     ops:
       scan-tb_00001-table-tb_00001:
-        prepared: "select * from ks_00001.tb_00001\nwhere \n LIMIT 10;\n"
+        prepared: "select * from ks_00001.tb_00001 LIMIT 10;"
         ratio: 1
   main-update:
     params:
@@ -159,7 +155,7 @@ blocks:
       update-tb_00001-table-tb_00001:
         prepared: |
           update ks_00001.tb_00001
-          set col_00001={text}, col_00002={bigint}, col_00003={blob}, col_00004={boolean}, col_00005={date}, col_00006={decimal}, col_00007={double}, col_00008={duration}, col_00009={float}, col_00010={frozen<list<int>>}, col_00011={list<text>}, col_00012={map<text,text>}, col_00013={set<text>}, col_00014={smallint}, col_00015={text}, col_00016={time}, col_00017={timestamp}, col_00018={timeuuid}, col_00019={tinyint}, col_00020={uuid}, col_00021={text}, col_00022={varint}, col_00023={ascii}, col_00024={inet}, col_00025={int}
+          set col_00002={bigint}, col_00003={blob}, col_00004={boolean}, col_00005={date}, col_00006={decimal}, col_00007={double}, col_00008={duration}, col_00009={float}, col_00010={frozen<list<int>>}, col_00011={list<text>}, col_00012={map<text,text>}, col_00013={set<text>}, col_00014={smallint}, col_00015={text}, col_00016={time}, col_00017={timestamp}, col_00018={timeuuid}, col_00019={tinyint}, col_00020={uuid}, col_00021={text}, col_00022={varint}, col_00023={ascii}, col_00024={inet}, col_00025={int}
           where col_00001={text};
         ratio: 1
 params:

adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagOpDispenser.java

@@ -23,7 +23,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfigModel;
 import io.nosqlbench.nb.api.config.standard.NBConfiguration;
 import io.nosqlbench.nb.api.config.standard.NBReconfigurable;
 import io.nosqlbench.nb.api.components.core.NBParentComponentInjection;
-import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
+import io.nosqlbench.engine.api.activityapi.simrate.RateLimiter;
 import io.nosqlbench.nb.annotations.ServiceSelector;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;

adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagSpace.java

@@ -17,7 +17,7 @@
 package io.nosqlbench.adapter.diag;
 
 import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
-import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
+import io.nosqlbench.engine.api.activityapi.simrate.RateLimiter;
 import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
 import io.nosqlbench.nb.api.config.standard.ConfigModel;
 import io.nosqlbench.nb.api.config.standard.NBConfigModel;

adapter-diag/src/main/java/io/nosqlbench/adapter/diag/optasks/DiagTask_diagrate.java

@@ -16,9 +16,9 @@
 
 package io.nosqlbench.adapter.diag.optasks;
 
-import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
-import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.SimRateSpec;
+import io.nosqlbench.engine.api.activityapi.simrate.RateLimiter;
+import io.nosqlbench.engine.api.activityapi.simrate.RateLimiters;
+import io.nosqlbench.engine.api.activityapi.simrate.SimRateSpec;
 import io.nosqlbench.nb.annotations.Service;
 import io.nosqlbench.nb.api.config.standard.*;
 

adapter-diag/src/main/java/io/nosqlbench/adapter/diag/optasks/DiagTask_gauge.java

@@ -21,6 +21,7 @@ import io.nosqlbench.nb.api.config.standard.ConfigModel;
 import io.nosqlbench.nb.api.config.standard.NBConfigModel;
 import io.nosqlbench.nb.api.config.standard.NBConfiguration;
 import io.nosqlbench.nb.api.config.standard.Param;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.labels.NBLabels;
 import io.nosqlbench.nb.api.components.core.NBParentComponentInjection;
 import io.nosqlbench.nb.annotations.Service;
@@ -136,7 +137,12 @@ public class DiagTask_gauge extends BaseDiagTask implements Gauge<Double>, NBPar
         }
 
         logger.info("Registering gauge for diag task with labels:" + getParentLabels().getLabels() + " label:" + label);
-        this.gauge=parent.create().gauge(label,() -> this.sampleValue);
+        this.gauge=parent.create().gauge(
+            label,
+            () -> this.sampleValue,
+            MetricCategory.Verification,
+            "a diagnostic gauge for the purposes of testing " + description()
+        );
     }
 
     @Override

adapter-dynamodb/pom.xml

@@ -43,7 +43,7 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-dynamodb</artifactId>
-            <version>1.12.595</version>
+            <version>1.12.635</version>
         </dependency>
 
     </dependencies>

adapter-http/pom.xml

@@ -72,7 +72,7 @@
         <dependency>
             <groupId>io.swagger.core.v3</groupId>
             <artifactId>swagger-core</artifactId>
-            <version>2.2.19</version>
+            <version>2.2.20</version>
         </dependency>
     </dependencies>
 

adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpMetrics.java

@@ -17,6 +17,7 @@
 package io.nosqlbench.adapter.http.core;
 
 import com.codahale.metrics.Histogram;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.labels.NBLabeledElement;
 import io.nosqlbench.nb.api.labels.NBLabels;
 import io.nosqlbench.nb.api.components.core.NBComponent;
@@ -29,7 +30,12 @@ public class HttpMetrics implements NBLabeledElement {
     public HttpMetrics(NBComponent parent, HttpSpace space) {
         this.parent = parent;
         this.space = space;
-        statusCodeHistogram = parent.create().histogram("statuscode",space.getHdrDigits());
+        statusCodeHistogram = parent.create().histogram(
+            "statuscode",
+            space.getHdrDigits(),
+            MetricCategory.Payload,
+            "A histogram of status codes received by the HTTP client"
+        );
     }
 
     public String getName() {

adapter-kafka/pom.xml

@@ -34,7 +34,7 @@
     </description>
 
     <properties>
-        <kafka.version>3.6.0</kafka.version>
+        <kafka.version>3.6.1</kafka.version>
     </properties>
 
     <dependencies>
@@ -55,7 +55,7 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
-            <version>3.13.0</version>
+            <version>3.14.0</version>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils -->

adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/util/KafkaAdapterMetrics.java

@@ -20,8 +20,10 @@ import com.codahale.metrics.Counter;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
 import io.nosqlbench.adapter.kafka.dispensers.KafkaBaseOpDispenser;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.labels.NBLabeledElement;
 import io.nosqlbench.nb.api.labels.NBLabels;
+import org.apache.kafka.common.Metric;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -50,37 +52,70 @@ public class KafkaAdapterMetrics {
 
     public KafkaAdapterMetrics(final KafkaBaseOpDispenser kafkaBaseOpDispenser, final NBLabeledElement labeledParent) {
         this.kafkaBaseOpDispenser = kafkaBaseOpDispenser;
-        labels=labeledParent.getLabels().and("name",KafkaAdapterMetrics.class.getSimpleName());
+        labels = labeledParent.getLabels().and("name", KafkaAdapterMetrics.class.getSimpleName());
     }
 
     public void initS4JAdapterInstrumentation() {
         // Histogram metrics
-        messageSizeHistogram = kafkaBaseOpDispenser.create().histogram("message_size");
+        messageSizeHistogram = kafkaBaseOpDispenser.create().histogram(
+            "kafka_message_size",
+            MetricCategory.Payload,
+            "kafka message size"
+        );
 
         // Timer metrics
         bindTimer =
             this.kafkaBaseOpDispenser.create().timer(
-                    "bind");
+                "kafka_bind",
+                MetricCategory.Payload,
+                "kafka bind timer");
         executeTimer =
             this.kafkaBaseOpDispenser.create().timer(
-                     "execute");
+                "kafka_execute",
+                MetricCategory.Payload,
+                "kafka execute timer"
+            );
 
         // End-to-end metrics
         // Latency
         e2eMsgProcLatencyHistogram =
-            kafkaBaseOpDispenser.create().histogram("e2e_msg_latency");
+            kafkaBaseOpDispenser.create().histogram(
+                "e2e_msg_latency",
+                MetricCategory.Driver,
+                "End-to-end kafka message latency"
+            );
         // Error metrics
         msgErrOutOfSeqCounter =
-            kafkaBaseOpDispenser.create().counter("err_msg_oos");
+            kafkaBaseOpDispenser.create().counter(
+                "kafka_err_msg_oos",
+                MetricCategory.Driver,
+                "kafka Out-of-sequence errors"
+            );
         msgErrLossCounter =
-            kafkaBaseOpDispenser.create().counter("err_msg_loss");
+            kafkaBaseOpDispenser.create().counter(
+                "kafka_err_msg_loss",
+                MetricCategory.Driver,
+                "kafka message loss errors"
+            );
         msgErrDuplicateCounter =
-            kafkaBaseOpDispenser.create().counter( "err_msg_dup");
+            kafkaBaseOpDispenser.create().counter(
+                "kafka_err_msg_dup",
+                MetricCategory.Driver,
+                "kafka duplicate message errors"
+            );
     }
 
-    public Timer getBindTimer() { return bindTimer; }
-    public Timer getExecuteTimer() { return executeTimer; }
-    public Histogram getMessagesizeHistogram() { return messageSizeHistogram; }
+    public Timer getBindTimer() {
+        return bindTimer;
+    }
+
+    public Timer getExecuteTimer() {
+        return executeTimer;
+    }
+
+    public Histogram getMessagesizeHistogram() {
+        return messageSizeHistogram;
+    }
 
     public Counter getMsgErrOutOfSeqCounter() {
         return msgErrOutOfSeqCounter;

adapter-pulsar/pom.xml

@@ -34,7 +34,7 @@
     </description>
 
     <properties>
-        <pulsar.version>3.1.1</pulsar.version>
+        <pulsar.version>3.1.2</pulsar.version>
     </properties>
 
     <dependencies>
@@ -61,7 +61,7 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
-            <version>3.13.0</version>
+            <version>3.14.0</version>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils -->

adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java

@@ -20,6 +20,7 @@ import com.codahale.metrics.Counter;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
 import io.nosqlbench.adapter.pulsar.dispensers.PulsarBaseOpDispenser;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.pulsar.client.api.Consumer;
@@ -66,22 +67,52 @@ public class PulsarAdapterMetrics {
     public void initPulsarAdapterInstrumentation() {
         // Counter metrics
         msgErrOutOfSeqCounter =
-            pulsarBaseOpDispenser.create().counter("err_msg_oos");
+            pulsarBaseOpDispenser.create().counter("pulsar_err_msg_oos",
+                MetricCategory.Driver,
+                "pulsar out-of-sequence error counter"
+            );
         msgErrLossCounter =
-            pulsarBaseOpDispenser.create().counter("err_msg_loss");
+            pulsarBaseOpDispenser.create().counter("pulsar_err_msg_loss",
+                MetricCategory.Driver,
+                "pulsar lost message error counter"
+            );
         msgErrDuplicateCounter =
-            pulsarBaseOpDispenser.create().counter("err_msg_dup");
+            pulsarBaseOpDispenser.create().counter("pulsar_err_msg_dup",
+                MetricCategory.Driver,
+                "pulsar duplicate message error counter"
+            );
 
         // Histogram metrics
-        messageSizeHistogram = pulsarBaseOpDispenser.create().histogram("message_size");
-        e2eMsgProcLatencyHistogram = pulsarBaseOpDispenser.create().histogram("e2e_msg_latency");
-        payloadRttHistogram = pulsarBaseOpDispenser.create().histogram("payload_rtt");
+        messageSizeHistogram = pulsarBaseOpDispenser.create().histogram("pulsar_message_size",
+            MetricCategory.Driver,
+            "pulsar message size"
+        );
+        e2eMsgProcLatencyHistogram = pulsarBaseOpDispenser.create().histogram("pulsar_e2e_msg_latency",
+            MetricCategory.Driver,
+            "pulsar end-to-end message latency"
+        );
+        payloadRttHistogram = pulsarBaseOpDispenser.create().histogram("pulsar_payload_rtt",
+            MetricCategory.Driver,
+            "pulsar payload round-trip-time"
+        );
 
         // Timer metrics
-        bindTimer = pulsarBaseOpDispenser.create().timer("bind");
-        executeTimer = pulsarBaseOpDispenser.create().timer("execute");
-        createTransactionTimer = pulsarBaseOpDispenser.create().timer("create_transaction");
-        commitTransactionTimer = pulsarBaseOpDispenser.create().timer("commit_transaction");
+        bindTimer = pulsarBaseOpDispenser.create().timer("pulsar_bind",
+            MetricCategory.Driver,
+            "pulsar bind timer"
+        );
+        executeTimer = pulsarBaseOpDispenser.create().timer("pulsar_execute",
+            MetricCategory.Driver,
+            "pulsar execution timer"
+        );
+        createTransactionTimer = pulsarBaseOpDispenser.create().timer("pulsar_create_transaction",
+            MetricCategory.Driver,
+            "pulsar create transaction timer"
+        );
+        commitTransactionTimer = pulsarBaseOpDispenser.create().timer("pulsar_commit_transaction",
+            MetricCategory.Driver,
+            "pulsar commit transaction timer"
+        );
     }
 
     public Counter getMsgErrOutOfSeqCounter() {
@@ -154,12 +185,36 @@ public class PulsarAdapterMetrics {
 
     public void registerProducerApiMetrics(final Producer<?> producer) {
 
-        pulsarBaseOpDispenser.create().gauge("total_bytes_sent", PulsarAdapterMetrics.producerSafeExtractMetric(producer, s -> (double) s.getTotalBytesSent() + s.getNumBytesSent()));
-        pulsarBaseOpDispenser.create().gauge("total_msg_sent", PulsarAdapterMetrics.producerSafeExtractMetric(producer, s -> (double) s.getTotalMsgsSent() + s.getNumMsgsSent()));
-        pulsarBaseOpDispenser.create().gauge("total_send_failed", PulsarAdapterMetrics.producerSafeExtractMetric(producer, s -> (double) s.getTotalSendFailed() + s.getNumSendFailed()));
-        pulsarBaseOpDispenser.create().gauge("total_ack_received", PulsarAdapterMetrics.producerSafeExtractMetric(producer, s -> (double) s.getTotalAcksReceived() + s.getNumAcksReceived()));
-        pulsarBaseOpDispenser.create().gauge("send_bytes_rate", PulsarAdapterMetrics.producerSafeExtractMetric(producer, ProducerStats::getSendBytesRate));
-        pulsarBaseOpDispenser.create().gauge("send_msg_rate", PulsarAdapterMetrics.producerSafeExtractMetric(producer, ProducerStats::getSendMsgsRate));
+        pulsarBaseOpDispenser.create().gauge(
+            "pulsar_total_bytes_sent",
+            PulsarAdapterMetrics.producerSafeExtractMetric(producer, s -> (double) s.getTotalBytesSent() + s.getNumBytesSent()),
+            MetricCategory.Driver,
+            "pulsar total bytes sent"
+        );
+        pulsarBaseOpDispenser.create().gauge(
+            "pulsar_total_msg_sent",
+            PulsarAdapterMetrics.producerSafeExtractMetric(producer, s -> (double) s.getTotalMsgsSent() + s.getNumMsgsSent()),
+            MetricCategory.Driver,
+            "pulsar total message sent"
+        );
+        pulsarBaseOpDispenser.create().gauge(
+            "pulsar_total_send_failed",
+            PulsarAdapterMetrics.producerSafeExtractMetric(producer, s -> (double) s.getTotalSendFailed() + s.getNumSendFailed()),
+            MetricCategory.Driver,
+            "pulsar message send failures"
+        );
+        pulsarBaseOpDispenser.create().gauge("pulsar_total_ack_received", PulsarAdapterMetrics.producerSafeExtractMetric(producer, s -> (double) s.getTotalAcksReceived() + s.getNumAcksReceived()),
+            MetricCategory.Driver,
+            "pulsar total acknowledgements received"
+        );
+        pulsarBaseOpDispenser.create().gauge("pulsar_send_bytes_rate", PulsarAdapterMetrics.producerSafeExtractMetric(producer, ProducerStats::getSendBytesRate),
+            MetricCategory.Driver,
+            "pulsar rate of bytes sent"
+        );
+        pulsarBaseOpDispenser.create().gauge("pulsar_send_msg_rate", PulsarAdapterMetrics.producerSafeExtractMetric(producer, ProducerStats::getSendMsgsRate),
+            MetricCategory.Driver,
+            "pulsar rate of messages sent"
+        );
     }
 
 
@@ -193,17 +248,38 @@ public class PulsarAdapterMetrics {
 
     public void registerConsumerApiMetrics(final Consumer<?> consumer, final String pulsarApiMetricsPrefix) {
 
-        pulsarBaseOpDispenser.create().gauge("total_bytes_recv",
-            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getTotalBytesReceived() + s.getNumBytesReceived()));
-        pulsarBaseOpDispenser.create().gauge("total_msg_recv",
-            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getTotalMsgsReceived() + s.getNumMsgsReceived()));
-        pulsarBaseOpDispenser.create().gauge("total_recv_failed",
-            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getTotalReceivedFailed() + s.getNumReceiveFailed()));
-        pulsarBaseOpDispenser.create().gauge("total_acks_sent",
-            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getTotalAcksSent() + s.getNumAcksSent()));
-        pulsarBaseOpDispenser.create().gauge("recv_bytes_rate",
-            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getRateBytesReceived()));
-        pulsarBaseOpDispenser.create().gauge("recv_msg_rate",
-            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getRateMsgsReceived()));
+        pulsarBaseOpDispenser.create().gauge(
+            "pulsar_total_bytes_recv",
+            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getTotalBytesReceived() + s.getNumBytesReceived()),
+            MetricCategory.Driver,
+            "pulsar total bytes received"
+        );
+        pulsarBaseOpDispenser.create().gauge(
+            "pulsar_total_msg_recv",
+            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getTotalMsgsReceived() + s.getNumMsgsReceived()),
+            MetricCategory.Driver,
+            "pulsar total messages received"
+        );
+        pulsarBaseOpDispenser.create().gauge(
+            "pulsar_total_recv_failed",
+            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getTotalReceivedFailed() + s.getNumReceiveFailed()),
+            MetricCategory.Driver,
+            "pulsar total receive failures"
+        );
+        pulsarBaseOpDispenser.create().gauge("pulsar_total_acks_sent",
+            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getTotalAcksSent() + s.getNumAcksSent()),
+            MetricCategory.Driver,
+            "pulsar total acknowledgements sent"
+        );
+        pulsarBaseOpDispenser.create().gauge("pulsar_recv_bytes_rate",
+            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getRateBytesReceived()),
+            MetricCategory.Driver,
+            "pulsar rate of bytes received"
+        );
+        pulsarBaseOpDispenser.create().gauge("pulsar_recv_msg_rate",
+            PulsarAdapterMetrics.consumerSafeExtractMetric(consumer, s -> (double) s.getRateMsgsReceived()),
+            MetricCategory.Driver,
+            "pulsar rate of message received"
+        );
     }
 }

adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JAdapterMetrics.java

@@ -19,6 +19,7 @@ package io.nosqlbench.adapter.s4j.util;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
 import io.nosqlbench.adapter.s4j.dispensers.S4JBaseOpDispenser;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -38,11 +39,23 @@ public class S4JAdapterMetrics  {
 
     public void initS4JAdapterInstrumentation() {
         // Histogram metrics
-        this.messageSizeHistogram = s4jBaseOpDispenser.create().histogram("message_size");
+        this.messageSizeHistogram = s4jBaseOpDispenser.create().histogram(
+            "s4j_message_size",
+            MetricCategory.Driver,
+            "S4J message size"
+        );
 
         // Timer metrics
-        this.bindTimer = s4jBaseOpDispenser.create().timer("bind");
-        this.executeTimer = s4jBaseOpDispenser.create().timer("execute");
+        this.bindTimer = s4jBaseOpDispenser.create().timer(
+            "s4j_bind",
+            MetricCategory.Driver,
+            "S4J bind timer"
+        );
+        this.executeTimer = s4jBaseOpDispenser.create().timer(
+            "s4j_execute",
+            MetricCategory.Driver,
+            "S4j execute timer"
+        );
     }
 
     public Timer getBindTimer() { return bindTimer; }

adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDef.java

@@ -43,7 +43,7 @@ public class OpDef extends OpTemplate {
 
     @Override
     public String getName() {
-        return block.getName() + "__" + rawOpDef.getName();
+        return rawOpDef.getName();
     }
 
     @Override
@@ -100,7 +100,7 @@ public class OpDef extends OpTemplate {
     private LinkedHashMap<String, String> composeTags() {
         LinkedHashMap<String, String> tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawOpDef.getTags(), block.getTags()));
         tagsWithName.put("block",block.getName());
-        tagsWithName.put("name",getName());
+        tagsWithName.put("name",this.rawOpDef.getName());
         tagsWithName.put("op",this.rawOpDef.getName());
         return tagsWithName;
     }

adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/BaseOpDispenser.java

@@ -23,6 +23,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
 import io.nosqlbench.adapters.api.evalctx.*;
 import io.nosqlbench.adapters.api.metrics.ThreadLocalNamedTimers;
 import io.nosqlbench.adapters.api.templating.ParsedOp;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.labels.NBLabels;
 import io.nosqlbench.nb.api.errors.OpConfigError;
 import io.nosqlbench.nb.api.components.core.NBBaseComponent;
@@ -99,7 +100,12 @@ public abstract class BaseOpDispenser<T extends Op, S> extends NBBaseComponent i
         verifiers = configureVerifiers(op);
         this._verifier = CycleFunctions.of((a, b) -> a && b, verifiers, true);
         this.tlVerifier = ThreadLocal.withInitial(_verifier::newInstance);
-        this.verifierTimer = create().timer("verifier",3);
+        this.verifierTimer = create().timer(
+            "verifier",
+            3,
+            MetricCategory.Verification,
+            "Time verifier execution, if any."
+        );
     }
 
     private CycleFunction<Boolean> cloneVerifiers() {
@@ -126,7 +132,6 @@ public abstract class BaseOpDispenser<T extends Op, S> extends NBBaseComponent i
         Binding variables = new Binding();
 
         Map<String, ParsedTemplateString> initBlocks = op.getTemplateMap().takeAsNamedTemplates(VERIFIER_INIT);
-        List<CycleFunction<?>> verifierInitFunctions = new ArrayList<>();
         try {
             initBlocks.forEach((initName, stringTemplate) -> {
                 GroovyCycleFunction<?> initFunction =
@@ -180,8 +185,18 @@ public abstract class BaseOpDispenser<T extends Op, S> extends NBBaseComponent i
         if (this.instrument) {
             final int hdrDigits = pop.getStaticConfigOr("hdr_digits", 4);
 
-            successTimer = create().timer("successfor_"+getOpName(),hdrDigits);
-            errorTimer = create().timer("errorsfor_"+getOpName(),hdrDigits);
+            successTimer = create().timer(
+                "successfor_"+getOpName(),
+                hdrDigits,
+                MetricCategory.Core,
+                "Successful result timer for specific operation '" + pop.getName() + "'"
+            );
+            errorTimer = create().timer(
+                "errorsfor_"+getOpName(),
+                hdrDigits,
+                MetricCategory.Core,
+                "Errored result timer for specific operation '" + pop.getName() + "'"
+            );
         }
     }
 

adapters-api/src/main/java/io/nosqlbench/adapters/api/metrics/ThreadLocalNamedTimers.java

@@ -19,6 +19,7 @@ package io.nosqlbench.adapters.api.metrics;
 import com.codahale.metrics.Timer;
 import com.codahale.metrics.Timer.Context;
 import io.nosqlbench.adapters.api.templating.ParsedOp;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricTimer;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -41,7 +42,12 @@ public class ThreadLocalNamedTimers {
     public static void addTimer(final ParsedOp pop, final String name) {
         if (ThreadLocalNamedTimers.timers.containsKey("name"))
             ThreadLocalNamedTimers.logger.warn("A timer named '{}' was already defined and initialized.", name);
-        NBMetricTimer timer = pop.create().timer(name, 3);
+        NBMetricTimer timer = pop.create().timer(
+            name,
+            3,
+            MetricCategory.User,
+            "User-provided metric, defined on op '" + pop.getName() + "'"
+        );
         ThreadLocalNamedTimers.timers.put(name, timer);
     }
 

adapters-api/src/main/resources/workload_definition/02_workload_structure.md

@@ -375,24 +375,24 @@ blocks:
 
 [
   {
-    "name": "namedblock1__op1",
+    "name": "op1",
     "op": {
       "stmt": "select * from bar.table;"
     },
     "tags": {
-      "name": "namedblock1__op1",
+      "name": "op1",
       "block": "namedblock1",
       "op": "op1"
     }
   },
   {
-    "name": "namedblock1__op2",
+    "name": "op2",
     "op": {
       "stmt": "insert into bar.table (a,b,c) values (1,2,3);",
       "type": "batch"
     },
     "tags": {
-      "name": "namedblock1__op2",
+      "name": "op2",
       "block": "namedblock1",
       "op": "op2"
     }
@@ -450,35 +450,35 @@ blocks:
 
 [
   {
-    "name": "block1__op1",
+    "name": "op1",
     "op": {
       "stmt": "select * from bar.table;"
     },
     "tags": {
-      "name": "block1__op1",
+      "name": "op1",
       "block": "block1",
       "op": "op1"
     }
   },
   {
-    "name": "block1__op2",
+    "name": "op2",
     "op": {
       "stmt": "insert into bar.table (a,b,c) values (1,2,3);",
       "type": "batch"
     },
     "tags": {
-      "name": "block1__op2",
+      "name": "op2",
       "block": "block1",
       "op": "op2"
     }
   },
   {
-    "name": "this_is_block_2__op3",
+    "name": "op3",
     "op": {
       "stmt": "select * from foo.table;"
     },
     "tags": {
-      "name": "this_is_block_2__op3",
+      "name": "op3",
       "block": "this_is_block_2",
       "op": "op3"
     }
@@ -523,12 +523,12 @@ blocks:
 
 [
   {
-    "name": "myblock__stmt1",
+    "name": "stmt1",
     "op": {
       "stmt": "test op"
     },
     "tags": {
-      "name": "myblock__stmt1",
+      "name": "stmt1",
       "block": "myblock",
       "op": "stmt1"
     }

adapters-api/src/main/resources/workload_definition/04_op_template_basics.md

@@ -34,12 +34,12 @@ op: select * from bar.table;
 
 [
   {
-    "name": "block0__stmt1",
+    "name": "stmt1",
     "op": {
       "stmt": "select * from bar.table;"
     },
     "tags": {
-      "name": "block0__stmt1",
+      "name": "stmt1",
       "block": "block0",
       "op": "stmt1"
     }
@@ -73,12 +73,12 @@ ops:
 
 [
   {
-    "name": "block0__stmt1",
+    "name": "stmt1",
     "op": {
       "stmt": "select * from bar.table;"
     },
     "tags": {
-      "name": "block0__stmt1",
+      "name": "stmt1",
       "block": "block0",
       "op": "stmt1"
     }
@@ -117,12 +117,12 @@ ops:
 
 [
   {
-    "name": "block0__op1",
+    "name": "op1",
     "op": {
       "stmt": "select * from bar.table;"
     },
     "tags": {
-      "name": "block0__op1",
+      "name": "op1",
       "block": "block0",
       "op": "op1"
     }
@@ -160,12 +160,12 @@ ops:
 
 [
   {
-    "name": "block0__op1",
+    "name": "op1",
     "op": {
       "stmt": "select * from bar.table;"
     },
     "tags": {
-      "name": "block0__op1",
+      "name": "op1",
       "block": "block0",
       "op": "op1"
     }
@@ -199,12 +199,12 @@ ops:
 
 [
   {
-    "name": "block0__op1",
+    "name": "op1",
     "op": {
       "stmt": "select * from bar.table;"
     },
     "tags": {
-      "name": "block0__op1",
+      "name": "op1",
       "block": "block0",
       "op": "op1"
     }
@@ -241,12 +241,12 @@ ops:
 
 [
   {
-    "name": "block0__op1",
+    "name": "op1",
     "op": {
       "stmt": "select * from bar.table;"
     },
     "tags": {
-      "name": "block0__op1",
+      "name": "op1",
       "block": "block0",
       "op": "op1"
     }
@@ -315,7 +315,7 @@ ops:
       "binding1": "NumberNameToString();"
     },
     "description": "This is just an example operation",
-    "name": "block0__special-op-name",
+    "name": "special-op-name",
     "op": {
       "stmt": "select * from ks1.tb1;"
     },
@@ -324,7 +324,7 @@ ops:
     },
     "tags": {
       "block": "block0",
-      "name": "block0__special-op-name",
+      "name": "special-op-name",
       "op": "special-op-name"
     }
   }
@@ -414,7 +414,7 @@ blocks:
     "bindings": {
       "binding1": "NumberNameToString();"
     },
-    "name": "block_named_fred__special-op-name",
+    "name": "special-op-name",
     "op": {
       "stmt": "select * from ks1.tb1;"
     },
@@ -424,7 +424,7 @@ blocks:
     "tags": {
       "block": "block_named_fred",
       "docleveltag": "is-tagging-everything",
-      "name": "block_named_fred__special-op-name",
+      "name": "special-op-name",
       "op": "special-op-name"
     }
   }

adapters-api/src/main/resources/workload_definition/05_op_template_payloads.md

@@ -39,14 +39,14 @@ ops: "cycle number '{{NumberNameToString}}'"
 [
   {
     "tags": {
-      "name": "block0__stmt1",
+      "name": "stmt1",
       "block": "block0",
       "op": "stmt1"
     },
     "op": {
       "stmt": "cycle number '{{NumberNameToString}}'"
     },
-    "name": "block0__stmt1"
+    "name": "stmt1"
   }
 ]
 ```
@@ -83,25 +83,25 @@ ops:
 [
   {
     "tags": {
-      "name": "block0__stmt1",
+      "name": "stmt1",
       "block": "block0",
       "op": "stmt1"
     },
     "op": {
       "stmt": "even cycle '{{NumberNameToString}}'"
     },
-    "name": "block0__stmt1"
+    "name": "stmt1"
   },
   {
     "tags": {
-      "name": "block0__stmt2",
+      "name": "stmt2",
       "block": "block0",
       "op": "stmt2"
     },
     "op": {
       "stmt": "odd  cycle '{{NumberNameToString}}'"
     },
-    "name": "block0__stmt2"
+    "name": "stmt2"
   }
 ]
 ```
@@ -138,25 +138,25 @@ ops:
 [
   {
     "tags": {
-      "name": "block0__myop1",
+      "name": "myop1",
       "block": "block0",
       "op": "myop1"
     },
     "op": {
       "stmt": "even cycle '{{NumberNameToString}}'"
     },
-    "name": "block0__myop1"
+    "name": "myop1"
   },
   {
     "tags": {
-      "name": "block0__myop2",
+      "name": "myop2",
       "block": "block0",
       "op": "myop2"
     },
     "op": {
       "stmt": "odd  cycle '{{NumberNameToString}}'"
     },
-    "name": "block0__myop2"
+    "name": "myop2"
   }
 ]
 ```
@@ -198,7 +198,7 @@ ops:
 [
   {
     "tags": {
-      "name": "block0__op1",
+      "name": "op1",
       "block": "block0",
       "op": "op1"
     },
@@ -206,7 +206,7 @@ ops:
       "opfield1": "opvalue1",
       "opfield2": "opvalue2"
     },
-    "name": "block0__op1"
+    "name": "op1"
   }
 ]
 ```
@@ -250,7 +250,7 @@ ops:
 [
   {
     "tags": {
-      "name": "block0__op1",
+      "name": "op1",
       "block": "block0",
       "op": "op1"
     },
@@ -260,7 +260,7 @@ ops:
         "opvalue2"
       ]
     },
-    "name": "block0__op1"
+    "name": "op1"
   }
 ]
 ```
@@ -310,7 +310,7 @@ ops:
 [
   {
     "tags": {
-      "name": "block0__op1",
+      "name": "op1",
       "block": "block0",
       "op": "op1"
     },
@@ -324,7 +324,7 @@ ops:
         "by_session_len/@1h"
       ]
     },
-    "name": "block0__op1"
+    "name": "op1"
   }
 ]
 
@@ -384,7 +384,7 @@ ops:
       "user_index": "Mod(1000L); ToString();"
     },
     "tags": {
-      "name": "block0__op1",
+      "name": "op1",
       "block": "block0",
       "op": "op1"
     },
@@ -398,7 +398,7 @@ ops:
         "by_session_len/@1h"
       ]
     },
-    "name": "block0__op1"
+    "name": "op1"
   }
 ]
 ```

adapters-api/src/main/resources/workload_definition/06_op_template_variations.md

@@ -42,13 +42,13 @@ ops:
 
 [
   {
-    "name": "block0__special-op-name",
+    "name": "special-op-name",
     "op": {
       "stmt": "select * from ks1.tb1;"
     },
     "tags": {
       "block": "block0",
-      "name": "block0__special-op-name",
+      "name": "special-op-name",
       "op": "special-op-name"
     }
   }
@@ -94,14 +94,14 @@ ops:
 
 [
   {
-    "name": "block0__op1",
+    "name": "op1",
     "op": {
       "field1": "select * from ks1.tb1;",
       "field2": "field 2 value"
     },
     "tags": {
       "block": "block0",
-      "name": "block0__op1",
+      "name": "op1",
       "op": "op1"
     }
   }
@@ -141,14 +141,14 @@ ops:
 
 [
   {
-    "name": "block0__op1",
+    "name": "op1",
     "op": {
       "field1": "select * from ks1.tb1;",
       "field2": "field 2 value"
     },
     "tags": {
       "block": "block0",
-      "name": "block0__op1",
+      "name": "op1",
       "op": "op1"
     }
   }
@@ -191,7 +191,7 @@ ops:
 
 [
   {
-    "name": "block0__op1",
+    "name": "op1",
     "op": {
       "field1": "select * from ks1.tb1;",
       "field2": "field 2 value"
@@ -201,7 +201,7 @@ ops:
     },
     "tags": {
       "block": "block0",
-      "name": "block0__op1",
+      "name": "op1",
       "op": "op1"
     }
   }
@@ -244,7 +244,7 @@ ops:
 
 [
   {
-    "name": "block0__op1",
+    "name": "op1",
     "op": {
       "field1": "select * from ks1.tb1;",
       "field2": "field 2 value"
@@ -254,7 +254,7 @@ ops:
     },
     "tags": {
       "block": "block0",
-      "name": "block0__op1",
+      "name": "op1",
       "op": "op1"
     }
   }
@@ -302,7 +302,7 @@ ops:
 
 [
   {
-    "name": "block0__op1",
+    "name": "op1",
     "op": {
       "field1": "select * from ks1.tb1;",
       "field2": "field 2 value"
@@ -312,7 +312,7 @@ ops:
     },
     "tags": {
       "block": "block0",
-      "name": "block0__op1",
+      "name": "op1",
       "op": "op1"
     }
   }
@@ -351,14 +351,14 @@ ops: "my test op"
       "pname": "pvalue"
     },
     "tags": {
-      "name": "block0__stmt1",
+      "name": "stmt1",
       "block": "block0",
       "op": "stmt1"
     },
     "op": {
       "stmt": "my test op"
     },
-    "name": "block0__stmt1"
+    "name": "stmt1"
   }
 ]
 ```
@@ -401,14 +401,14 @@ blocks:
       "pname": "pvalue"
     },
     "tags": {
-      "name": "block1__stmt1",
+      "name": "stmt1",
       "block": "block1",
       "op": "stmt1"
     },
     "op": {
       "stmt": "my test op"
     },
-    "name": "block1__stmt1"
+    "name": "stmt1"
   }
 ]
 ```
@@ -458,14 +458,14 @@ blocks:
       "pname": "pvalue"
     },
     "tags": {
-      "name": "block1__op1",
+      "name": "op1",
       "block": "block1",
       "op": "op1"
     },
     "op": {
       "stmt": "my test op"
     },
-    "name": "block1__op1"
+    "name": "op1"
   }
 ]
 ```
@@ -520,7 +520,7 @@ blocks:
 [
   {
     "tags": {
-      "name": "block1__op1",
+      "name": "op1",
       "block": "block1",
       "op": "op1"
     },
@@ -530,7 +530,7 @@ blocks:
         "pname": "pvalue"
       }
     },
-    "name": "block1__op1"
+    "name": "op1"
   }
 ]
 ```
@@ -583,18 +583,18 @@ blocks:
 [
   {
     "tags": {
-      "name": "block1__op1",
+      "name": "op1",
       "block": "block1",
       "op": "op1"
     },
     "op": {
       "stmt": "my test op"
     },
-    "name": "block1__op1"
+    "name": "op1"
   },
   {
     "tags": {
-      "name": "block1__params",
+      "name": "params",
       "block": "block1",
       "op": "params"
     },
@@ -602,7 +602,7 @@ blocks:
       "pname": "pvalue"
 
     },
-    "name": "block1__params"
+    "name": "params"
   }
 ]
 ```

adapters-api/src/test/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDetailOverrideTest.java

@@ -43,32 +43,32 @@ public class OpDetailOverrideTest {
         OpsBlock doc1block0 = doc1.getBlocks().get(0);
         assertThat(doc1block0.getOps().size()).isEqualTo(1);
         OpTemplate s = doc1block0.getOps().get(0);
-        assertThat(s.getName()).isEqualTo("block0__stmt1");
+        assertThat(s.getName()).isEqualTo("stmt1");
         assertThat(s.getStmt()).contains("globalstatement1");
         assertThat(s.getBindings()).hasSize(1);
         assertThat(s.getParams()).hasSize(1);
-        assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","block0__stmt1", "op","stmt1"));
+        assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","stmt1", "op","stmt1"));
 
         OpsBlock doc1block1 = doc1.getBlocks().get(1);
         List<OpTemplate> ops = doc1block1.getOps();
         assertThat(ops).hasSize(4);
 
         s = ops.get(0);
-        assertThat(s.getName()).isEqualTo("testblock1__stmt1");
+        assertThat(s.getName()).isEqualTo("stmt1");
         assertThat(s.getStmt()).contains("astatement1");
-        assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1__stmt1","op","stmt1"));
+        assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","stmt1","op","stmt1"));
         assertThat(s.getBindings()).hasSize(1);
         assertThat(s.getParams()).hasSize(1);
 
         s = ops.get(1);
-        assertThat(s.getName()).isEqualTo("testblock1__s2name");
+        assertThat(s.getName()).isEqualTo("s2name");
         assertThat(s.getStmt()).contains("s2statement data");
-        assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1__s2name","op","s2name"));
+        assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","s2name","op","s2name"));
         assertThat(s.getBindings()).hasSize(1);
         assertThat(s.getParams()).hasSize(1);
 
         s = ops.get(2);
-        assertThat(s.getName()).isEqualTo("testblock1__s3");
+        assertThat(s.getName()).isEqualTo("s3");
         assertThat(s.getStmt()).contains("statement three");
         assertThat(s.getTags()).containsEntry("tname1", "tval1");
         assertThat(s.getTags()).containsEntry("global_tag1", "tag value");
@@ -76,9 +76,9 @@ public class OpDetailOverrideTest {
         assertThat(s.getParams()).hasSize(2);
 
         s = ops.get(3);
-        assertThat(s.getName()).isEqualTo("testblock1__s4");
+        assertThat(s.getName()).isEqualTo("s4");
         assertThat(s.getStmt()).contains("statement 4");
-        assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1__s4","op","s4"));
+        assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","s4","op","s4"));
         assertThat(s.getBindings()).hasSize(1);
         assertThat(s.getParams()).hasSize(1);
 

docsys/pom.xml

@@ -22,7 +22,7 @@
     <name>docsys</name>
     <url>http://nosqlbench.io/</url>
     <properties>
-        <jersey.version>3.1.4</jersey.version>
+        <jersey.version>3.1.5</jersey.version>
     </properties>
 
     <parent>
@@ -45,22 +45,22 @@
         <dependency>
             <groupId>org.eclipse.jetty</groupId>
             <artifactId>jetty-server</artifactId>
-            <version>11.0.18</version>
+            <version>11.0.19</version>
         </dependency>
         <dependency>
             <groupId>org.eclipse.jetty</groupId>
             <artifactId>jetty-servlets</artifactId>
-            <version>11.0.18</version>
+            <version>11.0.19</version>
         </dependency>
         <dependency>
             <groupId>org.eclipse.jetty</groupId>
             <artifactId>jetty-servlet</artifactId>
-            <version>11.0.18</version>
+            <version>11.0.19</version>
         </dependency>
         <dependency>
             <groupId>org.eclipse.jetty</groupId>
             <artifactId>jetty-rewrite</artifactId>
-            <version>11.0.18</version>
+            <version>11.0.19</version>
         </dependency>
 
         <dependency>
@@ -94,13 +94,13 @@
         <dependency>
             <groupId>org.glassfish.jersey.media</groupId>
             <artifactId>jersey-media-json-jackson</artifactId>
-            <version>3.1.3</version>
+            <version>3.1.5</version>
         </dependency>
 
         <dependency>
             <groupId>com.fasterxml.jackson.jaxrs</groupId>
             <artifactId>jackson-jaxrs-json-provider</artifactId>
-            <version>2.16.0</version>
+            <version>2.16.1</version>
         </dependency>
 
         <dependency>

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -45,7 +45,7 @@ import io.nosqlbench.engine.core.lifecycle.ExecutionResult;
 import io.nosqlbench.engine.core.lifecycle.process.NBCLIErrorHandler;
 import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader;
 import io.nosqlbench.engine.core.lifecycle.session.NBSession;
-import io.nosqlbench.engine.core.logging.LoggerConfig;
+import io.nosqlbench.engine.core.logging.NBLoggerConfig;
 import io.nosqlbench.engine.core.metadata.MarkdownFinder;
 import io.nosqlbench.nb.annotations.Service;
 import io.nosqlbench.nb.annotations.ServiceSelector;
@@ -68,13 +68,13 @@ import java.util.stream.Collectors;
 public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
 
     private static Logger logger;
-    private static final LoggerConfig loggerConfig;
+    private static final NBLoggerConfig loggerConfig;
     private static final int EXIT_OK = 0;
     private static final int EXIT_WARNING = 1;
     private static final int EXIT_ERROR = 2;
 
     static {
-        loggerConfig = new LoggerConfig();
+        loggerConfig = new NBLoggerConfig();
         ConfigurationFactory.setConfigurationFactory(NBCLI.loggerConfig);
     }
 
@@ -169,9 +169,9 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
             .setAnsiEnabled(globalOptions.isEnableAnsi())
             .setDedicatedVerificationLogger(globalOptions.isDedicatedVerificationLogger())
             .activate();
-        ConfigurationFactory.setConfigurationFactory(NBCLI.loggerConfig);
+        ConfigurationFactory.setConfigurationFactory(NBCLI.loggerConfig); // THIS should be the first time log4j2 is invoked!
 
-        NBCLI.logger = LogManager.getLogger("NBCLI");
+        NBCLI.logger = LogManager.getLogger("NBCLI"); // TODO: Detect if the logger config was already initialized (error)
         NBCLI.loggerConfig.purgeOldFiles(LogManager.getLogger("SCENARIO"));
         if (NBCLI.logger.isInfoEnabled())
             NBCLI.logger.info(() -> "Configured scenario log at " + NBCLI.loggerConfig.getLogfileLocation());
@@ -185,7 +185,7 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
         }
 
         NBCLI.logger.info(() -> "Running NoSQLBench Version " + new VersionInfo().getVersion());
-        NBCLI.logger.info(() -> "command-line: " + Arrays.stream(args).collect(Collectors.joining(" ")));
+        NBCLI.logger.info(() -> "command-line: " + String.join(" ", args));
         NBCLI.logger.info(() -> "client-hardware: " + SystemId.getHostSummary());
 
 
@@ -216,7 +216,7 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
             annotatorsConfig = gson.toJson(annotatorsConfigs);
         }
 
-        final NBCLIOptions options = new NBCLIOptions(args);
+        final NBCLIOptions options = new NBCLIOptions(args, Mode.ParseAllOptions);
         NBCLI.logger = LogManager.getLogger("NBCLI");
 
         NBIO.addGlobalIncludes(options.wantsIncludes());
@@ -401,53 +401,64 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
         // intentionally not shown for warn-only
         NBCLI.logger.info(() -> "console logging level is " + options.getConsoleLogLevel());
 
+        Map<String, String> props = Map.of(
+            "summary", options.getReportSummaryTo(),
+            "logsdir", options.getLogsDirectory().toString(),
+            "progress", options.getProgressSpec(),
+            "prompush_cache", "prompush_cache.txt",
+            "heartbeat", String.valueOf(options.wantsHeartbeatIntervalMs())
+        );
         /**
          * At this point, the command stream from the CLI should be handed into the session, and the session should
          * marshal and transform it for any scenario invocations directly.
          */
-        NBSession session = new NBSession(
-            new NBBaseComponent(null,
-                options.getLabelMap()
-                    .andDefault("jobname", "nosqlbench")
-                    .andDefault("instance", "default")
-            ),
-            sessionName
-        );
-        // TODO: Decide whether this should be part of ctor consistency
-        Map.of(
-            "summary", options.getReportSummaryTo(),
-            "logsdir", options.getLogsDirectory().toString(),
-            "progress", options.getProgressSpec()
-        ).forEach(session::setComponentProp);
 
-        options.wantsReportCsvTo().ifPresent(cfg -> {
-            MetricInstanceFilter filter = new MetricInstanceFilter();
-            filter.addPattern(cfg.pattern);
-            new CsvReporter(session, Path.of(cfg.file), cfg.millis, filter);
-        });
+        try (
+            NBSession session = new NBSession(
+                new NBBaseComponent(null,
+                    options.getLabelMap()
+                        .andDefault("jobname", "nosqlbench")
+                        .andDefault("instance", "default")
+                ),
+                sessionName,
+                props
+            )) {
 
-        options.wantsReportPromPushTo().ifPresent(cfg -> {
-            String[] words = cfg.split(",");
-            String uri;
-            long intervalMs = 10_000L;
+            options.wantsReportCsvTo().ifPresent(cfg -> {
+                MetricInstanceFilter filter = new MetricInstanceFilter();
+                filter.addPattern(cfg.pattern);
+                new CsvReporter(session, Path.of(cfg.file), cfg.millis, filter);
+            });
 
-            switch (words.length) {
-                case 2:
-                    intervalMs = Unit.msFor(words[1]).orElseThrow(() -> new RuntimeException("can't parse '" + words[1] + "!"));
-                case 1:
-                    uri = words[0];
-                    break;
-                default:
-                    throw new RuntimeException("Unable to parse '" + cfg + "', must be in <URI> or <URI>,ms form");
+            options.wantsReportPromPushTo().ifPresent(cfg -> {
+                String[] words = cfg.split(",");
+                String uri;
+                long intervalMs = 10_000L;
+
+                switch (words.length) {
+                    case 2:
+                        intervalMs = Unit.msFor(words[1]).orElseThrow(() -> new RuntimeException("can't parse '" + words[1] + "!"));
+                    case 1:
+                        uri = words[0];
+                        break;
+                    default:
+                        throw new RuntimeException("Unable to parse '" + cfg + "', must be in <URI> or <URI>,ms form");
+                }
+                session.create().pushReporter(uri, intervalMs, NBLabels.forKV());
+            });
+
+
+            ExecutionResult sessionResult = session.apply(options.getCommands());
+            logger.info(sessionResult);
+            if (sessionResult.getException() instanceof RuntimeException rte) {
+                throw rte;
+            } else if (sessionResult.getException() instanceof Throwable t) {
+                throw new RuntimeException(t);
             }
-            session.create().pushReporter(uri, intervalMs, NBLabels.forKV());
-        });
 
-
-        ExecutionResult sessionResult = session.apply(options.getCommands());
+            return sessionResult.getStatus().code;
+        }
 //        sessionResult.printSummary(System.out);
-        logger.info(sessionResult);
-        return sessionResult.getStatus().code;
 
     }
 
@@ -464,10 +475,8 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
         }
         basicHelp = basicHelp.replaceAll("PROG", this.commandName);
         return basicHelp;
-
     }
 
-
     @Override
     public NBLabels getLabels() {
         return labels;

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java

@@ -17,6 +17,7 @@
 package io.nosqlbench.engine.cli;
 
 import io.nosqlbench.engine.api.scenarios.NBCLIScenarioPreprocessor;
+import io.nosqlbench.engine.cli.atfiles.NBAtFile;
 import io.nosqlbench.engine.cmdstream.Cmd;
 import io.nosqlbench.engine.cmdstream.PathCanonicalizer;
 import io.nosqlbench.engine.core.lifecycle.session.CmdParser;
@@ -62,6 +63,8 @@ public class NBCLIOptions {
     private static final String LABELSPEC = "--labelspec";
     private static final String ANNOTATORS_CONFIG = "--annotators";
 
+    private static final String HEARTBEAT_MILLIS = "--heartbeat-millis";
+
     // Enabled if the TERM env var is provided
     private static final String ANSI = "--ansi";
 
@@ -118,7 +121,7 @@ public class NBCLIOptions {
     private static final String GRAPHITE_LOG_LEVEL = "--graphite-log-level";
     private static final String REPORT_CSV_TO = "--report-csv-to";
     private static final String REPORT_SUMMARY_TO = "--report-summary-to";
-    private static final String REPORT_SUMMARY_TO_DEFAULT = "stdout:60,_LOGS_/_SESSION_.summary";
+    private static final String REPORT_SUMMARY_TO_DEFAULT = "stdout:60,_LOGS_/_SESSION__summary.txt";
     private static final String PROGRESS = "--progress";
     private static final String WITH_LOGGING_PATTERN = "--with-logging-pattern";
     private static final String LOGGING_PATTERN = "--logging-pattern";
@@ -201,6 +204,7 @@ public class NBCLIOptions {
     private String annotateLabelSpec = "";
     private String metricsLabelSpec = "";
     private String wantsToCatResource ="";
+    private long heartbeatIntervalMs = 10000;
 
     public boolean wantsLoggedMetrics() {
         return this.wantsConsoleMetrics;
@@ -275,10 +279,6 @@ public class NBCLIOptions {
         ParseAllOptions
     }
 
-    public NBCLIOptions(final String[] args) {
-        this(args, Mode.ParseAllOptions);
-    }
-
     public NBCLIOptions(final String[] args, final Mode mode) {
         switch (mode) {
             case ParseGlobalsOnly:
@@ -293,6 +293,8 @@ public class NBCLIOptions {
     private LinkedList<String> parseGlobalOptions(final String[] args) {
 
         LinkedList<String> arglist = new LinkedList<>(Arrays.asList(args));
+        NBAtFile.includeAt(arglist);
+
         if (null == arglist.peekFirst()) {
             this.wantsBasicHelp = true;
             return arglist;
@@ -300,8 +302,8 @@ public class NBCLIOptions {
 
         // Process --include and --statedir, separately first
         // regardless of position
-        LinkedList<String> nonincludes = new LinkedList<>();
-        while (null != arglist.peekFirst()) {
+        LinkedList<String> everythingButIncludes = new LinkedList<>();
+        while (!arglist.isEmpty()) {
             final String word = arglist.peekFirst();
             if (word.startsWith("--") && word.contains("=")) {
                 final String wordToSplit = arglist.removeFirst();
@@ -322,13 +324,13 @@ public class NBCLIOptions {
                     this.wantsToIncludePaths.add(include);
                     break;
                 default:
-                    nonincludes.addLast(arglist.removeFirst());
+                    everythingButIncludes.addLast(arglist.removeFirst());
             }
         }
         this.statepath = NBStatePath.initialize(statedirs);
 
-        arglist = nonincludes;
-        nonincludes = new LinkedList<>();
+        arglist = everythingButIncludes;
+        everythingButIncludes = new LinkedList<>();
 
         // Now that statdirs is settled, auto load argsfile if it is present
         final NBCLIArgsFile argsfile = new NBCLIArgsFile();
@@ -501,11 +503,11 @@ public class NBCLIOptions {
                     this.metricsLabelSpec = this.readWordOrThrow(arglist, "labels validator specification for metric labels");
                     break;
                 default:
-                    nonincludes.addLast(arglist.removeFirst());
+                    everythingButIncludes.addLast(arglist.removeFirst());
             }
         }
 
-        return nonincludes;
+        return everythingButIncludes;
     }
 
     private void setLabels(String labeldata) {
@@ -639,6 +641,11 @@ public class NBCLIOptions {
                     arglist.removeFirst();
                     this.wantsToCatResource = this.readWordOrThrow(arglist, "workload to cat");
                     break;
+                case HEARTBEAT_MILLIS:
+                    arglist.removeFirst();
+                    this.heartbeatIntervalMs =
+                            Long.parseLong(this.readWordOrThrow(arglist, "heartbeat interval in ms"));
+                    break;
                 default:
                     nonincludes.addLast(arglist.removeFirst());
             }
@@ -666,7 +673,21 @@ public class NBCLIOptions {
                 """
                 .replaceAll("ARG", cmdParam)
                 .replaceAll("PROG", "nb5")
-                .replaceAll("INCLUDES", String.join(",", wantsIncludes()));
+                .replaceAll("INCLUDES", String.join(",", wantsIncludes()))
+                ;
+
+            final String debugMessage = """
+
+                After parsing all global options out of the command line, the remaining commands were found,
+                and mapped to valid options as describe above. This command stream was:
+                COMMANDSTREAM
+                """
+                .replaceAll("COMMANDSTREAM",
+                    String.join(" ",arglist));
+            if (consoleLevel.isGreaterOrEqualTo(NBLogLevel.INFO)) {
+                System.out.println(debugMessage);
+            }
+
             throw new BasicError(helpmsg);
 
         }
@@ -736,6 +757,10 @@ public class NBCLIOptions {
         return this.wantsActivityTypes;
     }
 
+    public long wantsHeartbeatIntervalMs() {
+        return this.heartbeatIntervalMs;
+    }
+
     public boolean wantsTopicalHelp() {
         return this.wantsActivityHelp;
     }

engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java

@@ -0,0 +1,206 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.cli.atfiles;
+
+import io.nosqlbench.nb.api.nbio.Content;
+import io.nosqlbench.nb.api.nbio.NBIO;
+import io.nosqlbench.nb.api.nbio.NBPathsAPI;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.snakeyaml.engine.v2.api.Load;
+import org.snakeyaml.engine.v2.api.LoadSettings;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * TODO: provide before and after atfile processing logs for diagnostics
+ * TODO:ERRORHANDLER Cannot invoke "Object.getClass()" because "scopeOfInclude" is null on file full of comments only
+ */
+public class NBAtFile {
+//    private final static Logger logger = LogManager.getLogger(NBAtFile.class);
+
+    /**
+     * This will take a command line in raw form, which may include some arguments
+     * in the <pre>{@code @filepath:datapath>format}</pre> format. For each of these,
+     * the contents are expanded from the specified file, interior data path, and in
+     * the format requested.
+     * <UL>
+     *     <LI>{@code >= } formats maps as key=value</LI>
+     *     <LI>{@code >: } formats maps as key:value</LI>
+     *     <LI>{@code >-- } asserts each value starts with global option syntax (--)</LI>
+     * </UL>
+     *
+     * @param processInPlace The linked list which is statefully modified. If you need
+     *                       an unmodified copy, then this is the responsibility of the caller.
+     * @return An updated list with all values expanded and injected
+     * @throws RuntimeException for any errors finding, traversing, parsing, or rendering values
+     */
+    public static LinkedList<String> includeAt(LinkedList<String> processInPlace) {
+//        logger.trace("argv stream before processing: " + String.join("|",processInPlace));
+        ListIterator<String> iter = processInPlace.listIterator();
+        while (iter.hasNext()) {
+            String spec = iter.next();
+            if (spec.startsWith("@")) {
+                iter.previous();
+                iter.remove();
+                LinkedList<String> spliceIn = includeAt(spec);
+                for (String s : spliceIn) {
+                    iter.add(s);
+                }
+            }
+        }
+//        logger.trace("argv stream after atfile processing: "+ String.join("|",processInPlace));
+
+        return processInPlace;
+    }
+    private final static Pattern includePattern =
+        Pattern.compile("@(?<filepath>[a-zA-Z_][a-zA-Z0-9_./]*)(:(?<datapath>[a-zA-Z_][a-zA-Z0-9_./]*))?(>(?<formatter>.+))?");
+
+    /**
+     * Format specifiers:
+     * <pre>{@code
+     * -- means each value must come from a list, and that each line should contain a global argument
+     *    Specifically, each line must start with a --
+     *
+     * =  means each entry should be a key-value pair, and that it will be formatted as key=value on insertion
+     *
+     * :  means each entry should be a key-value pair, and that it will be formatted as key:value on insertion*
+     * }</pre>
+     * @param spec The include-at specifier, in the form of @file[:datapath]
+     * @return The linked list of arguments which is to be spliced into the caller's command list
+     */
+    public static LinkedList<String> includeAt(String spec) {
+        Matcher matcher = includePattern.matcher(spec);
+        if (matcher.matches()) {
+            String filepathSpec = matcher.group("filepath");
+            String dataPathSpec = matcher.group("datapath");
+            String formatSpec = matcher.group("formatter");
+            String[] datapath = (dataPathSpec!=null && !dataPathSpec.isBlank()) ? dataPathSpec.split("(/|\\.)") : new String[] {};
+
+            String[] parts = filepathSpec.split("\\.",2);
+            if (parts.length==2 && !parts[1].toLowerCase().matches("yaml")) {
+                throw new RuntimeException("Only the yaml format and extension is supported for at-files." +
+                    " You specified " + parts[1]);
+            }
+
+            filepathSpec=(filepathSpec.endsWith(".yaml") ? filepathSpec : filepathSpec+".yaml");
+            Path atPath = Path.of(filepathSpec);
+
+            String argsdata = "";
+            try {
+                argsdata = Files.readString(atPath);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+
+            NBAtFileFormats fmt = (formatSpec!=null) ? NBAtFileFormats.valueOfSymbol(formatSpec) : NBAtFileFormats.Default;
+
+            Object scopeOfInclude = null;
+            try {
+                Load yaml = new Load(LoadSettings.builder().build());
+                scopeOfInclude= yaml.loadFromString(argsdata);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+
+            if (datapath.length>0) {
+                if (scopeOfInclude instanceof Map<?,?> mapdata) {
+                    scopeOfInclude = traverseData(filepathSpec,(Map<String,Object>) mapdata,datapath);
+                } else {
+                    throw new RuntimeException("You can not traverse a non-map object type with spec '" + spec + "'");
+                }
+            }
+            LinkedList<String> formatted = formatted(scopeOfInclude, fmt);
+            formatted=interposePath(formatted, atPath);
+            return formatted;
+        } else {
+            throw new RuntimeException("Unable to match at-file specifier: " + spec + " to pattern '" + includePattern.pattern() + "'");
+        }
+
+    }
+
+    private static LinkedList<String> interposePath(LinkedList<String> formatted, Path atPath) {
+        Path parent = (atPath.getNameCount()==1 ? Path.of(".") : atPath.getParent());
+
+        ListIterator<String> iter = formatted.listIterator();
+        while (iter.hasNext()) {
+            String word = iter.next();
+            String modified = word.replaceAll("\\$\\{DIR}",parent.toString());
+            iter.remove();
+            iter.add(modified);
+        }
+        return formatted;
+    }
+
+    private static LinkedList<String> formatted(Object scopeOfInclude, NBAtFileFormats fmt) {
+        LinkedList<String> emitted = new LinkedList<>();
+        if (scopeOfInclude instanceof Map<?,?> map) {
+            Map<String,String> included = new LinkedHashMap<>();
+            map.forEach((k,v) -> {
+                included.put(k.toString(),v.toString());
+            });
+            included.forEach((k,v) -> {
+                fmt.validate(new String[]{k,v});
+                String formatted = fmt.format(new String[]{k,v});
+                emitted.add(formatted);
+            });
+        } else if (scopeOfInclude instanceof List<?> list) {
+            List<String> included = new LinkedList<>();
+            list.forEach(item -> included.add(item.toString()));
+            included.forEach(item -> {
+                fmt.validate(new String[]{item});
+                String formatted = fmt.format(new String[]{item});
+                emitted.add(formatted);
+            });
+        } else {
+            throw new RuntimeException(scopeOfInclude.getClass().getCanonicalName() + " is not a valid data structure at-file inclusion.");
+        }
+        return emitted;
+    }
+
+    private static Object traverseData(String sourceName, Map<String, Object> map, String[] datapath) {
+        String leaf = datapath[datapath.length-1];
+        String[] traverse = Arrays.copyOfRange(datapath,0,datapath.length-1);
+
+        for (String name : traverse) {
+            if (map.containsKey(name)) {
+                Object nextMap = map.get(name);
+                if (nextMap instanceof Map<?, ?> nextmap) {
+                    map = (Map<String, Object>) nextmap;
+                }
+            } else {
+                throw new RuntimeException(
+                    "Unable to traverse to '" + name + "' node " +
+                        " in path '" + String.join("/",Arrays.asList(datapath) +
+                        " in included data from source '" + sourceName + "'")
+                );
+            }
+        }
+
+        if (map.containsKey(leaf)) {
+            return (map.get(leaf));
+        } else {
+            throw new RuntimeException("Unable to find data path '" + String.join("/",Arrays.asList(datapath)) + " in included data from source '" + sourceName + "'");
+        }
+
+    }
+}

engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFileFormats.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.cli.atfiles;
+
+import java.util.Arrays;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public enum NBAtFileFormats {
+    Default("", s -> s.length <= 2, NBAtFileFormats::formatDefaultDashOption),
+    MapWithEquals("=", s -> s.length <= 2, NBAtFileFormats::formatNameEqualsValue),
+    MapWithColons(":", s -> s.length <= 2, NBAtFileFormats::formatNameColonValue),
+    GlobalWithDoubleDashes("--", s -> s.length <= 2 && s[0].startsWith("--"), NBAtFileFormats::formatDashDashOption);
+
+    private static String formatNameEqualsValue(String[] strings) {
+        if (strings.length == 2) {
+            return strings[0] + "=" + strings[1];
+        } else if (strings.length == 1 && strings[0].matches("[a-zA-Z_][a-zA-Z_]*[=:].*")) {
+            String[] parts = strings[0].split("[=:]", 2);
+            return parts[0]+"="+parts[1];
+        } else {
+            throw new RuntimeException("Unable to match data for namedd value form: " + String.join("|",Arrays.asList(strings)));
+        }
+    }
+
+    private static String formatNameColonValue(String[] strings) {
+        if (strings.length == 2) {
+            return strings[0] + ":" + strings[1];
+        } else if (strings.length == 1 && strings[0].matches("[a-zA-Z_][a-zA-Z_]*[=:].*")) {
+            String[] parts = strings[0].split("[=:]", 2);
+            return parts[0]+":"+parts[1];
+        } else {
+            throw new RuntimeException("Unable to match data for namedd value form: " + String.join("|",Arrays.asList(strings)));
+        }
+    }
+
+    private static String formatDefaultDashOption(String[] strings) {
+        if (strings.length == 1 && strings[0].startsWith("--")) {
+            return formatDashDashOption(strings);
+        } else if (strings.length == 1 && strings[0].matches("[a-zA-Z_][a-zA-Z0-9._]*[=:].*")) {
+            return formatNameEqualsValue(strings);
+        } else if (strings.length == 2) {
+            return formatNameEqualsValue(strings);
+        } else if (strings.length==1) {
+            return strings[0];
+        } else {
+            throw new RuntimeException("Unable to match data for --global option form: " + String.join("|",Arrays.asList(strings)));
+        }
+    }
+
+    private final String spec;
+    private final Predicate<String[]> validator;
+    private final Function<String[], String> formatter;
+
+    NBAtFileFormats(String spec, Predicate<String[]> validator, Function<String[], String> formatter) {
+        this.spec = spec;
+        this.validator = validator;
+        this.formatter = formatter;
+    }
+
+    public static NBAtFileFormats valueOfSymbol(String s) {
+        for (NBAtFileFormats value : values()) {
+            if (value.spec.equals(s)) {
+                return value;
+            }
+        }
+        throw new RuntimeException("Format for spec '" + s + "' not found.");
+    }
+
+    public void validate(String[] ary) {
+        if (!validator.test(ary)) {
+            throw new RuntimeException("With fmt '" + this.name() + "': input data not valid for format specifier '" + spec + "': data:[" + String.join("],[", Arrays.asList(ary)) + "]");
+        }
+    }
+
+    private final static Pattern doubleDashOption = Pattern.compile(
+        "^(?<optname>--[a-zA-Z][a-zA-Z0-9_.-]*)((=|\\s+)(?<optvalue>.+))?$"
+    );
+
+    private static String formatDashDashOption(String[] words) {
+
+        if (words.length > 1) {
+            throw new RuntimeException("too many values for rendering --option in at-file: " + Arrays.asList(words));
+        }
+        Matcher matcher = doubleDashOption.matcher(words[0]);
+        if (matcher.matches()) {
+            String optname = matcher.group("optname");
+            String optvalue = matcher.group("optvalue");
+            if (optvalue!=null) {
+                optvalue = (optvalue.matches("'.+'") ? optvalue.substring(1, optvalue.length() - 1) : optvalue);
+                optvalue = (optvalue.matches("\".+\"") ? optvalue.substring(1, optvalue.length() - 1) : optvalue);
+                return optname + "=" + optvalue;
+            } else {
+                return optname;
+            }
+        } else {
+            throw new RuntimeException("Unable to match option '" + words[0] + "' with pattern " + doubleDashOption.pattern());
+        }
+    }
+
+    public String format(String[] ary) {
+        return formatter.apply(ary);
+    }
+}

engine-cli/src/test/java/io/nosqlbench/engine/cli/BasicScriptBufferTest.java

@@ -28,7 +28,7 @@ public class BasicScriptBufferTest {
 
     @Test
     public void testScriptInterpolation() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "path=script_to_interpolate", "parameter1=replaced"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "path=script_to_interpolate", "parameter1=replaced"}, NBCLIOptions.Mode.ParseAllOptions);
 
         BasicScriptBuffer b = new BasicScriptBuffer();
         b.add(opts.getCommands().toArray(new Cmd[0]));
@@ -40,7 +40,7 @@ public class BasicScriptBufferTest {
 
     @Test
     public void testAutoScriptCommand() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "script","path=acommand" });
+        NBCLIOptions opts = new NBCLIOptions(new String[]{ "script","path=acommand" }, NBCLIOptions.Mode.ParseAllOptions);
         BasicScriptBuffer b = new BasicScriptBuffer();
         b.add(opts.getCommands().toArray(new Cmd[0]));
         String s = b.getParsedScript();
@@ -53,8 +53,8 @@ public class BasicScriptBufferTest {
         NBCLIOptions opts = new NBCLIOptions(new String[] {
             "script",
             "path=testscripts/printscript.js",
-            "param1=value1"
-        });
+            "param1=value1",
+        }, NBCLIOptions.Mode.ParseAllOptions);
         BasicScriptBuffer b = new BasicScriptBuffer();
         b.add(opts.getCommands().toArray(new Cmd[0]));
         String script = b.getParsedScript();
@@ -72,7 +72,7 @@ public class BasicScriptBufferTest {
             "path=testscripts/printparam.js",
             "paramname=another",
             "param2=andanother"
-        });
+        }, NBCLIOptions.Mode.ParseAllOptions);
         BasicScriptBuffer b = new BasicScriptBuffer();
         b.add(opts.getCommands().toArray(new Cmd[0]));
         String script = b.getParsedScript();
@@ -84,6 +84,6 @@ public class BasicScriptBufferTest {
     @Test
     public void shouldThrowErrorForInvalidWaitMillisOperand() {
         assertThatExceptionOfType(NumberFormatException.class)
-                .isThrownBy(() -> new NBCLIOptions(new String[]{ "waitmillis", "ms=noway" }));
+                .isThrownBy(() -> new NBCLIOptions(new String[]{ "waitmillis", "ms=noway" }, NBCLIOptions.Mode.ParseAllOptions));
     }
 }

engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTemplateVarTest.java

@@ -30,7 +30,7 @@ public class NBCLIScenarioPreprocessorTemplateVarTest {
 
     @Test
     public void testMultipleOccurencesOfSameTemplateVar() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example_scenarios_templatevars" });
+        NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example_scenarios_templatevars" }, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         cmds.forEach(System.out::println);
 
@@ -47,7 +47,7 @@ public class NBCLIScenarioPreprocessorTemplateVarTest {
 
     @Test
     public void testThatCLIOverridesWorkForTemplateVars() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example_scenarios_templatevars", "tvar1=overridden" });
+        NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example_scenarios_templatevars", "tvar1=overridden" }, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         cmds.forEach(System.out::println);
 
@@ -59,7 +59,7 @@ public class NBCLIScenarioPreprocessorTemplateVarTest {
 
     @Test
     public void testThatAdditionalCLIParamIsAdded() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"local/example_scenarios_templatevars", "tvar3=tval3"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"local/example_scenarios_templatevars", "tvar3=tval3"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         cmds.forEach(System.out::println);
         assertThat(cmds).hasSize(2);

engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTest.java

@@ -33,39 +33,39 @@ public class NBCLIScenarioPreprocessorTest {
 
     @Test
     public void providePathForScenario() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"local/example_scenarios"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"local/example_scenarios"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
     }
 
     @Test
     public void defaultScenario() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
     }
 
     @Test
     public void defaultScenarioWithParams() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "cycles=100"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "cycles=100"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.get(0).getArgValue("cycles")).isEqualTo("100");
     }
 
     @Test
     public void namedScenario() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "schema_only"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "schema_only"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
     }
 
     @Test
     public void namedScenarioWithParams() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "schema_only", "cycles=100"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "schema_only", "cycles=100"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.get(0).getArgValue("cycles")).containsOnlyOnce("100");
     }
 
     @Test
     public void testThatSilentFinalParametersPersist() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "type=foo"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "type=foo"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.get(0).getArgValue("driver")).isEqualTo("stdout");
     }
@@ -73,25 +73,25 @@ public class NBCLIScenarioPreprocessorTest {
     @Test
     public void testThatVerboseFinalParameterThrowsError() {
         assertThatExceptionOfType(BasicError.class)
-            .isThrownBy(() -> new NBCLIOptions(new String[]{"scenario_test", "workload=canttouchthis"}));
+            .isThrownBy(() -> new NBCLIOptions(new String[]{"scenario_test", "workload=canttouchthis"}, NBCLIOptions.Mode.ParseAllOptions));
     }
 
     @Test
     public void testThatMissingScenarioNameThrowsError() {
         assertThatExceptionOfType(BasicError.class)
-            .isThrownBy(() -> new NBCLIOptions(new String[]{"scenario_test", "missing_scenario"}));
+            .isThrownBy(() -> new NBCLIOptions(new String[]{"scenario_test", "missing_scenario"}, NBCLIOptions.Mode.ParseAllOptions));
     }
 
     @Test
     public void testThatMultipleScenariosConcatenate() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "default", "default"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "default", "default"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.size()).isEqualTo(6);
     }
 
     @Test
     public void testThatTemplatesAreExpandedDefault() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "template_test"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "template_test"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.size()).isEqualTo(1);
         assertThat(cmds.get(0).getArgValue("driver")).isEqualTo("stdout");
@@ -101,10 +101,11 @@ public class NBCLIScenarioPreprocessorTest {
 
     @Test
     public void testThatTemplateParamsAreExpandedAndNotRemovedOverride() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "template_test", "cycles-test=20"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "template_test", "cycles-test=20"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.size()).isEqualTo(1);
         assertThat(cmds.get(0).getArgMap()).isEqualTo(Map.of(
+            "_impl","run",
             "alias", "with_template",
             "container", "template_test",
             "cycles", "20",
@@ -118,10 +119,11 @@ public class NBCLIScenarioPreprocessorTest {
 
     @Test
     public void testThatUndefValuesAreUndefined() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "schema_only", "cycles-test=20"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "schema_only", "cycles-test=20"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.size()).isEqualTo(1);
         assertThat(cmds.get(0).getArgMap()).isEqualTo(Map.of(
+            "_impl", "run",
             "alias", "schema",
             "container", "schema_only",
             "cycles-test", "20",
@@ -131,7 +133,7 @@ public class NBCLIScenarioPreprocessorTest {
             "tags", "block:\"schema.*\"",
             "workload", "scenario_test"
         ));
-        NBCLIOptions opts1 = new NBCLIOptions(new String[]{"scenario_test", "schema_only", "doundef=20"});
+        NBCLIOptions opts1 = new NBCLIOptions(new String[]{"scenario_test", "schema_only", "doundef=20"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds1 = opts1.getCommands();
         assertThat(cmds1.size()).isEqualTo(1);
         assertThat(cmds1.get(0).getArgValueOrNull("cycles-test")).isNull();
@@ -148,7 +150,7 @@ public class NBCLIScenarioPreprocessorTest {
         Path absolute = rel.toAbsolutePath();
         assertThat(absolute).exists();
 
-        NBCLIOptions opts = new NBCLIOptions(new String[]{absolute.toString(), "schema_only", "cycles-test=20"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{absolute.toString(), "schema_only", "cycles-test=20"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.size()).isGreaterThan(0);
     }
@@ -159,7 +161,7 @@ public class NBCLIScenarioPreprocessorTest {
         //TODO: This might change?
         String urlScenario = "https://raw.githubusercontent.com/nosqlbench/nosqlbench/main/engine-cli/src/test/resources/activities/scenario_test.yaml";
 
-        NBCLIOptions opts = new NBCLIOptions(new String[]{urlScenario, "schema_only", "cycles-test=20"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{urlScenario, "schema_only", "cycles-test=20"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.size()).isGreaterThan(0);
     }
@@ -172,10 +174,11 @@ public class NBCLIScenarioPreprocessorTest {
 
     @Test
     public void testSubStepSelection() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "schema_only", "cycles-test=20"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"scenario_test", "schema_only", "cycles-test=20"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.size()).isEqualTo(1);
         assertThat(cmds.get(0).getArgMap()).isEqualTo(Map.of(
+            "_impl","run",
             "alias", "schema",
             "container", "schema_only",
             "cycles-test", "20",
@@ -185,7 +188,7 @@ public class NBCLIScenarioPreprocessorTest {
             "tags", "block:\"schema.*\"",
             "workload", "scenario_test"
         ));
-        NBCLIOptions opts1 = new NBCLIOptions(new String[]{"local/example_scenarios", "namedsteps.one", "testparam1=testvalue2"});
+        NBCLIOptions opts1 = new NBCLIOptions(new String[]{"local/example_scenarios", "namedsteps.one", "testparam1=testvalue2"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds1 = opts1.getCommands();
         assertThat(cmds1.size()).isEqualTo(1);
         assertThat(cmds1.get(0).getArgValueOrNull("cycles_test")).isNull();

engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java

@@ -36,7 +36,7 @@ public class TestNBCLIOptions {
 
     @Test
     public void shouldRecognizeActivities() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"start", "foo=wan", "start", "bar=lan"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"start", "foo=wan", "start", "bar=lan"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.getCommands()).isNotNull();
         assertThat(opts.getCommands().size()).isEqualTo(2);
 //        assertThat(opts.getCommands().get(0).getArgs()).containsEntry("foo","wan");
@@ -46,7 +46,7 @@ public class TestNBCLIOptions {
     @Test
     public void shouldParseLongActivityForm() {
         NBCLIOptions opts = new NBCLIOptions(new String[]{"start", "param1=param2", "param3=param4",
-                                                          "--report-graphite-to", "woot", "--report-interval", "23"});
+                                                          "--report-graphite-to", "woot", "--report-interval", "23"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.getCommands().size()).isEqualTo(1);
 //        assertThat(opts.getCommands().get(0).getArgs()).containsEntry("param1","param2");
 //        assertThat(opts.getCommands().get(0).getArgs()).containsEntry("param3","param4");
@@ -56,21 +56,21 @@ public class TestNBCLIOptions {
 
     @Test
     public void shouldRecognizeShortVersion() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"--version"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"--version"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.isWantsVersionShort()).isTrue();
         assertThat(opts.wantsVersionCoords()).isFalse();
     }
 
     @Test
     public void shouldRecognizeVersion() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"--version-coords"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"--version-coords"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.isWantsVersionShort()).isFalse();
         assertThat(opts.wantsVersionCoords()).isTrue();
     }
 
     @Test
     public void shouldRecognizeScripts() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "path=ascriptaone", "script", "path=ascriptatwo"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "path=ascriptaone", "script", "path=ascriptatwo"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.getCommands()).isNotNull();
         assertThat(opts.getCommands().size()).isEqualTo(2);
         assertThat(opts.getCommands().get(0).getCmdType()).isEqualTo(CmdType.script);
@@ -81,41 +81,41 @@ public class TestNBCLIOptions {
 
     @Test
     public void shouldRecognizeWantsActivityTypes() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"--list-activity-types"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"--list-activity-types"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.wantsActivityTypes()).isTrue();
-        opts = new NBCLIOptions(new String[]{"--version"});
+        opts = new NBCLIOptions(new String[]{"--version"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.wantsActivityTypes()).isFalse();
-        opts = new NBCLIOptions(new String[]{"--list-drivers"});
+        opts = new NBCLIOptions(new String[]{"--list-drivers"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.wantsActivityTypes()).isTrue();
 
     }
 
     @Test
     public void shouldRecognizeWantsBasicHelp() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"--help"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"--help"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.wantsBasicHelp()).isTrue();
-        opts = new NBCLIOptions(new String[]{"--version"});
+        opts = new NBCLIOptions(new String[]{"--version"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.wantsTopicalHelp()).isFalse();
     }
 
     @Test
     public void shouldRecognizeWantsActivityHelp() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"--help", "foo"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"--help", "foo"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.wantsTopicalHelp()).isTrue();
         assertThat(opts.wantsTopicalHelpFor()).isEqualTo("foo");
-        opts = new NBCLIOptions(new String[]{"--version"});
+        opts = new NBCLIOptions(new String[]{"--version"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.wantsTopicalHelp()).isFalse();
     }
 
     @Test
     public void shouldErrorSanelyWhenNoMatch() {
         assertThatExceptionOfType(IllegalArgumentException.class)
-                .isThrownBy(() -> new NBCLIOptions(new String[]{"unrecognizable command"}));
+                .isThrownBy(() -> new NBCLIOptions(new String[]{"unrecognizable command"}, NBCLIOptions.Mode.ParseAllOptions));
     }
 
     @Test
     public void testShouldRecognizeScriptParams() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "path=ascript", "param1=value1"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"script", "path=ascript", "param1=value1"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.getCommands().size()).isEqualTo(1);
         Cmd cmd = opts.getCommands().get(0);
         assertThat(cmd.getArgs().size()).isEqualTo(2);
@@ -127,33 +127,33 @@ public class TestNBCLIOptions {
     @Test
     public void testShouldErrorSanelyWhenScriptNameSkipped() {
         assertThatExceptionOfType(InvalidParameterException.class)
-                .isThrownBy(() -> new NBCLIOptions(new String[]{"script", "param1=value1"}));
+                .isThrownBy(() -> new NBCLIOptions(new String[]{"script", "param1=value1"}, NBCLIOptions.Mode.ParseAllOptions));
     }
 
     @Disabled("semantic parsing is deferred until later")
     @Test
     public void testShouldErrorForMissingScriptName() {
         assertThatExceptionOfType(InvalidParameterException.class)
-                .isThrownBy(() -> new NBCLIOptions(new String[]{"script"}));
+                .isThrownBy(() -> new NBCLIOptions(new String[]{"script"}, NBCLIOptions.Mode.ParseAllOptions));
     }
 
-    @Test
-    public void shouldRecognizeStartActivityCmd() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "start", "driver=woot" });
-        List<Cmd> cmds = opts.getCommands();
-        assertThat(cmds).hasSize(1);
-        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.start);
-
-    }
-
-    @Test
-    public void shouldRecognizeRunActivityCmd() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "run", "driver=runwoot" });
-        List<Cmd> cmds = opts.getCommands();
-        assertThat(cmds).hasSize(1);
-        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.run);
-
-    }
+//    @Test
+//    public void shouldRecognizeStartActivityCmd() {
+//        NBCLIOptions opts = new NBCLIOptions(new String[]{ "start", "driver=woot" });
+//        List<Cmd> cmds = opts.getCommands();
+//        assertThat(cmds).hasSize(1);
+//        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.start);
+//
+//    }
+//
+//    @Test
+//    public void shouldRecognizeRunActivityCmd() {
+//        NBCLIOptions opts = new NBCLIOptions(new String[]{ "run", "driver=runwoot" });
+//        List<Cmd> cmds = opts.getCommands();
+//        assertThat(cmds).hasSize(1);
+//        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.run);
+//
+//    }
 
 //    @Test
 //    public void shouldRecognizeStopActivityCmd() {
@@ -169,49 +169,49 @@ public class TestNBCLIOptions {
     @Test
     public void shouldThrowErrorForInvalidStopActivity() {
         assertThatExceptionOfType(InvalidParameterException.class)
-                .isThrownBy(() -> new NBCLIOptions(new String[]{ "stop", "woah=woah" }));
+                .isThrownBy(() -> new NBCLIOptions(new String[]{ "stop", "woah=woah" }, NBCLIOptions.Mode.ParseAllOptions));
     }
 
-    @Test
-    public void shouldRecognizeAwaitActivityCmd() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "await", "activity=awaitme" });
-        List<Cmd> cmds = opts.getCommands();
-        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.await);
-        assertThat(cmds.get(0).getArgValue("activity")).isEqualTo("awaitme");
-
-    }
+//    @Test
+//    public void shouldRecognizeAwaitActivityCmd() {
+//        NBCLIOptions opts = new NBCLIOptions(new String[]{ "await", "activity=awaitme" });
+//        List<Cmd> cmds = opts.getCommands();
+//        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.await);
+//        assertThat(cmds.get(0).getArgValue("activity")).isEqualTo("awaitme");
+//
+//    }
 
     @Disabled("semantic parsing is reserved until later after generalizing syntax")
     @Test
     public void shouldThrowErrorForInvalidAwaitActivity() {
         assertThatExceptionOfType(InvalidParameterException.class)
-                .isThrownBy(() -> new NBCLIOptions(new String[]{ "await", "awaitme=notvalid" }));
+                .isThrownBy(() -> new NBCLIOptions(new String[]{ "await", "awaitme=notvalid" }, NBCLIOptions.Mode.ParseAllOptions));
     }
 
-    @Test
-    public void shouldRecognizewaitMillisCmd() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "waitmillis", "ms=23234" });
-        List<Cmd> cmds = opts.getCommands();
-        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.waitMillis);
-        assertThat(cmds.get(0).getArgValue("ms")).isEqualTo("23234");
-
-    }
+//    @Test
+//    public void shouldRecognizewaitMillisCmd() {
+//        NBCLIOptions opts = new NBCLIOptions(new String[]{ "waitmillis", "ms=23234" });
+//        List<Cmd> cmds = opts.getCommands();
+//        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.waitMillis);
+//        assertThat(cmds.get(0).getArgValue("ms")).isEqualTo("23234");
+//
+//    }
 
     @Test
     public void listWorkloads() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "--list-workloads"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{ "--list-workloads"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.wantsWorkloadsList()).isTrue();
     }
 
     @Test
     public void listScenarios() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "--list-scenarios"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{ "--list-scenarios"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.wantsScenariosList()).isTrue();
     }
 
     @Test
     public void listScripts() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "--list-scripts"});
+        NBCLIOptions opts = new NBCLIOptions(new String[]{ "--list-scripts"}, NBCLIOptions.Mode.ParseAllOptions);
         assertThat(opts.wantsListScripts()).isTrue();
     }
 

engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.cli.atfiles;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.LinkedList;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.*;
+
+class NBAtFileTest {
+
+    @Test
+    public void testParseSimpleListDefaultFmt() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/simple_list.yaml");
+        assertThat(strings).containsExactly("arg1","arg2","arg3");
+    }
+
+    @Test
+    public void testRelativizedPaths() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/relativized.yaml");
+        assertThat(strings).containsExactly("--option1=src/test/resources/atfiles/value1");
+    }
+
+    @Test
+    public void testParseSimpleMapDefaultFmt() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/simple_map.yaml");
+        assertThat(strings).containsExactly("arg1=val1","arg2=val2","arg3=val3");
+    }
+
+    @Test
+    public void testThatEmptyPathWithPathSpecifierIsInvalid() {
+        assertThrows(RuntimeException.class, () -> NBAtFile.includeAt("@src/test/resources/atfiles/simple_map.yaml:>:"));
+    }
+
+    @Test
+    public void testParseSimpleMapWithFormatter() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/simple_map.yaml>:");
+        assertThat(strings).containsExactly("arg1:val1","arg2:val2","arg3:val3");
+    }
+
+
+    @Test
+    public void testParseSimpleMapSlashesOrDots() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/mixed_structures.yaml:amap/ofamap.ofalist");
+        assertThat(strings).containsExactly("option1","option2");
+    }
+
+    @Test
+    public void testMapPathWithColonFormat() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/mixed_structures.yaml:amap/ofamap.ofentries>:");
+        assertThat(strings).containsExactly("key1:value1","key2:value2");
+    }
+
+    @Test
+    public void testMapPathWithEqualsFormat() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/mixed_structures.yaml:amap/ofamap.ofentries>=");
+        assertThat(strings).containsExactly("key1=value1","key2=value2");
+    }
+
+    @Test
+    public void testGlobalOptionForms() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/global_opts.yaml>--");
+        assertThat(strings).containsExactly("--option1", "--option2=value2", "--option3=value3", "--option4=value4");
+    }
+
+}

engine-cli/src/test/resources/atfiles/global_opts.yaml

@@ -0,0 +1,4 @@
+- --option1
+- --option2=value2
+- --option3='value3'
+- --option4="value4"

engine-cli/src/test/resources/atfiles/mixed_structures.yaml

@@ -0,0 +1,17 @@
+# This not valid for traversas, as only maps are supported till the leaf node
+alist:
+  - ofmaps:
+      oflist:
+        - arg1
+        - arg2
+
+# This is valid, and both types are valid for reference at the leaf level
+amap:
+ ofamap:
+   ofentries:
+     key1: value1
+     key2: value2
+   ofalist:
+     - option1
+     - option2
+

engine-cli/src/test/resources/atfiles/relativized.yaml

@@ -0,0 +1 @@
+- --option1=${DIR}/value1

engine-cli/src/test/resources/atfiles/simple_list.yaml

@@ -0,0 +1,3 @@
+- arg1
+- arg2
+- arg3

engine-cli/src/test/resources/atfiles/simple_map.yaml

@@ -0,0 +1,3 @@
+arg1: val1
+arg2: val2
+arg3: val3

engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/core/Activity.java

@@ -25,7 +25,7 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.filters.IntPredicateDispens
 import io.nosqlbench.engine.api.activityapi.errorhandling.ErrorMetrics;
 import io.nosqlbench.engine.api.activityapi.input.InputDispenser;
 import io.nosqlbench.engine.api.activityapi.output.OutputDispenser;
-import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
+import io.nosqlbench.engine.api.activityapi.simrate.RateLimiter;
 import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
 import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
 

engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/core/ActivityInstrumentation.java

@@ -21,6 +21,7 @@ import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
 import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
 import io.nosqlbench.adapters.api.activityimpl.OpDispenser;
+import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge;
 
 import java.util.concurrent.Future;
 
@@ -33,6 +34,16 @@ import java.util.concurrent.Future;
  */
 public interface ActivityInstrumentation {
 
+    NBMetricGauge getOrCreateErrorsTotal();
+
+    NBMetricGauge getOrCreateErrorRate1m();
+
+    NBMetricGauge getOrCreateErrorRate5m();
+
+    NBMetricGauge getOrCreateErrorRate15m();
+
+    NBMetricGauge getOrCreateErrorRateTotal();
+
     /**
      * The input timer measures how long it takes to get the cycle value to be used for
      * an operation.
@@ -77,8 +88,6 @@ public interface ActivityInstrumentation {
      */
     Counter getOrCreatePendingOpCounter();
 
-    Counter getOrCreateOpTrackerBlockedCounter();
-
     /**
      * The bind timer keeps track of how long it takes for NoSQLBench to create an instance
      * of an executable operation, given the cycle. This is usually done by using an

engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/core/ComponentActivityInstrumentation.java

@@ -21,9 +21,7 @@ import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
 import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
 import io.nosqlbench.nb.api.engine.activityimpl.ParameterMap;
-import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricCounter;
-import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricHistogram;
-import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricTimer;
+import io.nosqlbench.nb.api.engine.metrics.instruments.*;
 
 public class ComponentActivityInstrumentation implements ActivityInstrumentation {
 
@@ -41,7 +39,6 @@ public class ComponentActivityInstrumentation implements ActivityInstrumentation
     private NBMetricTimer cyclesServiceTimer;
     private NBMetricTimer cyclesResponseTimer;
     private NBMetricCounter pendingOpsCounter;
-    private NBMetricCounter opTrackerBlockedCounter;
     private NBMetricTimer bindTimer;
     private NBMetricTimer executeTimer;
     private NBMetricTimer resultTimer;
@@ -49,6 +46,12 @@ public class ComponentActivityInstrumentation implements ActivityInstrumentation
     private NBMetricHistogram triesHistogram;
     private NBMetricTimer verifierTimer;
 
+    private NBMetricGauge errorRate1m;
+    private NBMetricGauge errorRate5m;
+    private NBMetricGauge errorRate15m;
+    private NBMetricGauge errorRateTotal;
+    private NBMetricGauge errorsTotal;
+
     public ComponentActivityInstrumentation(final Activity activity) {
         this.activity = activity;
         def = activity.getActivityDef();
@@ -58,95 +61,225 @@ public class ComponentActivityInstrumentation implements ActivityInstrumentation
     }
 
     private void initMetrics() {
-        readInputTimer=activity.create().timer("read_input",this.hdrdigits);
-        stridesServiceTimer=activity.create().timer("strides",this.hdrdigits);
+        readInputTimer = activity.create().timer(
+            "read_input",
+            this.hdrdigits,
+            MetricCategory.Internals,
+            "measures overhead of acquiring a cycle range for an activity thread"
+        );
+        stridesServiceTimer = activity.create().timer(
+            "strides",
+            this.hdrdigits,
+            MetricCategory.Core,
+            "service timer for a stride, which is the same as the op sequence length by default"
+        );
         if (null != activity.getStrideLimiter()) {
             this.stridesResponseTimer = activity.create().timer(
                 "strides" + ComponentActivityInstrumentation.RESPONSE_TIME,
-                hdrdigits
+                hdrdigits,
+                MetricCategory.Core,
+                "response timer for a stride, which is the same as the op sequence length by default;" +
+                    " response timers include scheduling delays which occur when an activity falls behind its target rate"
             );
         }
         this.cyclesServiceTimer = activity.create().timer(
-            "cycles"+ComponentActivityInstrumentation.SERVICE_TIME,
-            hdrdigits
+            "cycles" + ComponentActivityInstrumentation.SERVICE_TIME,
+            hdrdigits,
+            MetricCategory.Core,
+            "service timer for a cycle, including all of bind, execute, result and result_success;" +
+                " service timers measure the time between submitting a request and receiving the response"
         );
         if (null != activity.getCycleLimiter()) {
             this.cyclesResponseTimer = activity.create().timer(
                 "cycles" + ComponentActivityInstrumentation.RESPONSE_TIME,
-                hdrdigits
+                hdrdigits,
+                MetricCategory.Core,
+                "response timer for a cycle, including all of bind, execute, result and result_success;" +
+                    " response timers include scheduling delays which occur when an activity falls behind its target rate"
             );
         }
-        this.pendingOpsCounter=activity.create().counter("pending_ops");
-        this.opTrackerBlockedCounter=activity.create().counter("optracker_blocked");
+        this.pendingOpsCounter = activity.create().counter(
+            "pending_ops",
+            MetricCategory.Core,
+            "Indicate the number of operations which have been started, but which have not been completed." +
+                " This starts "
+        );
 
-        this.bindTimer = activity.create().timer("bind",hdrdigits);
-        this.executeTimer = activity.create().timer("execute",hdrdigits);
-        this.resultTimer = activity.create().timer("result",hdrdigits);
-        this.resultSuccessTimer = activity.create().timer("result_success",hdrdigits);
-        this.triesHistogram = activity.create().histogram("tries",hdrdigits);
-        this.verifierTimer = activity.create().timer("verifier",hdrdigits);
+        this.bindTimer = activity.create().timer(
+            "bind",
+            hdrdigits,
+            MetricCategory.Core,
+            "Time the step within a cycle which binds generated data to an op template to synthesize an executable operation."
+        );
+
+        this.executeTimer = activity.create().timer(
+            "execute",
+            hdrdigits,
+            MetricCategory.Core,
+            "Time how long it takes to submit a request and receive a result, including reading the result in the client."
+        );
+        this.resultTimer = activity.create().timer(
+            "result",
+            hdrdigits,
+            MetricCategory.Core,
+            "Time how long it takes to submit a request, receive a result, including binding, reading results, " +
+                "and optionally verifying them, including all operations whether successful or not, for each attempted request."
+        );
+        this.resultSuccessTimer = activity.create().timer(
+            "result_success",
+            hdrdigits,
+            MetricCategory.Core,
+            "The execution time of successful operations, which includes submitting the operation, waiting for a response, and reading the result"
+        );
+        this.triesHistogram = activity.create().histogram(
+            "tries",
+            hdrdigits,
+            MetricCategory.Core,
+            "A histogram of all tries for an activity. Perfect results mean all quantiles return 1." +
+                " Slight saturation is indicated by p99 or p95 returning higher values." +
+                " Lower quantiles returning more than 1, or higher values at high quantiles indicate incremental overload."
+        );
+        this.verifierTimer = activity.create().timer(
+            "verifier",
+            hdrdigits,
+            MetricCategory.Verification,
+            "Time the execution of verifier code, if any"
+        );
+        this.errorRate1m = activity.create().gauge("error_rate_1m",
+            () -> {
+                double result_1m_rate = this.resultTimer.getOneMinuteRate();
+                double result_success_1m_rate = this.resultSuccessTimer.getOneMinuteRate();
+                if (result_1m_rate==0.0d || Double.isNaN(result_1m_rate)) {
+                    return Double.NaN;
+                }
+                return (result_1m_rate-result_success_1m_rate)/result_1m_rate;
+            },
+            MetricCategory.Core,
+            "The relative one minute error rate estimated from the one minute successful and non-successful op rates"
+        );
+        this.errorRate5m = activity.create().gauge("error_rate_5m",
+            () -> {
+                double result_5m_rate = this.resultTimer.getFiveMinuteRate();
+                double result_success_5m_rate = this.resultSuccessTimer.getFiveMinuteRate();
+                if (result_5m_rate == 0.0d || Double.isNaN(result_5m_rate)) {
+                    return Double.NaN;
+                }
+                return (result_5m_rate - result_success_5m_rate) / result_5m_rate;
+            },
+            MetricCategory.Core,
+            "The relative five minute error rate estimated from the five minute successful and non-successful op rates"
+        );
+        this.errorRate15m = activity.create().gauge("error_rate_15m",
+            () -> {
+                double result_15m_rate = this.resultTimer.getFifteenMinuteRate();
+                double result_success_15m_rate = this.resultSuccessTimer.getFifteenMinuteRate();
+                if (result_15m_rate == 0.0d || Double.isNaN(result_15m_rate)) {
+                    return Double.NaN;
+                }
+                return (result_15m_rate - result_success_15m_rate) / result_15m_rate;
+            },
+            MetricCategory.Core,
+            "The relative fifteen minute error rate estimated from the fifteen minute successful and non-successful op rates"
+        );
+        this.errorRateTotal = activity.create().gauge("error_rate_total",
+            () -> {
+                double result_total = this.resultTimer.getCount();
+                double result_success_total = this.resultSuccessTimer.getCount();
+                if (result_total == 0.0d) {
+                    return Double.NaN;
+                }
+                return (result_total - result_success_total) / result_total;
+            },
+            MetricCategory.Core,
+            "The cumulative error ratio calculated from the cumulative successful and non-successful op totals"
+        );
+        this.errorsTotal = activity.create().gauge("errors_total",
+            () -> {
+                double result_total = this.resultTimer.getCount();
+                double result_success_total = this.resultSuccessTimer.getCount();
+                return (result_total - result_success_total);
+            },
+            MetricCategory.Core,
+            "The total number of errors calculated from the cumulative successful and non-successful op totals"
+        );
+    }
+
+    @Override
+    public NBMetricGauge getOrCreateErrorsTotal() {
+        return this.errorsTotal;
+    }
+    @Override
+    public NBMetricGauge getOrCreateErrorRate1m() {
+        return this.errorRate1m;
+    }
+    @Override
+    public NBMetricGauge getOrCreateErrorRate5m() {
+        return this.errorRate5m;
+    }
+    @Override
+    public NBMetricGauge getOrCreateErrorRate15m() {
+        return this.errorRate15m;
+    }
+    @Override
+    public NBMetricGauge getOrCreateErrorRateTotal() {
+        return this.errorRateTotal;
     }
 
 
     @Override
-    public  Timer getOrCreateInputTimer() {
+    public Timer getOrCreateInputTimer() {
         return readInputTimer;
     }
 
 
     @Override
-    public  Timer getOrCreateStridesServiceTimer() {
+    public Timer getOrCreateStridesServiceTimer() {
         return stridesServiceTimer;
     }
 
     @Override
-    public  Timer getStridesResponseTimerOrNull() {
+    public Timer getStridesResponseTimerOrNull() {
         return stridesResponseTimer;
     }
 
 
     @Override
-    public  Timer getOrCreateCyclesServiceTimer() {
+    public Timer getOrCreateCyclesServiceTimer() {
         return cyclesServiceTimer;
     }
 
     @Override
-    public  Timer getCyclesResponseTimerOrNull() {
+    public Timer getCyclesResponseTimerOrNull() {
         return cyclesResponseTimer;
     }
 
     @Override
-    public  Counter getOrCreatePendingOpCounter() {
+    public Counter getOrCreatePendingOpCounter() {
         return pendingOpsCounter;
     }
 
     @Override
-    public  Counter getOrCreateOpTrackerBlockedCounter() {
-        return opTrackerBlockedCounter;
-    }
-
-    @Override
-    public  Timer getOrCreateBindTimer() {
+    public Timer getOrCreateBindTimer() {
         return bindTimer;
     }
 
     @Override
-    public  Timer getOrCreateExecuteTimer() {
+    public Timer getOrCreateExecuteTimer() {
         return executeTimer;
     }
 
     @Override
-    public  Timer getOrCreateResultTimer() {
+    public Timer getOrCreateResultTimer() {
         return resultTimer;
     }
 
     @Override
-    public  Timer getOrCreateResultSuccessTimer() {
+    public Timer getOrCreateResultSuccessTimer() {
         return resultSuccessTimer;
     }
 
     @Override
-    public  Histogram getOrCreateTriesHistogram() {
+    public Histogram getOrCreateTriesHistogram() {
         return triesHistogram;
     }
 

engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/simrate/CycleRateSpec.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.api.activityapi.ratelimits.simrate;
+package io.nosqlbench.engine.api.activityapi.simrate;
 
 import io.nosqlbench.nb.api.engine.activityimpl.ParameterMap;
 

engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/simrate/RateLimiter.java

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.api.activityapi.ratelimits;
+package io.nosqlbench.engine.api.activityapi.simrate;
 
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.SimRateSpec;
+import io.nosqlbench.engine.api.activityapi.simrate.SimRateSpec;
 
 import java.time.Duration;
 

engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/simrate/RateLimiters.java

@@ -14,11 +14,9 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.api.activityapi.ratelimits;
+package io.nosqlbench.engine.api.activityapi.simrate;
 
 import io.nosqlbench.nb.api.components.core.NBComponent;
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.SimRate;
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.SimRateSpec;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 

engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/simrate/SimRate.java

@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.api.activityapi.ratelimits.simrate;
+package io.nosqlbench.engine.api.activityapi.simrate;
 
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.labels.NBLabels;
 import io.nosqlbench.nb.api.components.core.NBBaseComponent;
 import io.nosqlbench.nb.api.components.core.NBComponent;
-import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -86,9 +86,25 @@ public class SimRate extends NBBaseComponent implements RateLimiter, Thread.Unca
     }
 
     private void initMetrics() {
-        create().gauge("cycles_waittime",() -> (double)getWaitTimeDuration().get(ChronoUnit.NANOS));
-        create().gauge("config_cyclerate", () -> spec.opsPerSec);
-        create().gauge("config_burstrate", () -> spec.burstRatio);
+        create().gauge(
+            "cycles_waittime",
+            () -> (double)getWaitTimeDuration().get(ChronoUnit.NANOS),
+            MetricCategory.Core,
+            "The cumulative scheduling delay which accrues when" +
+                " an activity is not able to execute operations as fast as requested."
+        );
+        create().gauge(
+            "config_cyclerate",
+            () -> spec.opsPerSec,
+            MetricCategory.Config,
+            "The configured cycle rate in ops/s"
+        );
+        create().gauge(
+            "config_burstrate",
+            () -> spec.burstRatio,
+            MetricCategory.Config,
+            "the configured burst rate as a multiplier to the configured cycle rate. ex: 1.05 means 5% faster is allowed."
+        );
     }
 
     public long refill() {

engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/simrate/SimRateSpec.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.api.activityapi.ratelimits.simrate;
+package io.nosqlbench.engine.api.activityapi.simrate;
 
 import io.nosqlbench.nb.api.engine.activityimpl.ParameterMap;
 import io.nosqlbench.nb.api.engine.util.Unit;

engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/simrate/SimThreads.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.api.activityapi.ratelimits.simrate;
+package io.nosqlbench.engine.api.activityapi.simrate;
 
 import java.util.concurrent.Semaphore;
 import java.util.function.Function;

engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/simrate/StrideRateSpec.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.api.activityapi.ratelimits.simrate;
+package io.nosqlbench.engine.api.activityapi.simrate;
 
 import io.nosqlbench.nb.api.engine.activityimpl.ParameterMap;
 

engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java

@@ -20,7 +20,6 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.EmitterOpDispenserWrapper
 import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
 import io.nosqlbench.engine.core.lifecycle.scenario.container.InvokableResult;
 import io.nosqlbench.nb.api.components.core.NBComponent;
-import io.nosqlbench.nb.api.components.core.NBBaseComponent;
 import io.nosqlbench.nb.api.components.events.ParamChange;
 import io.nosqlbench.engine.api.activityapi.core.*;
 import io.nosqlbench.engine.api.activityapi.core.progress.ActivityMetricProgressMeter;
@@ -29,11 +28,12 @@ import io.nosqlbench.engine.api.activityapi.errorhandling.ErrorMetrics;
 import io.nosqlbench.engine.api.activityapi.errorhandling.modular.NBErrorHandler;
 import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
 import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
-import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.CycleRateSpec;
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.SimRateSpec;
+import io.nosqlbench.engine.api.activityapi.simrate.RateLimiters;
+import io.nosqlbench.engine.api.activityapi.simrate.CycleRateSpec;
+import io.nosqlbench.engine.api.activityapi.simrate.SimRateSpec;
 import io.nosqlbench.adapters.api.activityimpl.OpDispenser;
 import io.nosqlbench.adapters.api.activityimpl.OpMapper;
+import io.nosqlbench.nb.api.components.status.NBStatusComponent;
 import io.nosqlbench.nb.api.labels.NBLabels;
 import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
 import io.nosqlbench.nb.api.errors.BasicError;
@@ -42,12 +42,12 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.filters.IntPredicateDispens
 import io.nosqlbench.engine.api.activityapi.input.InputDispenser;
 import io.nosqlbench.engine.api.activityapi.output.OutputDispenser;
 import io.nosqlbench.engine.api.activityapi.planning.SequencePlanner;
-import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
+import io.nosqlbench.engine.api.activityapi.simrate.RateLimiter;
 import io.nosqlbench.adapters.api.activityconfig.OpsLoader;
 import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate;
 import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplateFormat;
 import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList;
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.StrideRateSpec;
+import io.nosqlbench.engine.api.activityapi.simrate.StrideRateSpec;
 import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
 import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
 import io.nosqlbench.adapters.api.activityimpl.uniform.DryRunOpDispenserWrapper;
@@ -67,7 +67,7 @@ import java.util.function.Supplier;
 /**
  * A default implementation of an Activity, suitable for building upon.
  */
-public class SimpleActivity extends NBBaseComponent implements Activity, InvokableResult {
+public class SimpleActivity extends NBStatusComponent implements Activity, InvokableResult {
     private static final Logger logger = LogManager.getLogger("ACTIVITY");
 
     protected ActivityDef activityDef;

engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/input/AtomicInput.java

@@ -23,6 +23,7 @@ import io.nosqlbench.nb.api.components.core.NBBaseComponent;
 import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
 import io.nosqlbench.engine.api.activityapi.cyclelog.buffers.results.CycleSegment;
 import io.nosqlbench.engine.api.activityapi.input.Input;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -60,22 +61,62 @@ public class AtomicInput extends NBBaseComponent implements Input, ActivityDefOb
         super(parent);
         this.activityDef = activityDef;
         onActivityDefUpdate(activityDef);
-        create().gauge("input_cycles_first",() -> (double) this.cycles_min.get());
-        create().gauge("input_cycles_last",() -> (double) this.cycles_max.get());
-        create().gauge("input_cycle",() -> (double) this.cycle_value.get());
-        create().gauge("input_cycles_total",this::getTotalCycles);
-        create().gauge("input_recycles_first",() -> (double) this.recycles_min.get());
-        create().gauge("input_recycles_last",() -> (double) this.recycles_max.get());
-        create().gauge("input_recycle",() -> (double) this.recycle_value.get());
-        create().gauge("input_recycles_total",this::getTotalRecycles);
+        create().gauge(
+            "input_cycles_first",
+            () -> (double) this.cycles_min.get(),
+            MetricCategory.Config,
+            "The first cycle of the cycle interval, inclusive"
+        );
+        create().gauge(
+            "input_cycles_last",
+            () -> (double) this.cycles_max.get(),
+            MetricCategory.Config,
+            "The last cycle of the cycle interval, exclusive"
+        );
+        create().gauge(
+            "input_cycle",
+            () -> (double) this.cycle_value.get(),
+            MetricCategory.Core,
+            "The next input cycle that will be dispatched to a thread"
+        );
+        create().gauge(
+            "input_cycles_total",
+            this::getTotalCycles,
+            MetricCategory.Config,
+            "The total number of cycles to be executed"
+        );
+        create().gauge(
+            "input_recycles_first",
+            () -> (double) this.recycles_min.get(),
+            MetricCategory.Config,
+            "The first recycle value, inclusive"
+        );
+        create().gauge(
+            "input_recycles_last",
+            () -> (double) this.recycles_max.get(),
+            MetricCategory.Config,
+            "The last recycle value, exclusive"
+        );
+        create().gauge(
+            "input_recycle",
+            () -> (double) this.recycle_value.get(),
+            MetricCategory.Core,
+            "The next recycle value that will be dispatched once cycles are completed"
+        );
+        create().gauge(
+            "input_recycles_total",
+            this::getTotalRecycles,
+            MetricCategory.Config,
+            "The total number of recycles to be executed, within which each set of cycles will be executed"
+        );
     }
 
     private double getTotalRecycles() {
-        return 0.0d;
+        return ((double)this.recycles_max.get())-((double)this.recycles_min.get());
     }
 
     private double getTotalCycles() {
-        return 0.0d;
+        return ((double)this.cycles_max.get())-((double)this.cycles_min.get());
     }
 
     @Override

engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java

@@ -15,20 +15,16 @@
  */
 package io.nosqlbench.engine.api.activityimpl.motor;
 
-import com.codahale.metrics.Counter;
 import com.codahale.metrics.Timer;
 import io.nosqlbench.engine.api.activityapi.core.*;
-import io.nosqlbench.engine.api.activityapi.core.ops.fluent.opfacets.TrackedOp;
-import io.nosqlbench.engine.api.activityapi.cyclelog.buffers.op_output.StrideOutputConsumer;
 import io.nosqlbench.engine.api.activityapi.cyclelog.buffers.results.CycleResultSegmentBuffer;
 import io.nosqlbench.engine.api.activityapi.cyclelog.buffers.results.CycleResultsSegment;
 import io.nosqlbench.engine.api.activityapi.cyclelog.buffers.results.CycleSegment;
 import io.nosqlbench.engine.api.activityimpl.MotorState;
 import io.nosqlbench.engine.api.activityapi.core.ops.fluent.OpTracker;
-import io.nosqlbench.engine.api.activityapi.core.ops.fluent.OpTrackerImpl;
 import io.nosqlbench.engine.api.activityapi.input.Input;
 import io.nosqlbench.engine.api.activityapi.output.Output;
-import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
+import io.nosqlbench.engine.api.activityapi.simrate.RateLimiter;
 import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
@@ -75,8 +71,6 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
     private int stride = 1;
 
     private OpTracker<D> opTracker;
-    private Counter optrackerBlockCounter;
-
 
     /**
      * Create an ActivityMotor.
@@ -194,7 +188,6 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
             inputTimer = activity.getInstrumentation().getOrCreateInputTimer();
             strideServiceTimer = activity.getInstrumentation().getOrCreateStridesServiceTimer();
             stridesResponseTimer = activity.getInstrumentation().getStridesResponseTimerOrNull();
-            optrackerBlockCounter = activity.getInstrumentation().getOrCreateOpTrackerBlockedCounter();
 
             strideRateLimiter = activity.getStrideLimiter();
             cycleRateLimiter = activity.getCycleLimiter();

engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java

@@ -26,6 +26,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
 import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider;
 import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
 import io.nosqlbench.adapters.api.templating.ParsedOp;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.lifecycle.Shutdownable;
 import io.nosqlbench.nb.api.components.core.NBComponent;
 import io.nosqlbench.nb.api.config.standard.*;
@@ -38,8 +39,8 @@ import io.nosqlbench.nb.api.components.events.ParamChange;
 import io.nosqlbench.nb.api.components.events.SetThreads;
 import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
 import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.CycleRateSpec;
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.StrideRateSpec;
+import io.nosqlbench.engine.api.activityapi.simrate.CycleRateSpec;
+import io.nosqlbench.engine.api.activityapi.simrate.StrideRateSpec;
 import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
 import io.nosqlbench.nb.annotations.ServiceSelector;
 import org.apache.logging.log4j.LogManager;
@@ -155,11 +156,23 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
         }
 
        create().gauge(
-            "ops_pending", () -> this.getProgressMeter().getSummary().pending());
+            "ops_pending",
+           () -> this.getProgressMeter().getSummary().pending(),
+           MetricCategory.Core,
+           "The current number of operations which have not been dispatched for processing yet."
+       );
        create().gauge(
-            "ops_active", () -> this.getProgressMeter().getSummary().current());
+            "ops_active",
+           () -> this.getProgressMeter().getSummary().current(),
+           MetricCategory.Core,
+           "The current number of operations which have been dispatched for processing, but which have not yet completed."
+       );
        create().gauge(
-            "ops_complete", () -> this.getProgressMeter().getSummary().complete());
+            "ops_complete",
+           () -> this.getProgressMeter().getSummary().complete(),
+           MetricCategory.Core,
+           "The current number of operations which have been completed"
+       );
     }
 
     @Override

engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/actions/StandardAction.java

@@ -48,9 +48,6 @@ import java.util.concurrent.TimeUnit;
  */
 public class StandardAction<A extends StandardActivity<R, ?>, R extends Op> implements SyncAction, ActivityDefObserver {
     private final static Logger logger = LogManager.getLogger("ACTION");
-
-    private final A activity;
-    private final int slot;
     private final Timer executeTimer;
     private final Histogram triesHistogram;
     private final Timer resultSuccessTimer;
@@ -62,9 +59,7 @@ public class StandardAction<A extends StandardActivity<R, ?>, R extends Op> impl
     private final Timer verifierTimer;
 
     public StandardAction(A activity, int slot) {
-        this.activity = activity;
         this.opsequence = activity.getOpSequence();
-        this.slot = slot;
         this.maxTries = activity.getMaxTries();
         bindTimer = activity.getInstrumentation().getOrCreateBindTimer();
         executeTimer = activity.getInstrumentation().getOrCreateExecuteTimer();
@@ -110,7 +105,7 @@ public class StandardAction<A extends StandardActivity<R, ?>, R extends Op> impl
                         throw new RuntimeException("The op implementation did not implement any active logic. Implement " +
                             "one of [RunnableOp, CycleOp, or ChainingOp]");
                     }
-
+                    // TODO: break out validation timer from execute
                     try (Timer.Context ignored = verifierTimer.time()) {
                         CycleFunction<Boolean> verifier = dispenser.getVerifier();
                         try {

engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionCountMetrics.java

@@ -18,6 +18,7 @@ package io.nosqlbench.engine.api.metrics;
 
 import com.codahale.metrics.Counter;
 import io.nosqlbench.nb.api.components.core.NBComponent;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -33,7 +34,11 @@ public class ExceptionCountMetrics {
 
     public ExceptionCountMetrics(final NBComponent parent) {
         this.parent = parent;
-        this.allerrors=parent.create().counter( "errors_ALL");
+        this.allerrors=parent.create().counter(
+            "errors_ALL",
+            MetricCategory.Errors,
+            "all errors, regardless of type type, for " + parent.description()
+        );
     }
 
     public void count(final String name) {
@@ -41,7 +46,11 @@ public class ExceptionCountMetrics {
         if (null == c) synchronized (this.counters) {
             c = this.counters.computeIfAbsent(
                 name,
-                k -> parent.create().counter("errors_" + name)
+                k -> parent.create().counter(
+                    "errors_" + name,
+                    MetricCategory.Errors,
+                    name + " errors for " + parent.description()
+                )
             );
         }
         c.inc();

engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionExpectedResultVerificationMetrics.java

@@ -18,6 +18,7 @@ package io.nosqlbench.engine.api.metrics;
 
 import com.codahale.metrics.Counter;
 import io.nosqlbench.nb.api.components.core.NBComponent;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 
 
 /**
@@ -30,8 +31,16 @@ public class ExceptionExpectedResultVerificationMetrics {
 
     public ExceptionExpectedResultVerificationMetrics(final NBComponent parent) {
         this.parent = parent;
-        this.verificationRetries=parent.create().counter("verificationcounts_RETRIES");
-        this.verificationErrors=parent.create().counter( "verificationcounts_ERRORS");
+        this.verificationRetries=parent.create().counter(
+            "verificationcounts_RETRIES",
+            MetricCategory.Verification,
+            "The number of retries used by the optional verifier logic for a given operation"
+        );
+        this.verificationErrors=parent.create().counter(
+            "verificationcounts_ERRORS",
+            MetricCategory.Verification,
+            "The number of errors encountered while attempting to verify the result of an soperation"
+        );
     }
 
     public void countVerificationRetries() {

engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionHistoMetrics.java

@@ -19,6 +19,7 @@ package io.nosqlbench.engine.api.metrics;
 import com.codahale.metrics.Histogram;
 import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
 import io.nosqlbench.nb.api.components.core.NBComponent;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -38,7 +39,12 @@ public class ExceptionHistoMetrics {
     public ExceptionHistoMetrics(final NBComponent parent, final ActivityDef activityDef) {
         this.parent = parent;
         this.activityDef = activityDef;
-        this.allerrors = parent.create().histogram( "errorhistos_ALL", activityDef.getParams().getOptionalInteger("hdr_digits").orElse(4));
+        this.allerrors = parent.create().histogram(
+            "errorhistos_ALL",
+            activityDef.getParams().getOptionalInteger("hdr_digits").orElse(4),
+            MetricCategory.Errors,
+            "A histogram for all exceptions"
+        );
     }
 
     public void update(final String name, final long magnitude) {
@@ -46,7 +52,12 @@ public class ExceptionHistoMetrics {
         if (null == h) synchronized (this.histos) {
             h = this.histos.computeIfAbsent(
                 name,
-                errName -> parent.create().histogram( "errorhistos_"+errName, this.activityDef.getParams().getOptionalInteger("hdr_digits").orElse(4))
+                errName -> parent.create().histogram(
+                    "errorhistos_"+errName,
+                    this.activityDef.getParams().getOptionalInteger("hdr_digits").orElse(4),
+                    MetricCategory.Errors,
+                    "error histogram for exception '" + errName + "'"
+                )
             );
         }
         h.update(magnitude);

engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionMeterMetrics.java

@@ -18,6 +18,7 @@ package io.nosqlbench.engine.api.metrics;
 
 import com.codahale.metrics.Meter;
 import io.nosqlbench.nb.api.components.core.NBComponent;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -33,7 +34,9 @@ public class ExceptionMeterMetrics {
 
     public ExceptionMeterMetrics(final NBComponent parent) {
         this.parent = parent;
-        this.allerrors = parent.create().meter("errormeters_ALL");
+        this.allerrors = parent.create().meter("errormeters_ALL", MetricCategory.Errors,
+            "all errors, regardless of type type, for the parent " + parent.description()
+        );
     }
 
     public void mark(final String name) {
@@ -41,7 +44,11 @@ public class ExceptionMeterMetrics {
         if (null == c) synchronized (this.meters) {
             c = this.meters.computeIfAbsent(
                 name,
-                k -> parent.create().meter("errormeters_" + name)
+                k -> parent.create().meter(
+                    "errormeters_" + name,
+                    MetricCategory.Errors,
+                    name + " errors for " + parent.description()
+                )
             );
         }
         c.mark();

engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionTimerMetrics.java

@@ -19,6 +19,7 @@ package io.nosqlbench.engine.api.metrics;
 import com.codahale.metrics.Timer;
 import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
 import io.nosqlbench.nb.api.components.core.NBComponent;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -38,14 +39,24 @@ public class ExceptionTimerMetrics {
         this.activityDef = activityDef;
         this.parentLabels = parent;
 
-        this.allerrors=parent.create().timer("errortimers_ALL",4);
+        this.allerrors=parent.create().timer(
+            "errortimers_ALL",
+            4,
+            MetricCategory.Errors,
+            "exception timers for all error types"
+        );
     }
 
     public void update(final String name, final long nanosDuration) {
         Timer timer = this.timers.get(name);
         if (null == timer) synchronized (this.timers) {
             timer = this.timers.computeIfAbsent(
-                name, k -> parentLabels.create().timer("errortimers_" + name, 3)
+                name, k -> parentLabels.create().timer(
+                    "errortimers_" + name,
+                    3,
+                    MetricCategory.Errors,
+                    "exception timers for specific error '" + name + "'"
+                )
             );
         }
         timer.update(nanosDuration, TimeUnit.NANOSECONDS);

engine-core/src/main/java/io/nosqlbench/engine/cli/NBJavaNativeResolver.java

@@ -27,11 +27,21 @@ import io.nosqlbench.nb.annotations.Service;
 @Service(value= NBInvokableResolver.class,selector = "java")
 public class NBJavaNativeResolver implements NBInvokableResolver {
     @Override
-    public NBInvokableCommand resolve(Cmd cmd, NBBufferedContainer parent, String phaseName) {
+    public NBInvokableCommand resolve(Cmd cmd, NBBufferedContainer parent, String stepname) {
         return switch (cmd.getCmdType()) {
-            case CmdType.indirect -> NBJavaCommandLoader.init(cmd.getArgValue("_impl"), parent, phaseName, cmd.getTargetContext());
-            case CmdType.java -> NBJavaCommandLoader.init(cmd.getArgValue("class"), parent, phaseName, cmd.getTargetContext());
+            case CmdType.indirect -> {
+                String implName = cmd.getArgValue("_impl");
+                NBInvokableCommand invokable = NBJavaCommandLoader.init(implName, parent, stepname, cmd.getTargetContext());
+                cmd.takeArgValue("_impl");
+                yield invokable;
+            }
+            case CmdType.java -> NBJavaCommandLoader.init(cmd.getArgValue("class"), parent, stepname, cmd.getTargetContext());
             default -> null;
         };
     }
+
+    @Override
+    public boolean verify(Cmd cmd) {
+        return NBJavaCommandLoader.oneExists(cmd.getArgValue("_impl")) != null;
+    }
 }

engine-core/src/main/java/io/nosqlbench/engine/cmdstream/BasicScriptBuffer.java

@@ -74,12 +74,7 @@ public class BasicScriptBuffer implements ScriptBuffer {
                 }
                 break;
             case java:
-            case start: // start activity
-            case run: // run activity
-            case await: // await activity
 //            case stop: // stop activity
-            case forceStop: // force stopping activity
-            case waitMillis:
 
                 sb.append("controller.").append(cmd.asScriptText()).append("\n");
 ////                // Sanity check that this can parse before using it

engine-core/src/main/java/io/nosqlbench/engine/cmdstream/Cmd.java

@@ -55,6 +55,12 @@ public class Cmd {
         return cmdArg.getValue();
 
     }
+
+    public String takeArgValue(String paramName) {
+        String argValue = getArgValue(paramName);
+        this.cmdArgs.remove(paramName);
+        return argValue;
+    }
     public String getArgValue(String paramName) {
         CmdArg cmdArg = this.cmdArgs.get(paramName);
         if (cmdArg==null) {
@@ -127,8 +133,12 @@ public class Cmd {
 
     public String toString() {
         StringBuilder sb = new StringBuilder();
-        sb.append(cmdType.toString());
+        sb.append(switch (cmdType) {
+            case indirect -> getArgs().containsKey("_impl") ? getArgs().get("_impl").getValue() : "[]";
+            default -> cmdType.name();
+        });
         for (CmdArg value : getArgs().values()) {
+            if (value.getParam().name.startsWith("_impl")) continue;
             sb.append(" ").append(value);
         }
         return sb.toString();

engine-core/src/main/java/io/nosqlbench/engine/cmdstream/CmdType.java

@@ -23,14 +23,14 @@ package io.nosqlbench.engine.cmdstream;
  * or an error is thrown.
  */
 public enum CmdType {
-    run(),
-    start(),
+//    run(),
+//    start(),
 //    stop(CmdParam.of("activity")),
-    forceStop(CmdParam.of("activity")),
+//    forceStop(CmdParam.of("activity")),
     script(CmdParam.of("path", s -> s)),
     java(CmdParam.of("class", s -> s)),
-    await(CmdParam.of("activity")),
-    waitMillis(CmdParam.of("ms", Long::parseLong)),
+//    await(CmdParam.of("activity")),
+//    waitMillis(CmdParam.of("ms", Long::parseLong)),
     fragment(CmdParam.ofFreeform("fragment")),
     container(CmdParam.of("name")),
     indirect(CmdParam.of("indirect"));

engine-core/src/main/java/io/nosqlbench/engine/cmdstream/NBJavaCommandLoader.java

@@ -39,9 +39,9 @@ public class NBJavaCommandLoader {
         }
     }
 
-    public static NBInvokableCommand init(String cmdSelector, NBComponent parent, String cmdAlias, String ctxName) {
+    public static NBInvokableCommand init(String cmdSelector, NBComponent parent, String stepName, String ctxName) {
         NBCommandInfo cmdInfo = getSelector(cmdSelector).getOne();
-        NBInvokableCommand command = cmdInfo.create(parent, cmdAlias, ctxName);
+        NBInvokableCommand command = cmdInfo.create(parent, cmdSelector, ctxName);
         return command;
     }
 

engine-core/src/main/java/io/nosqlbench/engine/core/clientload/MemInfoReader.java

@@ -46,9 +46,10 @@ public class MemInfoReader extends LinuxSystemFileReader {
 
     public Double getMemUsedkB() {
         Double memTotal = getMemTotalkB();
-        Double memFree = getMemFreekB();
-        if (memTotal != null && memFree != null)
-            return memTotal - memFree;
+        Double memAvailable = getMemAvailablekB();
+//        Double memFree = getMemFreekB();
+        if (memTotal != null && memAvailable != null)
+            return memTotal - memAvailable;
         return null;
     }
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java

@@ -16,6 +16,7 @@
 package io.nosqlbench.engine.core.lifecycle.activity;
 
 import com.codahale.metrics.Gauge;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge;
 import io.nosqlbench.nb.api.labels.NBLabeledElement;
 import io.nosqlbench.nb.api.labels.NBLabels;
@@ -463,7 +464,12 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
 //    }
 
     private void registerMetrics() {
-        this.activity.create().gauge("threads",() -> (double) this.motors.size());
+        this.activity.create().gauge(
+            "threads",
+            () -> (double) this.motors.size(),
+            MetricCategory.Core,
+            "The current number of threads in activity " + this.description()
+        );
     }
 
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/CMD_await.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand;
+import io.nosqlbench.nb.annotations.Service;
+import io.nosqlbench.nb.api.engine.util.Unit;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.util.concurrent.locks.LockSupport;
+
+@Service(value = NBBaseCommand.class, selector = "await")
+public class CMD_await extends NBBaseCommand {
+    public final static Logger logger = LogManager.getLogger("await");
+
+    public CMD_await(NBBufferedContainer parentComponent, String stepName, String targetScenario) {
+        super(parentComponent, stepName, targetScenario);
+    }
+
+    @Override
+    public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) {
+        controller.await(params);
+        return null;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/CMD_example.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import io.nosqlbench.engine.api.activityapi.core.Activity;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand;
+import io.nosqlbench.nb.annotations.Service;
+import io.nosqlbench.nb.api.errors.BasicError;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.util.Optional;
+
+@Service(value = NBBaseCommand.class,selector = "example")
+public class CMD_example extends NBBaseCommand {
+    public final static Logger logger = LogManager.getLogger("example");
+
+    public CMD_example(NBBufferedContainer parentComponent, String stepName, String targetScenario) {
+        super(parentComponent, stepName, targetScenario);
+    }
+
+    @Override
+    public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) {
+        return null;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/CMD_forceStop.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import io.nosqlbench.engine.api.activityapi.core.Activity;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand;
+import io.nosqlbench.nb.annotations.Service;
+import io.nosqlbench.nb.api.errors.BasicError;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.util.Optional;
+
+@Service(value = NBBaseCommand.class,selector = "force_stop")
+public class CMD_forceStop extends NBBaseCommand {
+    public final static Logger logger = LogManager.getLogger("force_stop");
+
+    public CMD_forceStop(NBBufferedContainer parentComponent, String stepName, String targetScenario) {
+        super(parentComponent, stepName, targetScenario);
+    }
+
+    @Override
+    public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) {
+        controller.forceStop(params);
+        return null;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/CMD_run.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand;
+import io.nosqlbench.nb.annotations.Service;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.PrintWriter;
+import java.io.Reader;
+
+@Service(value = NBBaseCommand.class, selector = "run")
+public class CMD_run extends NBBaseCommand {
+    public final static Logger logger = LogManager.getLogger("run");
+
+    public CMD_run(NBBufferedContainer parentComponent, String stepName, String targetScenario) {
+        super(parentComponent, stepName, targetScenario);
+    }
+
+    @Override
+    public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) {
+        controller.run(params);
+        return null;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/CMD_start.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import io.nosqlbench.engine.api.activityapi.core.Activity;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand;
+import io.nosqlbench.nb.annotations.Service;
+import io.nosqlbench.nb.api.errors.BasicError;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.util.Optional;
+
+@Service(value = NBBaseCommand.class,selector = "start")
+public class CMD_start extends NBBaseCommand {
+    public final static Logger logger = LogManager.getLogger("start");
+
+    public CMD_start(NBBufferedContainer parentComponent, String stepName, String targetScenario) {
+        super(parentComponent, stepName, targetScenario);
+    }
+
+    @Override
+    public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) {
+        return controller.start(params);
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/CMD_wait.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import com.amazonaws.services.s3.model.transform.Unmarshallers;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand;
+import io.nosqlbench.nb.annotations.Service;
+import io.nosqlbench.nb.api.engine.util.Unit;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.util.Optional;
+import java.util.concurrent.locks.LockSupport;
+import java.util.stream.LongStream;
+
+@Service(value = NBBaseCommand.class, selector = "example")
+public class CMD_wait extends NBBaseCommand {
+    public final static Logger logger = LogManager.getLogger("example");
+
+    public CMD_wait(NBBufferedContainer parentComponent, String stepName, String targetScenario) {
+        super(parentComponent, stepName, targetScenario);
+    }
+
+    @Override
+    public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) {
+        long ns = 0L;
+        ns += params.maybeGet("ms")
+            .or(() -> params.maybeGet("millis"))
+            .map(Long::parseLong)
+            .map(l -> l * 1_000_000L)
+            .orElse(0L);
+        ns += params.maybeGet("us")
+            .or(() -> params.maybeGet("micros"))
+            .map(Long::parseLong)
+            .map(l -> l * 1_000L)
+            .orElse(0L);
+        ns += params.maybeGet("ns")
+            .or(() -> params.maybeGet("nanos"))
+            .map(Long::parseLong)
+            .orElse(0L);
+        ns += params.maybeGet("unit")
+                .flatMap(Unit::nanosecondsFor)
+                .orElse(0L);
+        LockSupport.parkNanos(ns);
+        return ns;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/INFO_await.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBCommandInfo;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
+import io.nosqlbench.nb.annotations.Service;
+
+@Service(value = NBCommandInfo.class,selector = "await")
+public class INFO_await extends NBCommandInfo {
+    @Override
+    public Class<? extends NBInvokableCommand> getType() {
+        return CMD_await.class;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/INFO_example.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBCommandInfo;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
+import io.nosqlbench.nb.annotations.Service;
+
+@Service(value = NBCommandInfo.class,selector = "example")
+public class INFO_example extends NBCommandInfo {
+    @Override
+    public Class<? extends NBInvokableCommand> getType() {
+        return CMD_example.class;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/INFO_forceStop.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBCommandInfo;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
+import io.nosqlbench.nb.annotations.Service;
+
+@Service(value = NBCommandInfo.class,selector = "force_stop")
+public class INFO_forceStop extends NBCommandInfo {
+    @Override
+    public Class<? extends NBInvokableCommand> getType() {
+        return CMD_forceStop.class;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/INFO_run.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBCommandInfo;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
+import io.nosqlbench.nb.annotations.Service;
+
+@Service(value = NBCommandInfo.class,selector = "run")
+public class INFO_run extends NBCommandInfo {
+    @Override
+    public Class<? extends NBInvokableCommand> getType() {
+        return CMD_run.class;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/INFO_start.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBCommandInfo;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
+import io.nosqlbench.nb.annotations.Service;
+
+@Service(value = NBCommandInfo.class,selector = "start")
+public class INFO_start extends NBCommandInfo {
+    @Override
+    public Class<? extends NBInvokableCommand> getType() {
+        return CMD_start.class;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/INFO_wait.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBCommandInfo;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
+import io.nosqlbench.nb.annotations.Service;
+
+@Service(value = NBCommandInfo.class,selector = "wait")
+public class INFO_wait extends NBCommandInfo {
+    @Override
+    public Class<? extends NBInvokableCommand> getType() {
+        return CMD_wait.class;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/fortesting/CMD_error.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands.fortesting;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand;
+
+import java.io.PrintWriter;
+import java.io.Reader;
+
+public class CMD_error extends NBBaseCommand {
+
+
+    public CMD_error(NBBufferedContainer parentComponent, String stepName, String targetScenario) {
+        super(parentComponent, stepName, targetScenario);
+    }
+
+    @Override
+    public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) {
+        throw new RuntimeException("Command '" + this + "' throws ERROR.");
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/fortesting/CMD_ok.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands.fortesting;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
+import io.nosqlbench.nb.api.labels.NBLabels;
+
+import java.io.PrintWriter;
+import java.io.Reader;
+
+public class CMD_ok extends NBBaseCommand {
+
+    public CMD_ok(NBBufferedContainer parentComponent, String stepName, String targetScenario) {
+        super(parentComponent, stepName, targetScenario);
+    }
+
+    @Override
+    public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) {
+        stdout.write("Command '" + this.toString() + "' says OK and exits with no object or exception.");
+        for (String pkey : params.keySet()) {
+            stdout.println("diagnostic 'ok' command setting key '" + pkey + " to " + params.get(pkey));
+        }
+        return params;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/fortesting/INFO_error.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands.fortesting;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBCommandInfo;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
+import io.nosqlbench.nb.annotations.Service;
+
+@Service(value = NBCommandInfo.class,selector = "test_error")
+public class INFO_error extends NBCommandInfo {
+    @Override
+    public Class<? extends NBInvokableCommand> getType() {
+        return CMD_error.class;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/commands/fortesting/INFO_test_ok.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.commands.fortesting;
+
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBCommandInfo;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
+import io.nosqlbench.nb.annotations.Service;
+
+@Service(value = NBCommandInfo.class,selector = "test_ok")
+public class INFO_test_ok extends NBCommandInfo {
+    @Override
+    public Class<? extends NBInvokableCommand> getType() {
+        return CMD_ok.class;
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/execution/NBCommandInfo.java

@@ -40,7 +40,7 @@ public abstract class NBCommandInfo {
             cmdCtor = getType().getConstructor(NBBufferedContainer.class, String.class, String.class);
             return cmdCtor.newInstance(parent, cmdName, ctxName);
         } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
-            throw new RuntimeException("Unable to instantiate command via ctor(parent,name,ctx): " + e,e);
+            throw new RuntimeException("Unable to instantiate command via ctor(parent,name,ctx): " + e + (e.getCause()!=null ? "cause: " + e.getCause().toString() : ""),e);
         }
     }
 }

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/CmdParser.java

@@ -24,7 +24,6 @@ import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-
 /**
  * <P>Take zero or more strings containing combined argv and return
  * a single {@link Cmd} list containing zero or more commands.</P>
@@ -46,7 +45,7 @@ public class CmdParser {
     private record parameter(String name, String op, String value) {}
     private record command(String name){}
     private final static Pattern combinedPattern =
-        Pattern.compile("(?<varname>[a-zA-Z_][a-zA-Z0-9_.-]+)(?<operator>=+)(?<value>.+)|(?<command>[a-zA-Z_][a-zA-Z0-9_.]+)",Pattern.DOTALL);
+        Pattern.compile("(?<varname>[a-zA-Z_][a-zA-Z0-9_.-]*)(?<operator>=+)(?<value>.+)|(?<command>[a-zA-Z_][a-zA-Z0-9_.]*)",Pattern.DOTALL);
     private final static Pattern commandName =Pattern.compile("^$");
     public static LinkedList<Cmd> parseArgvCommands(LinkedList<String> args) {
         LinkedList<Record> cmdstructs = new LinkedList<>();

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBAutoScriptResolver.java

@@ -34,7 +34,7 @@ import java.util.Optional;
 @Service(value = NBInvokableResolver.class, selector = "autojs")
 public class NBAutoScriptResolver implements NBInvokableResolver {
     @Override
-    public NBInvokableCommand resolve(Cmd cmd, NBBufferedContainer parent, String phaseName) {
+    public NBInvokableCommand resolve(Cmd cmd, NBBufferedContainer parent, String stepname) {
 
         Optional<Content<?>> scriptfile = NBIO.local()
             .searchPrefixes("scripts/auto")
@@ -47,11 +47,21 @@ public class NBAutoScriptResolver implements NBInvokableResolver {
             Map<String, CmdArg> newArgs = new LinkedHashMap<>(cmd.getArgs());
             newArgs.put("path",new CmdArg(new CmdParam("name",s->s,false),"=",pathOf.toString()));
             Cmd reformattedCmd = new Cmd("script", newArgs);
-            return new NBScriptedCommand(parent, phaseName, cmd.getTargetContext()).add(reformattedCmd);
+            return new NBScriptedCommand(parent, stepname, cmd.getTargetContext()).add(reformattedCmd);
         } else {
             return null;
         }
     }
 
+    @Override
+    public boolean verify(Cmd cmd) {
+        return NBIO.local()
+                .searchPrefixes("scripts/auto")
+                .pathname(cmd.getArgValue("_impl"))
+                .extensionSet("js")
+                .first()
+            .isPresent();
+    }
+
 
 }

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBCommandAssembly.java

@@ -29,15 +29,35 @@ import java.util.function.Function;
 
 public class NBCommandAssembly {
 
+    private static NBCoreInvokableResolver core_resolver = new NBCoreInvokableResolver();
+
     private final static Logger logger = LogManager.getLogger(NBCommandAssembly.class);
 
+    public static NBCommandParams paramsFor(Cmd cmd) {
+        return switch (cmd.getCmdType()) {
+            case indirect, java, container -> {
+                Map<String, String> params = cmd.getArgMap();
+                params.remove("_impl");
+                yield NBCommandParams.of(params);
+            }
+            default -> NBCommandParams.of(Map.of());
+        };
+    }
+
     public static record CommandInvocation(NBInvokableCommand command, NBCommandParams params, String containerName) {
     }
 
-    public static List<CommandInvocation> assemble(List<Cmd> cmds, Function<String, NBBufferedContainer> ctxprovider) {
+    public static List<Cmd> assemble(List<Cmd> cmds, Function<String, NBBufferedContainer> ctxprovider) {
         List<Cmd> mappedCmds = tagCommandsWithContext(cmds);
-        List<CommandInvocation> invocations = prepareMappedPhases(mappedCmds, ctxprovider);
-        return invocations;
+        NBCoreInvokableResolver core_resolver = new NBCoreInvokableResolver();
+        for (Cmd mappedCmd : mappedCmds) {
+            core_resolver.verify(mappedCmd);
+        }
+
+        return mappedCmds;
+
+//        List<CommandInvocation> invocations = prepareMappedPhases(mappedCmds, ctxprovider);
+//        return invocations;
     }
 
     private static List<Cmd> tagCommandsWithContext(List<Cmd> cmds) {
@@ -62,6 +82,40 @@ public class NBCommandAssembly {
         return new ArrayList<>(tagged);
     }
 
+
+    public static NBInvokableCommand resolve(Cmd cmd, Function<String, NBBufferedContainer> ctxProvider) {
+        try {
+            NBCommandParams params = switch (cmd.getCmdType()) {
+                case indirect, java, container -> NBCommandParams.of(cmd.getArgMap());
+                default -> NBCommandParams.of(Map.of());
+            };
+
+            String targetContext = cmd.getTargetContext();
+            NBInvokableCommand command = core_resolver.resolve(cmd, ctxProvider.apply(targetContext), cmd.getStepName());
+            return command;
+        } catch (Exception e) {
+            throw new UnresolvedCommand(cmd, e);
+        }
+    }
+
+    public static CommandInvocation assembleCommand(Cmd cmd, Function<String, NBBufferedContainer> ctxProvider) {
+        NBCommandParams params = switch (cmd.getCmdType()) {
+            case indirect, java, container -> NBCommandParams.of(cmd.getArgMap());
+            default -> NBCommandParams.of(Map.of());
+        };
+
+        String targetContext = cmd.getTargetContext();
+        NBInvokableCommand command = core_resolver.resolve(cmd, ctxProvider.apply(targetContext), cmd.getStepName());
+        if (command == null) {
+            throw new BasicError("Found zero commands for spec;" + cmd);
+        }
+        String containerName = cmd.getTargetContext();
+
+        // TODO, make this unnecessary by moving the impl out of the map to a dedicated cmd structure
+        params.remove("_impl");
+        return new CommandInvocation(command, params, containerName);
+    }
+
     private static List<CommandInvocation> prepareMappedPhases(List<Cmd> mappedCmds, Function<String, NBBufferedContainer> ctxProvider) {
         List<CommandInvocation> parameterizedInvocations = new ArrayList<>();
         NBCoreInvokableResolver core_resolver = new NBCoreInvokableResolver();
@@ -73,10 +127,13 @@ public class NBCommandAssembly {
 
             String targetContext = cmd.getTargetContext();
             NBInvokableCommand command = core_resolver.resolve(cmd, ctxProvider.apply(targetContext), cmd.getStepName());
-            if (command==null) {
+            if (command == null) {
                 throw new BasicError("Found zero commands for spec;" + cmd);
             }
             String containerName = cmd.getTargetContext();
+
+            // TODO, make this unnecessary by moving the impl out of the map to a dedicated cmd structure
+            params.remove("_impl");
             parameterizedInvocations.add(new CommandInvocation(command, params, containerName));
         }
         return parameterizedInvocations;

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBCoreInvokableResolver.java

@@ -45,6 +45,16 @@ public class NBCoreInvokableResolver implements NBInvokableResolver {
         return null;
     }
 
+    @Override
+    public boolean verify(Cmd cmd) {
+        for (NBInvokableResolver resolver : getResolvers().values()) {
+            if (resolver.verify(cmd)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private SequencedMap<String, NBInvokableResolver> getResolvers() {
         if (this.resolvers == null || this.resolvers.isEmpty()) {
             SequencedMap<String,NBInvokableResolver> resolverMap = new LinkedHashMap<>();

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBInvokableResolver.java

@@ -21,6 +21,8 @@ import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContaine
 import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
 
 public interface NBInvokableResolver {
-    NBInvokableCommand resolve(Cmd cmd, NBBufferedContainer parent, String phaseName);
+    NBInvokableCommand resolve(Cmd cmd, NBBufferedContainer parent, String stepname);
+
+    boolean verify(Cmd cmd);
 
 }

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBScriptCommandResolver.java

@@ -25,10 +25,10 @@ import io.nosqlbench.nb.annotations.Service;
 @Service(value = NBInvokableResolver.class, selector = "js")
 public class NBScriptCommandResolver implements NBInvokableResolver {
     @Override
-    public NBInvokableCommand resolve(Cmd cmd, NBBufferedContainer parent, String phaseName) {
+    public NBInvokableCommand resolve(Cmd cmd, NBBufferedContainer parent, String stepname) {
         return switch (cmd.getCmdType()) {
-            case run, await, forceStop, start, waitMillis, fragment, script->
-            new NBScriptedCommand(parent, phaseName, cmd.getTargetContext()).add(cmd);
+            case fragment, script->
+            new NBScriptedCommand(parent, stepname, cmd.getTargetContext()).add(cmd);
 //            case fragment ->
 //                new NBScriptedCommand(parent, phaseName, cmd.getTargetContext()).addScriptText(cmd.getArgValue("fragment"));
 //            case script ->
@@ -37,5 +37,8 @@ public class NBScriptCommandResolver implements NBInvokableResolver {
         };
     }
 
-
+    @Override
+    public boolean verify(Cmd cmd) {
+        return true;
+    }
 }

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java

@@ -16,16 +16,14 @@
 
 package io.nosqlbench.engine.core.lifecycle.session;
 
-import io.nosqlbench.nb.api.components.core.NBComponentProps;
+import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams;
+import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
+import io.nosqlbench.nb.api.components.status.NBHeartbeatComponent;
 import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
-import io.nosqlbench.nb.api.engine.metrics.instruments.NBFunctionGauge;
-import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.labels.NBLabeledElement;
-import io.nosqlbench.nb.api.labels.NBLabels;
-import io.nosqlbench.nb.api.components.core.NBBaseComponent;
 import io.nosqlbench.nb.api.components.decorators.NBTokenWords;
 import io.nosqlbench.engine.cmdstream.Cmd;
-import io.nosqlbench.engine.core.clientload.*;
 import io.nosqlbench.engine.core.lifecycle.ExecutionResult;
 import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
 import io.nosqlbench.engine.core.lifecycle.scenario.container.NBContainer;
@@ -43,9 +41,9 @@ import java.util.function.Function;
  * on.
  * All NBScenarios are run within an NBSession.
  */
-public class NBSession extends NBBaseComponent implements Function<List<Cmd>, ExecutionResult>, NBTokenWords {
+public class NBSession extends NBHeartbeatComponent implements Function<List<Cmd>, ExecutionResult>, NBTokenWords {
     private final static Logger logger = LogManager.getLogger(NBSession.class);
-    private final ClientSystemMetricChecker clientMetricChecker;
+//    private final ClientSystemMetricChecker clientMetricChecker;
 
     private final Map<String, NBBufferedContainer> containers = new ConcurrentHashMap<>();
 
@@ -57,55 +55,53 @@ public class NBSession extends NBBaseComponent implements Function<List<Cmd>, Ex
 
     public NBSession(
         NBLabeledElement labelContext,
-        String sessionName
+        String sessionName,
+        Map<String, String> props
     ) {
         super(
             null,
             labelContext.getLabels()
-                .and("session", sessionName)
+                .and("session", sessionName),
+            props,
+            "session"
+        );
+
+        new NBSessionSafetyMetrics(this);
+
+        create().gauge(
+            "session_time",
+            () -> (double) System.nanoTime(),
+            MetricCategory.Core,
+            "session time in nanoseconds"
         );
 
-        this.clientMetricChecker = new ClientSystemMetricChecker(this, NBLabels.forKV(), 10);
-        registerLoadAvgMetrics();
-        registerMemInfoMetrics();
-//        registerDiskStatsMetrics();
-        registerNetworkInterfaceMetrics();
-        registerCpuStatMetrics();
-        clientMetricChecker.start();
         bufferOrphanedMetrics = true;
     }
 
 
-    /**
-     * Notes on scenario names:
-     * <UL>
-     * <LI>If none are provided, then all cmds are implicitly allocated to the "default" scenario.</LI>
-     * <LI>If the name "default" is provided directly, then this is considered an error.</LI>
-     * <LI>Otherwise, the most recently set scenario name is the one in which all following commands are run.</LI>
-     * <LI></LI>
-     * </UL>
-     *
-     * @param cmds
-     *     the function argument
-     * @return
-     */
     public ExecutionResult apply(List<Cmd> cmds) {
 
         // TODO: add container closing command
         // TODO: inject container closing commands after the last command referencing each container
-        List<NBCommandAssembly.CommandInvocation> invocationCalls = NBCommandAssembly.assemble(cmds, this::getContext);
+        List<Cmd> assembledCommands = NBCommandAssembly.assemble(cmds, this::getContext);
         ResultCollector collector = new ResultCollector();
+
         try (ResultContext results = new ResultContext(collector).ok()) {
-            for (NBCommandAssembly.CommandInvocation invocation : invocationCalls) {
-                try {
-                    String targetContext = invocation.containerName();
-                    NBBufferedContainer container = getContext(targetContext);
-                    NBCommandResult cmdResult = container.apply(invocation.command(), invocation.params());
+            for (Cmd cmd : assembledCommands) {
+                String explanation = " in context " + cmd.getTargetContext() + ", command '" + cmd.toString() + "'";
+                try (NBInvokableCommand command = NBCommandAssembly.resolve(cmd,this::getContext)) {
+                    NBCommandParams params = NBCommandAssembly.paramsFor(cmd);
+                    NBBufferedContainer container = getContext(cmd.getTargetContext());
+                    NBCommandResult cmdResult = container.apply(command, params);
                     results.apply(cmdResult);
+                    if (cmdResult.hasException()) {
+                        throw cmdResult.getException();
+                    }
                 } catch (Exception e) {
-                    String msg = "While running command '" + invocation.command() + "' in container '" + invocation.containerName() + "', an error occurred: " + e.toString();
-                    logger.error(msg);
+                    String msg = "While running " + explanation + ", an error occurred: " + e.toString();
                     results.error(e);
+                    onError(e);
+                    logger.error(msg);
                     break;
                 }
             }
@@ -124,79 +120,6 @@ public class NBSession extends NBBaseComponent implements Function<List<Cmd>, Ex
     }
 
 
-    private void registerLoadAvgMetrics() {
-        LoadAvgReader reader = new LoadAvgReader();
-        if (!reader.fileExists())
-            return;
-
-        NBFunctionGauge load1m = create().gauge("loadavg_1min", reader::getOneMinLoadAverage);
-        clientMetricChecker.addMetricToCheck(load1m, 50.0);
-
-        NBFunctionGauge load5m = create().gauge("loadavg_5min", reader::getFiveMinLoadAverage);
-        clientMetricChecker.addMetricToCheck(load5m, 50.0);
-
-        NBFunctionGauge load15m = create().gauge("loadavg_15min", reader::getFifteenMinLoadAverage);
-        clientMetricChecker.addMetricToCheck(load15m, 50.0);
-        // add checking for CPU load averages; TODO: Modify thresholds
-
-    }
-
-    private void registerMemInfoMetrics() {
-        MemInfoReader reader = new MemInfoReader();
-        if (!reader.fileExists())
-            return;
-
-        NBMetricGauge memTotalGauge = create().gauge("mem_total", reader::getMemTotalkB);
-        NBMetricGauge memUsedGauge = create().gauge("mem_used", reader::getMemUsedkB);
-        NBMetricGauge memFreeGauge = create().gauge("mem_free", reader::getMemFreekB);
-        NBMetricGauge memAvailableGauge = create().gauge("mem_avaialble", reader::getMemAvailablekB);
-        NBMetricGauge memCachedGauge = create().gauge("mem_cache", reader::getMemCachedkB);
-        NBMetricGauge memBufferedGauge = create().gauge("mem_buffered", reader::getMemBufferskB);
-        // add checking for percent memory used at some given time; TODO: Modify percent threshold
-        clientMetricChecker.addRatioMetricToCheck(memUsedGauge, memTotalGauge, 90.0, false);
-
-        NBMetricGauge swapTotalGauge = create().gauge("swap_total", reader::getSwapTotalkB);
-        NBMetricGauge swapFreeGauge = create().gauge("swap_free", reader::getSwapFreekB);
-        NBMetricGauge swapUsedGauge = create().gauge("swap_used", reader::getSwapUsedkB);
-    }
-
-    private void registerDiskStatsMetrics() {
-        DiskStatsReader reader = new DiskStatsReader();
-        if (!reader.fileExists())
-            return;
-
-        for (String device : reader.getDevices()) {
-            create().gauge(device + "_transactions", () -> reader.getTransactionsForDevice(device));
-            create().gauge(device + "_kB_read", () -> reader.getKbReadForDevice(device));
-            create().gauge(device + "_kB_written", () -> reader.getKbWrittenForDevice(device));
-        }
-    }
-
-    private void registerNetworkInterfaceMetrics() {
-        NetDevReader reader = new NetDevReader();
-        if (!reader.fileExists())
-            return;
-        for (String iface : reader.getInterfaces()) {
-            create().gauge(iface + "_rx_bytes", () -> reader.getBytesReceived(iface));
-            create().gauge(iface + "_rx_packets", () -> reader.getPacketsReceived(iface));
-            create().gauge(iface + "_tx_bytes", () -> reader.getBytesTransmitted(iface));
-            create().gauge(iface + "_tx_packets", () -> reader.getPacketsTransmitted(iface));
-        }
-    }
-
-    private void registerCpuStatMetrics() {
-        StatReader reader = new StatReader();
-        if (!reader.fileExists())
-            return;
-        NBMetricGauge cpuUserGauge = create().gauge("cpu_user", reader::getUserTime);
-        NBMetricGauge cpuSystemGauge = create().gauge("cpu_system", reader::getSystemTime);
-        NBMetricGauge cpuIdleGauge = create().gauge("cpu_idle", reader::getIdleTime);
-        NBMetricGauge cpuIoWaitGauge = create().gauge("cpu_iowait", reader::getIoWaitTime);
-        NBMetricGauge cpuTotalGauge = create().gauge("cpu_total", reader::getTotalTime);
-        // add checking for percent of time spent in user space; TODO: Modify percent threshold
-        clientMetricChecker.addRatioMetricToCheck(cpuUserGauge, cpuTotalGauge, 50.0, true);
-    }
-
     private NBBufferedContainer getContext(String name) {
         return containers.computeIfAbsent(
             name,

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSessionSafetyMetrics.java

@@ -0,0 +1,253 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.session;
+
+import io.nosqlbench.engine.core.clientload.*;
+import io.nosqlbench.nb.api.components.core.NBBaseComponent;
+import io.nosqlbench.nb.api.components.core.NBComponent;
+import io.nosqlbench.nb.api.components.core.NBComponentMetrics;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
+import io.nosqlbench.nb.api.engine.metrics.instruments.NBFunctionGauge;
+import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge;
+import io.nosqlbench.nb.api.labels.NBLabels;
+
+public class NBSessionSafetyMetrics extends NBBaseComponent {
+
+    public NBSessionSafetyMetrics(NBSession parent) {
+        super(parent, NBLabels.forKV());
+        ClientSystemMetricChecker metricsChecker = registerSessionSafetyMetrics(parent);
+        metricsChecker.start();
+    }
+
+    private ClientSystemMetricChecker registerSessionSafetyMetrics(NBSession nbSession) {
+        ClientSystemMetricChecker metricsChecker = new ClientSystemMetricChecker(
+            this,
+            NBLabels.forKV(),
+            10
+        );
+        registerLoadAvgMetrics(metricsChecker);
+        registerMemInfoMetrics(metricsChecker);
+//        registerDiskStatsMetrics();
+        registerNetworkInterfaceMetrics(metricsChecker);
+        registerCpuStatMetrics(metricsChecker);
+        return metricsChecker;
+    }
+
+    private void registerCpuStatMetrics(ClientSystemMetricChecker metricsChecker) {
+        StatReader reader = new StatReader();
+        if (!reader.fileExists())
+            return;
+        NBMetricGauge cpuUserGauge = create().gauge(
+            "cpu_user",
+            reader::getUserTime,
+            MetricCategory.Internals,
+            ""
+        );
+        NBMetricGauge cpuSystemGauge = create().gauge(
+            "cpu_system",
+            reader::getSystemTime,
+            MetricCategory.Internals,
+            ""
+        );
+        NBMetricGauge cpuIdleGauge = create().gauge(
+            "cpu_idle",
+            reader::getIdleTime,
+            MetricCategory.Internals,
+            ""
+        );
+        NBMetricGauge cpuIoWaitGauge = create().gauge(
+            "cpu_iowait",
+            reader::getIoWaitTime,
+            MetricCategory.Internals,
+            ""
+        );
+        NBMetricGauge cpuTotalGauge = create().gauge(
+            "cpu_total",
+            reader::getTotalTime,
+            MetricCategory.Internals,
+            ""
+        );
+        // add checking for percent of time spent in user space; TODO: Modify percent threshold
+        metricsChecker.addRatioMetricToCheck(
+            cpuUserGauge,
+            cpuTotalGauge,
+            50.0,
+            true
+        );
+    }
+
+    private void registerDiskStatsMetrics(ClientSystemMetricChecker metricsChecker) {
+        DiskStatsReader reader = new DiskStatsReader();
+        if (!reader.fileExists())
+            return;
+
+        for (String device : reader.getDevices()) {
+            create().gauge(
+                device + "_transactions",
+                () -> reader.getTransactionsForDevice(device),
+                MetricCategory.Internals,
+                ""
+            );
+            create().gauge(device + "_kB_read",
+                () -> reader.getKbReadForDevice(device),
+                MetricCategory.Internals,
+                ""
+            );
+            create().gauge(device + "_kB_written",
+                () -> reader.getKbWrittenForDevice(device),
+                MetricCategory.Internals,
+                ""
+            );
+        }
+    }
+
+    private void registerNetworkInterfaceMetrics(ClientSystemMetricChecker metricsChecker) {
+        NetDevReader reader = new NetDevReader();
+        if (!reader.fileExists())
+            return;
+        for (String iface : reader.getInterfaces()) {
+            create().gauge(
+                iface + "_rx_bytes",
+                () -> reader.getBytesReceived(iface),
+                MetricCategory.Internals,
+                ""
+            );
+            create().gauge(
+                iface + "_rx_packets",
+                () -> reader.getPacketsReceived(iface),
+                MetricCategory.Internals,
+                ""
+            );
+            create().gauge(
+                iface + "_tx_bytes",
+                () -> reader.getBytesTransmitted(iface),
+                MetricCategory.Internals,
+                ""
+            );
+            create().gauge(
+                iface + "_tx_packets",
+                () -> reader.getPacketsTransmitted(iface),
+                MetricCategory.Internals,
+                ""
+            );
+        }
+    }
+
+    private void registerLoadAvgMetrics(ClientSystemMetricChecker metricsChecker) {
+        LoadAvgReader reader = new LoadAvgReader();
+        if (!reader.fileExists())
+            return;
+
+        NBFunctionGauge load1m = create().gauge(
+            "loadavg_1min",
+            reader::getOneMinLoadAverage,
+            MetricCategory.Internals,
+            "the 1 minute load average of the test client client"
+        );
+        metricsChecker.addMetricToCheck(load1m, 50.0);
+
+        NBFunctionGauge load5m = create().gauge(
+            "loadavg_5min",
+            reader::getFiveMinLoadAverage,
+            MetricCategory.Internals,
+            "the 5 minute load average of the test client client"
+        );
+        metricsChecker.addMetricToCheck(load5m, 50.0);
+
+        NBFunctionGauge load15m = create().gauge(
+            "loadavg_15min",
+            reader::getFifteenMinLoadAverage,
+            MetricCategory.Internals,
+            "the 15 minute load average of the test client"
+        );
+        metricsChecker.addMetricToCheck(load15m, 50.0);
+        // add checking for CPU load averages; TODO: Modify thresholds
+
+    }
+
+    private void registerMemInfoMetrics(ClientSystemMetricChecker metricsChecker) {
+        MemInfoReader reader = new MemInfoReader();
+        if (!reader.fileExists())
+            return;
+
+        NBMetricGauge memTotalGauge = create().gauge(
+            "mem_total",
+            reader::getMemTotalkB,
+            MetricCategory.Internals,
+            ""
+
+        );
+        NBMetricGauge memUsedGauge = create().gauge(
+            "mem_used",
+            reader::getMemUsedkB,
+            MetricCategory.Internals,
+            ""
+        );
+        NBMetricGauge memFreeGauge = create().gauge(
+            "mem_free",
+            reader::getMemFreekB,
+            MetricCategory.Internals,
+            ""
+        );
+        NBMetricGauge memAvailableGauge = create().gauge(
+            "mem_available",
+            reader::getMemAvailablekB,
+            MetricCategory.Internals,
+            ""
+        );
+        NBMetricGauge memCachedGauge = create().gauge(
+            "mem_cache",
+            reader::getMemCachedkB,
+            MetricCategory.Internals,
+            ""
+        );
+        NBMetricGauge memBufferedGauge = create().gauge(
+            "mem_buffered",
+            reader::getMemBufferskB,
+            MetricCategory.Internals,
+            ""
+        );
+        // add checking for percent memory used at some given time; TODO: Modify percent threshold
+        metricsChecker.addRatioMetricToCheck(
+            memUsedGauge,
+            memTotalGauge,
+            90.0,
+            false
+        );
+
+        NBMetricGauge swapTotalGauge = create().gauge(
+            "swap_total",
+            reader::getSwapTotalkB,
+            MetricCategory.Internals,
+            ""
+        );
+        NBMetricGauge swapFreeGauge = create().gauge(
+            "swap_free",
+            reader::getSwapFreekB,
+            MetricCategory.Internals,
+            ""
+        );
+        NBMetricGauge swapUsedGauge = create().gauge(
+            "swap_used",
+            reader::getSwapUsedkB,
+            MetricCategory.Internals,
+            ""
+        );
+    }
+
+
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/UnresolvedCommand.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.session;
+
+import io.nosqlbench.engine.cmdstream.Cmd;
+import scala.concurrent.impl.FutureConvertersImpl;
+
+public class UnresolvedCommand extends RuntimeException {
+    private final Cmd cmd;
+
+    public UnresolvedCommand(Cmd cmd, Throwable cause) {
+        super(cause);
+        this.cmd = cmd;
+    }
+
+    public String toString() {
+        final String helpmsg = """
+                Could not recognize command 'ARG'.
+                This means that all of the following searches for a compatible command failed:
+                1. commands: no scenario command named 'ARG' is known. (start, run, await, ...)
+                2. scripts: no auto script named './scripts/auto/ARG.js' in the local filesystem.
+                3. scripts: no auto script named 'scripts/auto/ARG.js' was found in the PROG binary.
+                4. workloads: no workload file named ARG[.yaml] was found in the local filesystem, even in include paths INCLUDES.
+                5. workloads: no workload file named ARG[.yaml] was bundled in PROG binary, even in include paths INCLUDES.
+                6. apps: no application named ARG was bundled in PROG.
+
+                You can discover available ways to invoke PROG by using the various --list-* commands:
+                [ --list-commands, --list-scripts, --list-workloads (and --list-scenarios), --list-apps ]
+                """
+            .replaceAll("ARG", getCmdName())
+            .replaceAll("PROG", "nb5");
+
+        return super.toString() + "\nadditional diagnostics:\n" + helpmsg;
+    }
+
+    private String getCmdName() {
+        String impl = cmd.getArgMap().get("_impl");
+        if (impl!=null) return impl;
+        return cmd.toString();
+    }
+
+}

engine-core/src/main/java/io/nosqlbench/engine/core/logging/NBLoggerConfig.java

@@ -19,11 +19,10 @@ package io.nosqlbench.engine.core.logging;
 import io.nosqlbench.nb.api.logging.NBLogLevel;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
 import org.apache.logging.log4j.core.Filter;
 import org.apache.logging.log4j.core.LoggerContext;
-import org.apache.logging.log4j.core.appender.ConsoleAppender;
-import org.apache.logging.log4j.core.appender.FileAppender;
-import org.apache.logging.log4j.core.appender.RollingFileAppender;
+import org.apache.logging.log4j.core.appender.*;
 import org.apache.logging.log4j.core.config.Configuration;
 import org.apache.logging.log4j.core.config.ConfigurationFactory;
 import org.apache.logging.log4j.core.config.ConfigurationSource;
@@ -40,6 +39,8 @@ import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 //@Plugin(name = "CustomConfigurationFactory", category = ConfigurationFactory.CATEGORY)
@@ -52,13 +53,14 @@ import java.util.stream.Collectors;
  *
  * @see <a href="https://logging.apache.org/log4j/2.x/manual/layouts.html#Pattern_Layout">Pattern Layout</a>
  */
-public class LoggerConfig extends ConfigurationFactory {
+public class NBLoggerConfig extends ConfigurationFactory {
 
+    public static final String SESSION_APPENDER = "SESSION_APPENDER";
     public static Map<String, String> STANDARD_FORMATS = Map.of(
-            "TERSE", "%8r %-5level [%t] %-12logger{0} %msg%n%throwable",
-            "VERBOSE", "%d{DEFAULT}{GMT} [%t] %logger %-5level: %msg%n%throwable",
-            "TERSE-ANSI", "%8r %highlight{%-5level} %style{%C{1.} [%t] %-12logger{0}} %msg%n%throwable",
-            "VERBOSE-ANSI", "%d{DEFAULT}{GMT} [%t] %highlight{%logger %-5level}: %msg%n%throwable"
+        "TERSE", "%8r %-5level [%t] %-12logger{0} %msg%n%throwable",
+        "VERBOSE", "%d{DEFAULT}{GMT} [%t] %logger %-5level: %msg%n%throwable",
+        "TERSE-ANSI", "%8r %highlight{%-5level} %style{%C{1.} [%t] %-12logger{0}} %msg%n%throwable",
+        "VERBOSE-ANSI", "%d{DEFAULT}{GMT} [%t] %highlight{%logger %-5level}: %msg%n%throwable"
     );
 
     /**
@@ -66,8 +68,8 @@ public class LoggerConfig extends ConfigurationFactory {
      * we squelch them to some reasonable level so they aren't a nuisance.
      */
     public static Map<String, Level> BUILTIN_OVERRIDES = Map.of(
-            // ERROR StatusConsoleListener Unable to locate appender "SCENARIO_APPENDER" for logger config "oshi.util"
-            "oshi.util", Level.INFO
+        // ERROR StatusConsoleListener Unable to locate appender "SESSION_APPENDER" for logger config "oshi.util"
+        "oshi.util", Level.INFO
     );
 
     /**
@@ -92,25 +94,25 @@ public class LoggerConfig extends ConfigurationFactory {
     private boolean isDedicatedVerificationLoggerEnabled = false;
 
 
-    public LoggerConfig() {
+    public NBLoggerConfig() {
     }
 
-    public LoggerConfig setAnsiEnabled(boolean ansiEnabled) {
+    public NBLoggerConfig setAnsiEnabled(boolean ansiEnabled) {
         this.ansiEnabled = ansiEnabled;
         return this;
     }
 
-    public LoggerConfig setConsoleLevel(NBLogLevel level) {
+    public NBLoggerConfig setConsoleLevel(NBLogLevel level) {
         this.consoleLevel = level;
         return this;
     }
 
-    public LoggerConfig setLogfileLevel(NBLogLevel level) {
+    public NBLoggerConfig setLogfileLevel(NBLogLevel level) {
         this.fileLevel = level;
         return this;
     }
 
-    public LoggerConfig setDedicatedVerificationLogger(boolean enabled) {
+    public NBLoggerConfig setDedicatedVerificationLogger(boolean enabled) {
         this.isDedicatedVerificationLoggerEnabled = enabled;
         return this;
     }
@@ -128,7 +130,7 @@ public class LoggerConfig extends ConfigurationFactory {
         }
     }
 
-    public LoggerConfig setMaxLogs(int maxLogfiles) {
+    public NBLoggerConfig setMaxLogs(int maxLogfiles) {
         this.maxLogfiles = maxLogfiles;
         return this;
     }
@@ -158,20 +160,20 @@ public class LoggerConfig extends ConfigurationFactory {
         builder.setStatusLevel(internalLoggingStatusThreshold);
 
         builder.add(
-                builder.newFilter(
-                        "ThresholdFilter",
-                        Filter.Result.ACCEPT,
-                        Filter.Result.NEUTRAL
-                ).addAttribute("level", builderThresholdLevel)
+            builder.newFilter(
+                "ThresholdFilter",
+                Filter.Result.ACCEPT,
+                Filter.Result.NEUTRAL
+            ).addAttribute("level", builderThresholdLevel)
         );
 
         // CONSOLE appender
         AppenderComponentBuilder appenderBuilder =
-                builder.newAppender("console", "CONSOLE")
-                        .addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT);
+            builder.newAppender("console", "CONSOLE")
+                .addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT);
 
         appenderBuilder.add(builder.newLayout("PatternLayout")
-                .addAttribute("pattern", consolePattern));
+            .addAttribute("pattern", consolePattern));
 
 //        appenderBuilder.add(
 //                builder.newFilter("MarkerFilter", Filter.Result.DENY, Filter.Result.NEUTRAL)
@@ -181,8 +183,15 @@ public class LoggerConfig extends ConfigurationFactory {
 
         // Log4J internal logging
         builder.add(builder.newLogger("org.apache.logging.log4j", Level.DEBUG)
-                .add(builder.newAppenderRef("console"))
-                .addAttribute("additivity", false));
+            .add(builder.newAppenderRef("console"))
+            .addAttribute("additivity", false));
+
+        if (sessionName == null) {
+            sessionName = "LOGINIT";
+            System.out.println("Session was not set in logger config. If you are getting this error, then it means" +
+                " that the session has not initialized properly, and that some unexpected error occurred before the logging" +
+                " system was initialized. Look for errors prior to this.");
+        }
 
         if (sessionName != null) {
 
@@ -196,25 +205,32 @@ public class LoggerConfig extends ConfigurationFactory {
 
             // LOGFILE appender
             LayoutComponentBuilder logfileLayout = builder.newLayout("PatternLayout")
-                    .addAttribute("pattern", logfilePattern);
+                .addAttribute("pattern", logfilePattern);
 
 
             String logfilePath = loggerDir.resolve(getFileBase() + ".log").toString();
             this.logfileLocation = logfilePath;
             String archivePath = loggerDir.resolve(getFileBase() + "-TIMESTAMP.log.gz").toString()
-                    .replaceAll("TIMESTAMP", "%d{MM-dd-yy}");
+                .replaceAll("TIMESTAMP", "%d{MM-dd-yy}");
 
-            ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
-                    .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
-                    .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "100M"));
+//            ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
+//                .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
+//                .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "100M"));
 
+//            AppenderComponentBuilder logsAppenderBuilder =
+//                builder.newAppender("SESSION_APPENDER", RollingFileAppender.PLUGIN_NAME)
+//                    .addAttribute("fileName", logfilePath)
+//                    .addAttribute("filePattern", archivePath)
+//                    .addAttribute("append", false)
+//                    .add(logfileLayout)
+//                    .addComponent(triggeringPolicy);
             AppenderComponentBuilder logsAppenderBuilder =
-                    builder.newAppender("SCENARIO_APPENDER", RollingFileAppender.PLUGIN_NAME)
-                            .addAttribute("fileName", logfilePath)
-                            .addAttribute("filePattern", archivePath)
-                            .addAttribute("append", false)
-                            .add(logfileLayout)
-                            .addComponent(triggeringPolicy);
+                builder.newAppender("SESSION_APPENDER", FileAppender.PLUGIN_NAME)
+                    .addAttribute("fileName", logfilePath)
+//                    .addAttribute("filePattern", archivePath)
+                    .addAttribute("append", false)
+                    .add(logfileLayout);
+
             builder.add(logsAppenderBuilder);
 
             if (isDedicatedVerificationLoggerEnabled) {
@@ -222,46 +238,89 @@ public class LoggerConfig extends ConfigurationFactory {
             }
             // TODO: build stop-words transcoder, add padding to end of alphabet, substitute stopwords
 
-            if (fileLevel.isInRange(Level.INFO,Level.TRACE)) {
+            if (fileLevel.isInRange(Level.INFO, Level.TRACE)) {
                 attachAuxLogger(builder, "RUNTIME", fileLevel);
             }
 
             rootBuilder.add(
-                    builder.newAppenderRef("SCENARIO_APPENDER")
-                            .addAttribute("level", fileLevel)
+                builder.newAppenderRef("SESSION_APPENDER")
+                    .addAttribute("level", fileLevel)
             );
         }
 
         rootBuilder.add(
-                builder.newAppenderRef("console")
-                        .addAttribute("level",
-                                consoleLevel
-                        )
+            builder.newAppenderRef("console")
+                .addAttribute("level",
+                    consoleLevel
+                )
         );
 
         builder.add(rootBuilder);
 
         BUILTIN_OVERRIDES.forEach((k, v) -> {
             builder.add(builder.newLogger(k, v)
-                    .add(builder.newAppenderRef("console"))
-                    .add(builder.newAppenderRef("SCENARIO_APPENDER"))
-                    .addAttribute("additivity", true));
+                .add(builder.newAppenderRef("console"))
+                .add(builder.newAppenderRef("SESSION_APPENDER"))
+                .addAttribute("additivity", true));
         });
 
         logLevelOverrides.forEach((k, v) -> {
             Level olevel = Level.valueOf(v);
             builder.add(builder.newLogger(k, olevel)
-                    .add(builder.newAppenderRef("console"))
-                    .add(builder.newAppenderRef("SCENARIO_APPENDER"))
-                    .addAttribute("additivity", true));
+                .add(builder.newAppenderRef("console"))
+                .add(builder.newAppenderRef(SESSION_APPENDER))
+                .addAttribute("additivity", true));
         });
 
         BuiltConfiguration builtConfig = builder.build();
+        updateLink(logfileLocation);
         return builtConfig;
     }
 
+    private void updateLink(String logfileLocation) {
+        Path logpath = Path.of(logfileLocation);
+        Path logdir = logpath.getParent();
+        if (Files.exists(logpath)) {
+            String basename = logpath.getFileName().toString();
+            String linkname = basename.replace(getSessionName() + "_", "");
+            Path linkPath = logdir.resolve(linkname);
+            try {
+                Files.deleteIfExists(linkPath);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            Path targetPath = Path.of(basename);
+
+            try {
+                Files.createSymbolicLink(
+                    linkPath,
+                    targetPath.getFileName()
+                );
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        } else {
+            System.out.println("Unable to find " + logfileLocation + " for symlink update");
+        }
+    }
+
+    private void attachFileRotationObserver(BuiltConfiguration builtConfig) {
+        Appender appender = builtConfig.getAppender(SESSION_APPENDER);
+        if (appender instanceof AbstractOutputStreamAppender<?> sa) {
+            OutputStreamManager manager = sa.getManager();
+
+        }
+
+    }
+
+    private final static Pattern validSessionName = Pattern.compile("^[a-zA-Z~][a-zA-Z0-9_~]*$");
+
     private String getFileBase() {
-        return getSessionName().replaceAll("\\s", "_");
+        Matcher matcher = validSessionName.matcher(getSessionName());
+        if (!matcher.matches()) {
+            throw new RuntimeException("Session name invalid. Must follow pattern " + validSessionName.pattern());
+        }
+        return getSessionName() + "_session";
     }
 
     private String getSessionName() {
@@ -288,7 +347,7 @@ public class LoggerConfig extends ConfigurationFactory {
         if (!Files.exists(loggerDir)) {
             try {
                 FileAttribute<Set<PosixFilePermission>> attrs = PosixFilePermissions.asFileAttribute(
-                        PosixFilePermissions.fromString("rwxrwx---")
+                    PosixFilePermissions.fromString("rwxrwx---")
                 );
                 Path directory = Files.createDirectory(loggerDir, attrs);
             } catch (Exception e) {
@@ -298,24 +357,24 @@ public class LoggerConfig extends ConfigurationFactory {
         ConfigurationFactory.setConfigurationFactory(this);
     }
 
-    public LoggerConfig setConsolePattern(String consoleLoggingPattern) {
+    public NBLoggerConfig setConsolePattern(String consoleLoggingPattern) {
 
         consoleLoggingPattern = (ansiEnabled && STANDARD_FORMATS.containsKey(consoleLoggingPattern + "-ANSI"))
-                ? consoleLoggingPattern + "-ANSI" : consoleLoggingPattern;
+            ? consoleLoggingPattern + "-ANSI" : consoleLoggingPattern;
 
         this.consolePattern = STANDARD_FORMATS.getOrDefault(consoleLoggingPattern, consoleLoggingPattern);
         return this;
     }
 
-    public LoggerConfig setLogfilePattern(String logfileLoggingPattern) {
+    public NBLoggerConfig setLogfilePattern(String logfileLoggingPattern) {
         logfileLoggingPattern = (logfileLoggingPattern.endsWith("-ANSI") && STANDARD_FORMATS.containsKey(logfileLoggingPattern))
-                ? logfileLoggingPattern.substring(logfileLoggingPattern.length() - 5) : logfileLoggingPattern;
+            ? logfileLoggingPattern.substring(logfileLoggingPattern.length() - 5) : logfileLoggingPattern;
 
         this.logfileLocation = STANDARD_FORMATS.getOrDefault(logfileLoggingPattern, logfileLoggingPattern);
         return this;
     }
 
-    public LoggerConfig setLoggerLevelOverrides(Map<String, String> logLevelOverrides) {
+    public NBLoggerConfig setLoggerLevelOverrides(Map<String, String> logLevelOverrides) {
         this.logLevelOverrides = logLevelOverrides;
         return this;
     }
@@ -324,12 +383,12 @@ public class LoggerConfig extends ConfigurationFactory {
         return logLevelOverrides;
     }
 
-    public LoggerConfig setSessionName(String sessionName) {
+    public NBLoggerConfig setSessionName(String sessionName) {
         this.sessionName = sessionName;
         return this;
     }
 
-    public LoggerConfig purgeOldFiles(Logger logger) {
+    public NBLoggerConfig purgeOldFiles(Logger logger) {
         if (maxLogfiles == 0) {
             logger.debug("Not purging old files, since maxLogFiles is 0.");
             return this;
@@ -354,9 +413,9 @@ public class LoggerConfig extends ConfigurationFactory {
         }
 
         List<File> toDelete = filesList.stream()
-                .sorted(fileTimeComparator)
-                .limit(remove)
-                .collect(Collectors.toList());
+            .sorted(fileTimeComparator)
+            .limit(remove)
+            .collect(Collectors.toList());
 
         for (File file : toDelete) {
             logger.info(() -> "removing extra logfile: " + file.getPath());
@@ -384,18 +443,20 @@ public class LoggerConfig extends ConfigurationFactory {
         return logfileLocation;
     }
 
-    public LoggerConfig setLogsDirectory(Path logsDirectory) {
+    public NBLoggerConfig setLogsDirectory(Path logsDirectory) {
         this.loggerDir = logsDirectory;
         return this;
     }
 
     private void attachAuxLogger(ConfigurationBuilder<BuiltConfiguration> builder, String loggerName, Level fileLevel) {
-        String appenderName = loggerName+(("_APPENDER").toUpperCase());
-        String fileName = loggerDir.resolve(getFileBase() + "_"+loggerName+".log").toString().toLowerCase();
+        String appenderName = loggerName + (("_APPENDER").toUpperCase());
+        Path logPath = loggerDir.resolve(getFileBase() + "_" + loggerName.toLowerCase() + ".log");
+        Path linkPath = loggerDir.resolve(loggerName.toLowerCase() + ".log");
+
         var appender = builder
             .newAppender(appenderName, FileAppender.PLUGIN_NAME)
             .addAttribute("append", false)
-            .addAttribute("fileName", fileName)
+            .addAttribute("fileName", logPath.toString())
             .add(builder
                 .newLayout("PatternLayout")
                 .addAttribute("pattern", "%d %p %C{1.} [%t] %m%n")
@@ -407,5 +468,13 @@ public class LoggerConfig extends ConfigurationFactory {
         builder.add(appender);
         builder.add(logger);
 
+        try {
+            Files.deleteIfExists(linkPath);
+            Files.createSymbolicLink(linkPath, logPath.getFileName());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+
     }
 }

engine-core/src/test/java/io/nosqlbench/engine/api/activityapi/ratelimits/SimRateSpecTest.java

@@ -16,7 +16,7 @@
 
 package io.nosqlbench.engine.api.activityapi.ratelimits;
 
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.SimRateSpec;
+import io.nosqlbench.engine.api.activityapi.simrate.SimRateSpec;
 import org.junit.jupiter.api.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;

engine-core/src/test/java/io/nosqlbench/engine/api/metrics/HistoIntervalLoggerTest.java

@@ -16,6 +16,7 @@
 
 package io.nosqlbench.engine.api.metrics;
 
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.labels.NBLabels;
 import io.nosqlbench.nb.api.engine.metrics.DeltaHdrHistogramReservoir;
 import io.nosqlbench.nb.api.engine.metrics.HistoIntervalLogger;
@@ -47,7 +48,14 @@ public class HistoIntervalLoggerTest {
         final int significantDigits = 4;
 
         NBMetricHistogram NBHistogram = new NBMetricHistogram(
-                NBLabels.forKV("name", "histo1"), new DeltaHdrHistogramReservoir(NBLabels.forKV("name", "histo1"), significantDigits));
+            NBLabels.forKV("name", "histo1"),
+            new DeltaHdrHistogramReservoir(
+                NBLabels.forKV("name", "histo1"),
+                significantDigits
+            ),
+            "test basic logger",
+            MetricCategory.Verification
+        );
 
         hil.onHistogramAdded("histo1", NBHistogram);
 

engine-core/src/test/java/io/nosqlbench/engine/api/metrics/NBMetricHistogramTest.java

@@ -16,6 +16,7 @@
 
 package io.nosqlbench.engine.api.metrics;
 
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
 import io.nosqlbench.nb.api.labels.NBLabels;
 import io.nosqlbench.nb.api.engine.metrics.ConvenientSnapshot;
 import io.nosqlbench.nb.api.engine.metrics.DeltaHdrHistogramReservoir;
@@ -28,8 +29,15 @@ public class NBMetricHistogramTest {
 
     @Test
     public void testNicerHistogramValues() {
-        NBMetricHistogram nh = new NBMetricHistogram(NBLabels.forKV("name","testhisto"), new DeltaHdrHistogramReservoir(
-                NBLabels.forKV("name", "testhisto"), 4));
+        NBMetricHistogram nh = new NBMetricHistogram(
+            NBLabels.forKV("name","testhisto"),
+            new DeltaHdrHistogramReservoir(
+                NBLabels.forKV("name", "testhisto"),
+                4
+            ),
+            "test nicer histogram values",
+            MetricCategory.Verification
+        );
         for (int i = 1; 100 >= i; i++) {
             nh.update(i);
         }

engine-core/src/test/java/io/nosqlbench/engine/core/lifecycle/session/CmdParserTest.java

@@ -19,6 +19,7 @@ package io.nosqlbench.engine.core.lifecycle.session;
 import io.nosqlbench.engine.cmdstream.Cmd;
 import io.nosqlbench.engine.cmdstream.CmdType;
 import io.nosqlbench.nb.api.errors.BasicError;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import java.util.LinkedList;
@@ -34,36 +35,37 @@ class CmdParserTest {
     public void testSingleCommand() {
         List<Cmd> cmds = CmdParser.parse("testcmd42");
         assertThat(cmds).hasSize(1);
-        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.indirect);
-        assertThat(cmds.get(0).getArgValue("_impl")).isEqualTo("testcmd42");
+        assertThat(cmds.getFirst().getCmdType()).isEqualTo(CmdType.indirect);
+        assertThat(cmds.getFirst().getArgValue("_impl")).isEqualTo("testcmd42");
     }
 
     @Test
     public void testSingleCommandWithArgs() {
         List<Cmd> cmds = CmdParser.parse("testcmd43 param1=value1");
         assertThat(cmds).hasSize(1);
-        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.indirect);
-        assertThat(cmds.get(0).getArgValue("_impl")).isEqualTo("testcmd43");
-        assertThat(cmds.get(0).getArgValue("param1")).isEqualTo("value1");
+        assertThat(cmds.getFirst().getCmdType()).isEqualTo(CmdType.indirect);
+        assertThat(cmds.getFirst().getArgValue("_impl")).isEqualTo("testcmd43");
+        assertThat(cmds.getFirst().getArgValue("param1")).isEqualTo("value1");
     }
 
     @Test
     public void testSingleDquotedArg() {
         List<Cmd> cmds = CmdParser.parse("testcmd44 param1=\"value1\"");
         assertThat(cmds).hasSize(1);
-        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.indirect);
-        assertThat(cmds.get(0).getArgValue("_impl")).isEqualTo("testcmd44");
-        assertThat(cmds.get(0).getArgValue("param1")).isEqualTo("value1");
+        assertThat(cmds.getFirst().getCmdType()).isEqualTo(CmdType.indirect);
+        assertThat(cmds.getFirst().getArgValue("_impl")).isEqualTo("testcmd44");
+        assertThat(cmds.getFirst().getArgValue("param1")).isEqualTo("value1");
     }
 
+    @Disabled
     @Test
     public void testSpecialSymbolValue() {
         List<Cmd> cmds = CmdParser.parse("start param1="+ CmdParser.SYMBOLS+ " param2='"+ CmdParser.SYMBOLS+ "' param3=\""+ CmdParser.SYMBOLS+ "\"");
         assertThat(cmds).hasSize(1);
-        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.start);
-        assertThat(cmds.get(0).getArgValue("param1")).isEqualTo(CmdParser.SYMBOLS);
-        assertThat(cmds.get(0).getArgValue("param2")).isEqualTo(CmdParser.SYMBOLS);
-        assertThat(cmds.get(0).getArgValue("param3")).isEqualTo(CmdParser.SYMBOLS);
+//        assertThat(cmds.getFirst().getCmdType()).isEqualTo(CmdType.start);
+        assertThat(cmds.getFirst().getArgValue("param1")).isEqualTo(CmdParser.SYMBOLS);
+        assertThat(cmds.getFirst().getArgValue("param2")).isEqualTo(CmdParser.SYMBOLS);
+        assertThat(cmds.getFirst().getArgValue("param3")).isEqualTo(CmdParser.SYMBOLS);
     }
 
     @Test
@@ -76,23 +78,24 @@ class CmdParserTest {
             "an error should be thrown if a named parameter is specified without a prior command.");
     }
 
+    @Disabled
     @Test
     public void testThatSymbolsAreQuotedInStringForm() {
         List<Cmd> cmds = CmdParser.parse("start param1=value1 param2='~should be quoted'");
         assertThat(cmds.size()).isEqualTo(1);
-        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.start);
-        assertThat(cmds.get(0).getArgValue("param1")).isEqualTo("value1");
-        assertThat(cmds.get(0).getArgValue("param2")).isEqualTo("~should be quoted");
-        assertThat(cmds.get(0).toString()).isEqualTo("start param1=value1 param2='~should be quoted'");
+//        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.start);
+        assertThat(cmds.getFirst().getArgValue("param1")).isEqualTo("value1");
+        assertThat(cmds.getFirst().getArgValue("param2")).isEqualTo("~should be quoted");
+        assertThat(cmds.getFirst().toString()).isEqualTo("start param1=value1 param2='~should be quoted'");
     }
 
     @Test
     public void testBasicArgvParser() {
         LinkedList<Cmd> cmds = CmdParser.parseArgvCommands(new LinkedList<>(List.of("_cmd4", "param1=value1")));
         assertThat(cmds.size()).isEqualTo(1);
-        assertThat(cmds.get(0).getCmdType()).isEqualTo(CmdType.indirect);
-        assertThat(cmds.get(0).getArgValue("_impl")).isEqualTo("_cmd4");
-        assertThat(cmds.get(0).getArgValue("param1")).isEqualTo("value1");
+        assertThat(cmds.getFirst().getCmdType()).isEqualTo(CmdType.indirect);
+        assertThat(cmds.getFirst().getArgValue("_impl")).isEqualTo("_cmd4");
+        assertThat(cmds.getFirst().getArgValue("param1")).isEqualTo("value1");
     }
 
 }

engine-core/src/test/java/io/nosqlbench/engine/core/lifecycle/session/NBSessionTest.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.core.lifecycle.session;
+
+import io.nosqlbench.engine.cmdstream.Cmd;
+import io.nosqlbench.engine.cmdstream.CmdArg;
+import io.nosqlbench.nb.api.labels.NBLabeledElement;
+import io.nosqlbench.nb.api.labels.NBLabels;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class NBSessionTest implements NBLabeledElement {
+
+    @Test
+    public void testThatSessionShortCircuitsCommandErrors() {
+        NBSession session = new NBSession(this, "session_name", Map.of());
+        Cmd c1 = new Cmd("test_ok", Map.of("test_ok", CmdArg.of("test_ok","key1","===","value1")));
+        Cmd c2 = new Cmd("test_error", Map.of());
+        Cmd c3 = new Cmd("test_ok", Map.of("test_ok", CmdArg.of("test_ok","key2","===","value2")));
+        List<Cmd> cmdStream = List.of(c1, c2, c3);
+
+        session.apply(cmdStream);
+    }
+
+    @Override
+    public NBLabels getLabels() {
+        return NBLabels.forKV("testing","session");
+    }
+}

engine-core/src/test/java/io/nosqlbench/engine/sandbox/SimRateSanityTest.java

@@ -18,8 +18,8 @@ package io.nosqlbench.engine.sandbox;
 
 import io.nosqlbench.nb.api.config.standard.TestComponent;
 import io.nosqlbench.nb.api.components.core.NBComponent;
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.SimRate;
-import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.SimRateSpec;
+import io.nosqlbench.engine.api.activityapi.simrate.SimRate;
+import io.nosqlbench.engine.api.activityapi.simrate.SimRateSpec;
 import org.junit.jupiter.api.Disabled;
 import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.runner.Runner;