Merge remote-tracking branch 'origin/main' into jk-test-eng-95-expected-result-verification

# Conflicts:
#	adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java
#	engine-api/src/test/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/NBErrorHandlerTest.java
#	engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java

nb-api/src/main/java/io/nosqlbench/api/config/LabeledScenarioContext.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.config;
+
+import javax.script.ScriptContext;
+
+public interface LabeledScenarioContext extends ScriptContext, NBLabeledElement {
+}

nb-api/src/main/java/io/nosqlbench/api/config/MapLabels.java

@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.config;
+
+import java.util.*;
+import java.util.function.Function;
+
+public class MapLabels implements NBLabels {
+    private final Map<String,String> labels;
+
+    public MapLabels(final Map<String, String> labels) {
+        this.labels = Collections.unmodifiableMap(labels);
+    }
+
+    public MapLabels(final Map<String,String> parentLabels, final Map<String,String> childLabels) {
+        final Map<String,String> combined = new LinkedHashMap<>();
+        parentLabels.forEach(combined::put);
+        childLabels.forEach((k,v) -> {
+            if (combined.containsKey(k))
+                throw new RuntimeException("Can't overlap label keys between parent and child elements. parent:" + parentLabels + ", child:" + childLabels);
+            combined.put(k,v);
+        });
+        labels=Collections.unmodifiableMap(combined);
+    }
+
+    @Override
+    public String linearizeValues(final char delim, final String... included) {
+        final StringBuilder sb = new StringBuilder();
+        final List<String> includedNames = new ArrayList<>();
+        if (0 < included.length) Collections.addAll(includedNames, included);
+        else this.labels.keySet().forEach(includedNames::add);
+
+        for (String includedName : includedNames) {
+            final boolean optional= includedName.startsWith("[") && includedName.endsWith("]");
+            includedName=optional?includedName.substring(1,includedName.length()-1):includedName;
+
+            final String component = this.labels.get(includedName);
+            if (null == component) {
+                if (optional) continue;
+                throw new RuntimeException("label component '" + includedName + "' was null.");
+            }
+            sb.append(component).append(delim);
+        }
+        sb.setLength(sb.length()-1);
+        return sb.toString();
+    }
+
+    @Override
+    public String linearize(String bareName, String... included) {
+        final StringBuilder sb = new StringBuilder();
+
+        final List<String> includedNames = new ArrayList<>();
+        if (0 < included.length) Collections.addAll(includedNames, included);
+        else this.labels.keySet().forEach(includedNames::add);
+        String rawName = null;
+        if (null != bareName) {
+            rawName = this.labels.get(bareName);
+            if (null == rawName) throw new RuntimeException("Unable to get value for key '" + bareName + '\'');
+            sb.append(rawName);
+        }
+        if (0 < includedNames.size()) {
+            sb.append('{');
+            for (final String includedName : includedNames) {
+                if (includedName.equals(bareName)) continue;
+                final String includedValue = this.labels.get(includedName);
+                Objects.requireNonNull(includedValue);
+                sb.append(includedName)
+                    .append("=\"")
+                    .append(includedValue)
+                    .append('"')
+                    .append(',');
+            }
+            sb.setLength(sb.length()-",".length());
+            sb.append('}');
+        }
+
+        return sb.toString();
+    }
+
+    @Override
+    public NBLabels and(final String... labelsAndValues) {
+        if (0 != (labelsAndValues.length % 2))
+            throw new RuntimeException("Must provide even number of keys and values: " + Arrays.toString(labelsAndValues));
+        final Map<String,String> childLabels = new LinkedHashMap<>();
+        for (int i = 0; i < labelsAndValues.length; i+=2) childLabels.put(labelsAndValues[i], labelsAndValues[i + 1]);
+        return new MapLabels(labels,childLabels);
+    }
+
+    @Override
+    public NBLabels modifyName(final String nameToModify, final Function<String, String> transform) {
+        if (!this.labels.containsKey(nameToModify))
+            throw new RuntimeException("Missing name in labels for transform: '" + nameToModify + '\'');
+        final LinkedHashMap<String, String> newLabels = new LinkedHashMap<>(this.labels);
+        final String removedValue = newLabels.remove(nameToModify);
+        final String newName = transform.apply(nameToModify);
+        newLabels.put(newName,removedValue);
+        return new MapLabels(newLabels);
+    }
+
+    @Override
+    public NBLabels modifyValue(final String labelName, final Function<String, String> transform) {
+        if(!this.labels.containsKey(labelName))
+            throw new RuntimeException("Unable to find label name '" + labelName + "' for value transform.");
+        final LinkedHashMap<String, String> newMap = new LinkedHashMap<>(this.labels);
+        final String value = newMap.remove(labelName);
+        if (null == value) throw new RuntimeException("The value for named label '" + labelName + "' is null.");
+        newMap.put(labelName,transform.apply(value));
+        return NBLabels.forMap(newMap);
+    }
+
+    public String toString() {
+        return this.linearize("name");
+    }
+
+    @Override
+    public String only(final String name) {
+        if (!this.labels.containsKey(name))
+            throw new RuntimeException("The specified key does not exist: '" + name + '\'');
+        final String only = labels.get(name);
+        if (null == only) throw new RuntimeException("The specified value is null for key '" + name + '\'');
+        return only;
+    }
+
+    @Override
+    public Map<String, String> asMap() {
+        return Collections.unmodifiableMap(labels);
+    }
+
+    @Override
+    public NBLabels and(final Map<String, String> moreLabels) {
+        return new MapLabels(this.labels, moreLabels);
+    }
+}

nb-api/src/main/java/io/nosqlbench/api/config/NBComponent.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.config;
+
+/**
+ * A Component is a functional element of the NoSQLBench runtime which is:
+ * <UL>
+ *     <LI>Contract Oriented - Components are based on well-defined interfaces.</LI>
+ *     <LI>Modular - Components are wired together by configuration.</LI>
+ *     <LI>Configurable - Components have configurations which are well defined and type safe.</LI>
+ *     <LI>User Facing - Components are top level constructs which users interact with.</LI>
+ *     <LI>Hierarchic - Components fit together in a runtime hierarchy. Only the ROOT component is allowed to have no parents.</LI>
+ *     <LI>Addressable - Each component has a set of metadata which allows it to be identified clearly under its parent.</LI>
+ * </UL>
+ *
+ * This interface will start as a tagging interface, but will eventually include aspects of above by extension.
+ */
+public interface NBComponent {
+}

nb-api/src/main/java/io/nosqlbench/api/config/NBLabeledElement.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2022-2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.config;
+
+import java.util.Map;
+
+public interface NBLabeledElement extends NBComponent {
+
+    NBLabeledElement EMPTY = forKV();
+
+    static NBLabeledElement forKV(final String... labelData) {
+        return new BasicLabeledElement(labelData);
+    }
+
+    static NBLabeledElement forMap(final Map<String, String> labels) {
+        return new BasicLabeledElement(labels);
+    }
+
+    NBLabels getLabels();
+
+    class BasicLabeledElement implements NBLabeledElement {
+        private final NBLabels labels;
+        public BasicLabeledElement(final String... kvs) {
+            labels=NBLabels.forKV(kvs);
+        }
+
+        public BasicLabeledElement(final Map<String, String> labels) {
+            this.labels = NBLabels.forMap(labels);
+        }
+
+        @Override
+        public NBLabels getLabels() {
+            return this.labels;
+        }
+    }
+
+//
+//    NBLabeledElement EMPTY = forMap(Map.of());
+//
+//    Map<String, String> getLabels();
+//
+//    /**
+//     * TODO: Should throw an error when new keys are duplicated
+//     * @param keyvalues
+//     * @return
+//     */
+//    default Map<String, String> getLabelsAnd(final String... keyvalues) {
+//        final LinkedHashMap<String, String> map = new LinkedHashMap<>(this.getLabels());
+//        for (int idx = 0; idx < keyvalues.length; idx+=2) map.put(keyvalues[idx], keyvalues[idx + 1]);
+//        return map;
+//    }
+//
+////    default NBLabeledElement and(String... keyvalues) {
+////
+////    }
+//
+//    default Map<String, String> getLabelsAnd(final Map<String,String> extra) {
+//        final LinkedHashMap<String,String> map = new LinkedHashMap<>(this.getLabels());
+//        map.putAll(extra);
+//        return map;
+//    }
+//
+//    static MapLabels forMap(final Map<String,String> labels) {
+//        return new MapLabels(labels);
+//    }
+//
+//    class MapLabels implements NBLabeledElement {
+//        private final Map<String, String> labels;
+//
+//        public MapLabels(final Map<String,String> labels) {
+//            this.labels = labels;
+//        }
+//
+//        @Override
+//        public Map<String, String> getLabels() {
+//            return this.labels;
+//        }
+//    }
+//
+//    /**
+//     * Create a single String representation of the label set, preserving key order,
+//     * with optional additional labels, in the form of:
+//     * <pre>{@code
+//     *  key1:value1,key2:value2,...
+//     * }</pre>
+//     * @param and
+//     * @return
+//     */
+//    default String linearizeLabels(final Map<String,String> and) {
+//        final StringBuilder sb= new StringBuilder();
+//        final Map<String, String> allLabels = getLabelsAnd(and);
+//        final ArrayList<String> sortedLabels = new ArrayList<>(allLabels.keySet());
+//        for (final String label : sortedLabels) sb.append(label).append(':').append(allLabels.get(label)).append(',');
+//        sb.setLength(sb.length()-",".length());
+//        return sb.toString();
+//    }
+//
+//    /**
+//     * Equivalent to {@link #linearizeLabels(Map)}, except that additional key-value pairs can
+//     * be expressed as a pairs of Strings in the argument list.
+//     * @param and - An even numbered list of strings as key1, value1, key2, value2, ...
+//     * @return A linearized string representation
+//     */
+//    default String linearizeLabels(final String... and) {
+//        return this.linearizeLabels(this.getLabelsAnd(and));
+//    }
+//
+//    default String linearizeLabelsByValueGraphite(final String... and) {
+//        return this.linearizeLabelsByValueDelim(".",and);
+//    }
+//
+}

nb-api/src/main/java/io/nosqlbench/api/config/NBLabels.java

@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.config;
+
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+
+/**
+ * <P>
+ * The NBLabels type represents sets of identifying label names and values for any element in the
+ * NoSQLBench runtime which needs to be named. This allows for hierarchic naming for instances by including
+ * the naming elements of parents as owned objects are created.
+ * </P>
+ *
+ * <p>
+ * The recommended way to use this type is to require a parent element in constructors, and to combine
+ * instance data at that time into a cached view. This means that further processing will be minimized,
+ * since these elements will be consulted frequently, such as when rendering metrics values.
+ * </P>
+ */
+public interface NBLabels {
+
+    /**
+     * Create a string representation of the label data, including only the values.
+     * Each value is concatenated with the others using the delim character.
+     * If a specified label name is included in square brackets like {@pre [name]}, then
+     * it is considered optional, and will be skipped over gracefully if it is not present
+     * in the label set. Otherwise all names are considered required.
+     *
+     * @param delim
+     *     A single character
+     * @param included
+     *     Which fields from the label set to include in the rendered string. If none are specified then all are
+     *     included.
+     * @return A string representation of the labels
+     * @throws RuntimeException if a required label name is not present, or its value is null.
+     */
+    String linearizeValues(char delim, String... included);
+
+
+    /**
+     * This is equivalent to call ing {@link #linearizeValues(char, String...)} with the '.' character.
+     *
+     * @param included
+     *     Which fields from the label set to include in the rendered string. If none are specified, then all are
+     *     included.
+     * @return A string representation of the labels
+     */
+    default String linearizeValues(final String... included) {
+        return this.linearizeValues('.', included);
+    }
+
+    /**
+     * Render a string representation of the label set according to the prometheus exposition naming format.
+     * This means that a label set which includes the JSON data:
+     * <pre>{@code
+     * {
+     *   "name": "fooname",
+     *   "label1": "label1value"
+     * }
+     * }</pre> would render to <pre>{@code fooname{label1="label1value"}}</pre> IF called as {@code linearize("name")}.
+     * <p>
+     * The included fields are added to the label set. If none are specified then all are included by default.
+     *
+     * @param barename
+     *     The field from the label set to use as the nominal <em>metric family name</em> part.
+     * @param included
+     *     Fields to be used in rendered label set.
+     * @return A string representation of the labels that is parsable in the prometheus exposition format.
+     */
+    String linearize(String barename, String... included);
+
+    /**
+     * Create an NBLabels instance from the given map.
+     *
+     * @param labels
+     *     label data
+     * @return a new NBLabels instance
+     */
+    static NBLabels forMap(final Map<String, String> labels) {
+        return new MapLabels(labels);
+    }
+
+    /**
+     * Create an NBLabels instance from the given keys and values (even,odd,...)
+     *
+     * @param keysAndValues
+     *     Keys and values such as "key1", "value1", "key2", "value2", ...
+     * @return a new NBLabels instance
+     */
+    static NBLabels forKV(final String... keysAndValues) {
+        if (0 != (keysAndValues.length % 2))
+            throw new RuntimeException("keys and values must be provided in pairs, not as: " + Arrays.toString(keysAndValues));
+        final LinkedHashMap<String, String> labels = new LinkedHashMap<>(keysAndValues.length >> 1);
+        for (int i = 0; i < keysAndValues.length; i += 2) labels.put(keysAndValues[i], keysAndValues[i + 1]);
+        return new MapLabels(labels);
+    }
+
+    /**
+     * Return a new NBLabels value with the specified key transformed according to the provided Lambda.
+     * The associated value is not modified.
+     *
+     * @param element
+     *     The key to modify
+     * @param transform
+     *     A Lambda which will modify the existing key name.
+     * @return A new NBLabels value, separate from the original
+     */
+    NBLabels modifyName(String element, Function<String, String> transform);
+
+    /**
+     * Return a new NBLabels value with the specified value transformed according to the provided Lambda.
+     * The associated key name is not modified.
+     * @param labelName The named label to modify
+     * @param transform A Lambda which will modify the existing value.
+     * @return A new NBLabels value, separate from the original
+     * @@throws RuntimeException if either the key is not found or the values is null.
+     */
+    NBLabels modifyValue(String labelName, Function<String,String> transform);
+
+    /**
+     * Create a new NBLabels value with the additional keys and values appended.
+     *
+     * @param labelsAndValues
+     *     Keys and values in "key1", "value1", "key2", "value2", ... form
+     * @return A new NBLabels instance
+     */
+    NBLabels and(String... labelsAndValues);
+
+    /**
+     * Create a new NBLabels value with the additional keys and values appended.
+     *
+     * @param labels
+     *     a map of keys and values
+     * @return A new NBLabels instance
+     */
+    NBLabels and(Map<String, String> labels);
+
+    /**
+     * Return the value of the specified label key.
+     *
+     * @param name
+     *     The label name
+     * @return The named label's value
+     * @throws RuntimeException
+     *     if the specified label does not exist in the set, or the value is null.
+     */
+    String only(String name);
+
+    /**
+     * Return a map representation of the label set, regardless of the underlying form.
+     *
+     * @return a {@link Map} of keys and values, in deterministic order
+     */
+    Map<String, String> asMap();
+}

nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java

@@ -43,7 +43,7 @@ public class ActivityDef implements NBNamedElement {
     public static final String DEFAULT_ATYPE = "stdout  ";
     public static final String DEFAULT_CYCLES = "0";
     public static final int DEFAULT_THREADS = 1;
-    private final static Logger logger = LogManager.getLogger(ActivityDef.class);
+    private static final Logger logger = LogManager.getLogger(ActivityDef.class);
     // an alias with which to control the activity while it is running
     private static final String FIELD_ALIAS = "alias";
     // a file or URL containing the activity: op templates, generator bindings, ...
@@ -52,8 +52,8 @@ public class ActivityDef implements NBNamedElement {
     private static final String FIELD_CYCLES = "cycles";
     // initial thread concurrency for this activity
     private static final String FIELD_THREADS = "threads";
-    private static final String[] field_list = new String[]{
-            FIELD_ALIAS, FIELD_ATYPE, FIELD_CYCLES, FIELD_THREADS
+    private static final String[] field_list = {
+        FIELD_ALIAS, FIELD_ATYPE, FIELD_CYCLES, FIELD_THREADS
     };
     // parameter map has its own internal atomic map
     private final ParameterMap parameterMap;
@@ -76,7 +76,7 @@ public class ActivityDef implements NBNamedElement {
         ActivityDef activityDef = new ActivityDef(activityParameterMap.orElseThrow(
                 () -> new RuntimeException("Unable to parse:" + namedActivitySpec)
         ));
-        logger.info("parsed activityDef " + namedActivitySpec + " to-> " + activityDef);
+        logger.info("parsed activityDef {} to-> {}", namedActivitySpec, activityDef);
 
         return activityDef;
     }
@@ -110,7 +110,7 @@ public class ActivityDef implements NBNamedElement {
         String cycles = parameterMap.getOptionalString("cycles").orElse(DEFAULT_CYCLES);
         int rangeAt = cycles.indexOf("..");
         String startCycle;
-        if (rangeAt > 0) {
+        if (0 < rangeAt) {
             startCycle = cycles.substring(0, rangeAt);
         } else {
             startCycle = "0";
@@ -122,7 +122,7 @@ public class ActivityDef implements NBNamedElement {
     }
 
     public void setStartCycle(long startCycle) {
-        parameterMap.set(FIELD_CYCLES, "" + startCycle + ".." + getEndCycle());
+        parameterMap.set(FIELD_CYCLES, startCycle + ".." + getEndCycle());
     }
 
     public void setStartCycle(String startCycle) {
@@ -146,7 +146,7 @@ public class ActivityDef implements NBNamedElement {
         String cycles = parameterMap.getOptionalString(FIELD_CYCLES).orElse(DEFAULT_CYCLES);
         int rangeAt = cycles.indexOf("..");
         String endCycle;
-        if (rangeAt > 0) {
+        if (0 < rangeAt) {
             endCycle = cycles.substring(rangeAt + 2);
         } else {
             endCycle = cycles;
@@ -157,7 +157,7 @@ public class ActivityDef implements NBNamedElement {
     }
 
     public void setEndCycle(long endCycle) {
-        parameterMap.set(FIELD_CYCLES, "" + getStartCycle() + ".." + endCycle);
+        parameterMap.set(FIELD_CYCLES, getStartCycle() + ".." + endCycle);
     }
 
     /**
@@ -201,12 +201,12 @@ public class ActivityDef implements NBNamedElement {
     }
 
     public long getCycleCount() {
-        return (getEndCycle() - getStartCycle());
+        return getEndCycle() - getStartCycle();
     }
 
     private void checkInvariants() {
         if (getStartCycle() >= getEndCycle()) {
-            throw new InvalidParameterException("Start cycle must be strictly less than end cycle, but they are [" + getStartCycle() + "," + getEndCycle() + ")");
+            throw new InvalidParameterException("Start cycle must be strictly less than end cycle, but they are [" + getStartCycle() + ',' + getEndCycle() + ')');
         }
     }
 
@@ -217,19 +217,19 @@ public class ActivityDef implements NBNamedElement {
 
     public ActivityDef deprecate(String deprecatedName, String newName) {
         Object deprecatedParam = this.parameterMap.get(deprecatedName);
-        if (deprecatedParam==null) {
+        if (null == deprecatedParam) {
             return this;
         }
         if (deprecatedParam instanceof CharSequence chars) {
             if (this.parameterMap.containsKey(newName)) {
-                throw new BasicError("You have specified activity param '" + deprecatedName + "' in addition to the valid name '" + newName +"'. Remove '" + deprecatedName + "'.");
-            } else {
-                logger.warn("Auto replacing deprecated activity param '" + deprecatedName + "="+ chars +"' with new '" + newName +"="+ chars +"'.");
-                parameterMap.put(newName,parameterMap.remove(deprecatedName));
+                throw new BasicError("You have specified activity param '" + deprecatedName + "' in addition to the valid name '" + newName + "'. Remove '" + deprecatedName + "'.");
             }
+            logger.warn("Auto replacing deprecated activity param '{}={}' with new '{}={}'.", deprecatedName, chars, newName, chars);
+            parameterMap.put(newName, parameterMap.remove(deprecatedName));
         } else {
             throw new BasicError("Can't replace deprecated name with value of type " + deprecatedName.getClass().getCanonicalName());
         }
         return this;
     }
+
 }

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,24 +17,27 @@
 package io.nosqlbench.api.engine.metrics;
 
 import com.codahale.metrics.*;
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
 import io.nosqlbench.api.config.NBNamedElement;
 import io.nosqlbench.api.engine.activityapi.core.MetricRegistryService;
+import io.nosqlbench.api.engine.metrics.instruments.*;
 import io.nosqlbench.api.engine.util.Unit;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import javax.script.ScriptContext;
 import java.io.File;
 import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.ServiceLoader;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 
 public class ActivityMetrics {
 
-    private final static Logger logger = LogManager.getLogger(ActivityMetrics.class);
+    private static final Logger logger = LogManager.getLogger(ActivityMetrics.class);
 
     public static final String HDRDIGITS_PARAM = "hdr_digits";
     public static final int DEFAULT_HDRDIGITS = 4;
@@ -56,48 +59,35 @@ public class ActivityMetrics {
         ActivityMetrics._HDRDIGITS = hdrDigits;
     }
 
-    private ActivityMetrics() {
-    }
-
     /**
      * Register a named metric for an activity, synchronized on the activity
      *
-     * @param named    The activity def that the metric will be for
-     * @param name           The full metric name
-     * @param metricProvider A function to actually create the metric if needed
+     * @param named
+     *     The activity def that the metric will be for
+     * @param metricFamilyName
+     *     The full metric name
+     * @param metricProvider
+     *     A function to actually create the metric if needed
      * @return a Metric, or null if the metric for the name was already present
      */
     @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
-    private static Metric register(NBNamedElement named, String name, MetricProvider metricProvider) {
-        String fullMetricName = named.getName() + "." + name;
-        Metric metric = get().getMetrics().get(fullMetricName);
-        if (metric == null) {
-            synchronized (named) {
-                metric = get().getMetrics().get(fullMetricName);
-                if (metric == null) {
-                    metric = metricProvider.getMetric();
-                    return get().register(fullMetricName, metric);
-                }
-            }
-        }
-        return metric;
-    }
+    private static Metric register(NBLabels labels, MetricProvider metricProvider) {
 
-    private static Metric register(ScriptContext context, String name, MetricProvider metricProvider) {
-        Metric metric = get().getMetrics().get(name);
-        if (metric == null) {
-            synchronized (context) {
-                metric = get().getMetrics().get(name);
-                if (metric == null) {
+        final String graphiteName = labels.linearizeValues('.',"[activity]","[space]","[op]","name");
+        Metric metric = get().getMetrics().get(graphiteName);
+
+        if (null == metric) {
+            synchronized (labels) {
+                metric = get().getMetrics().get(graphiteName);
+                if (null == metric) {
                     metric = metricProvider.getMetric();
-                    Metric registered = get().register(name, metric);
-                    logger.info(() -> "registered scripting metric: " + name);
+                    Metric registered = get().register(graphiteName, metric);
+                    logger.debug(() -> "registered metric: " + graphiteName);
                     return registered;
                 }
             }
         }
         return metric;
-
     }
 
     /**
@@ -109,33 +99,25 @@ public class ActivityMetrics {
      * <p>This method ensures that if multiple threads attempt to create the same-named metric on a given activity,
      * that only one of them succeeds.</p>
      *
-     * @param named an associated activity def
-     * @param name        a simple, descriptive name for the timer
+     * @param named
+     *     an associated activity def
+     * @param metricFamilyName
+     *     a simple, descriptive name for the timer
      * @return the timer, perhaps a different one if it has already been registered
      */
-    public static Timer timer(NBNamedElement named, String name, int hdrdigits) {
-        String fullMetricName = named.getName() + "." + name;
-        Timer registeredTimer = (Timer) register(named, name, () ->
-            new NicerTimer(fullMetricName,
+    public static Timer timer(NBLabeledElement parent, String metricFamilyName, int hdrdigits) {
+        final NBLabels labels = parent.getLabels().and("name",metricFamilyName);
+
+        Timer registeredTimer = (Timer) register(labels, () ->
+            new NBMetricTimer(labels,
                 new DeltaHdrHistogramReservoir(
-                    fullMetricName,
+                    labels,
                     hdrdigits
                 )
             ));
         return registeredTimer;
     }
 
-    public static Timer timer(String fullMetricName) {
-        NicerTimer timer = get().register(fullMetricName, new NicerTimer(
-            fullMetricName,
-            new DeltaHdrHistogramReservoir(
-                fullMetricName,
-                _HDRDIGITS
-            ))
-        );
-        return timer;
-    }
-
     /**
      * <p>Create an HDR histogram associated with an activity.</p>
      *
@@ -145,44 +127,38 @@ public class ActivityMetrics {
      * <p>This method ensures that if multiple threads attempt to create the same-named metric on a given activity,
      * that only one of them succeeds.</p>
      *
-     * @param named an associated activity def
-     * @param name        a simple, descriptive name for the histogram
+     * @param named
+     *     an associated activity def
+     * @param metricFamilyName
+     *     a simple, descriptive name for the histogram
      * @return the histogram, perhaps a different one if it has already been registered
      */
-    public static Histogram histogram(NBNamedElement named, String name, int hdrdigits) {
-        String fullMetricName = named.getName() + "." + name;
-        return (Histogram) register(named, name, () ->
-            new NicerHistogram(
-                fullMetricName,
+    public static Histogram histogram(NBLabeledElement labeled, String metricFamilyName, int hdrdigits) {
+        final NBLabels labels = labeled.getLabels().and("name", metricFamilyName);
+        return (Histogram) register(labels, () ->
+            new NBMetricHistogram(
+                labels,
                 new DeltaHdrHistogramReservoir(
-                    fullMetricName,
+                    labels,
                     hdrdigits
                 )
             ));
     }
 
-    public static Histogram histogram(String fullname) {
-        NicerHistogram histogram = get().register(fullname, new NicerHistogram(
-            fullname,
-            new DeltaHdrHistogramReservoir(
-                fullname,
-                _HDRDIGITS
-            )
-        ));
-        return histogram;
-    }
-
     /**
      * <p>Create a counter associated with an activity.</p>
      * <p>This method ensures that if multiple threads attempt to create the same-named metric on a given activity,
      * that only one of them succeeds.</p>
      *
-     * @param named an associated activity def
-     * @param name        a simple, descriptive name for the counter
+     * @param named
+     *     an associated activity def
+     * @param name
+     *     a simple, descriptive name for the counter
      * @return the counter, perhaps a different one if it has already been registered
      */
-    public static Counter counter(NBNamedElement named, String name) {
-        return (Counter) register(named, name, Counter::new);
+    public static Counter counter(NBLabeledElement parent, String metricFamilyName) {
+        final NBLabels labels = parent.getLabels().and("name",metricFamilyName);
+        return (Counter) register(labels, () -> new NBMetricCounter(labels));
     }
 
     /**
@@ -190,20 +166,23 @@ public class ActivityMetrics {
      * <p>This method ensures that if multiple threads attempt to create the same-named metric on a given activity,
      * that only one of them succeeds.</p>
      *
-     * @param named an associated activity def
-     * @param name        a simple, descriptive name for the meter
+     * @param named
+     *     an associated activity def
+     * @param metricFamilyName
+     *     a simple, descriptive name for the meter
      * @return the meter, perhaps a different one if it has already been registered
      */
-    public static Meter meter(NBNamedElement named, String name) {
-        return (Meter) register(named, name, Meter::new);
+    public static Meter meter(NBLabeledElement parent, String metricFamilyName) {
+        final NBLabels labels = parent.getLabels().and("name",metricFamilyName);
+        return (Meter) register(labels, () -> new NBMetricMeter(labels));
     }
 
     private static MetricRegistry get() {
-        if (registry != null) {
+        if (null != ActivityMetrics.registry) {
             return registry;
         }
         synchronized (ActivityMetrics.class) {
-            if (registry == null) {
+            if (null == ActivityMetrics.registry) {
                 registry = lookupRegistry();
             }
         }
@@ -211,29 +190,24 @@ public class ActivityMetrics {
     }
 
     @SuppressWarnings("unchecked")
-    public static <T> Gauge<T> gauge(NBNamedElement named, String name, Gauge<T> gauge) {
-        return (Gauge<T>) register(named, name, () -> gauge);
-    }
+    public static <T> Gauge<T> gauge(NBLabeledElement parent, String metricFamilyName, Gauge<T> gauge) {
+        final NBLabels labels = parent.getLabels().and("name",metricFamilyName);
 
-    @SuppressWarnings("unchecked")
-    public static <T> Gauge<T> gauge(ScriptContext scriptContext, String name, Gauge<T> gauge) {
-        return (Gauge<T>) register(scriptContext, name, () -> gauge);
+        return (Gauge<T>) register(labels, () -> new NBMetricGauge(labels,gauge));
     }
 
-
     private static MetricRegistry lookupRegistry() {
         ServiceLoader<MetricRegistryService> metricRegistryServices =
             ServiceLoader.load(MetricRegistryService.class);
         List<MetricRegistryService> mrss = new ArrayList<>();
         metricRegistryServices.iterator().forEachRemaining(mrss::add);
 
-        if (mrss.size() == 1) {
+        if (1 == mrss.size()) {
             return mrss.get(0).getMetricRegistry();
-        } else {
-            String infoMsg = "Unable to load a dynamic MetricRegistry via ServiceLoader, using the default.";
-            logger.info(infoMsg);
-            return new MetricRegistry();
         }
+        final String infoMsg = "Unable to load a dynamic MetricRegistry via ServiceLoader, using the default.";
+        logger.info(infoMsg);
+        return new MetricRegistry();
 
     }
 
@@ -245,10 +219,14 @@ public class ActivityMetrics {
     /**
      * Add a histogram interval logger to matching metrics in this JVM instance.
      *
-     * @param sessionName The name for the session to be annotated in the histogram log
-     * @param pattern     A regular expression pattern to filter out metric names for logging
-     * @param filename    A file to log the histogram data in
-     * @param interval    How many seconds to wait between writing each interval histogram
+     * @param sessionName
+     *     The name for the session to be annotated in the histogram log
+     * @param pattern
+     *     A regular expression pattern to filter out metric names for logging
+     * @param filename
+     *     A file to log the histogram data in
+     * @param interval
+     *     How many seconds to wait between writing each interval histogram
      */
     public static void addHistoLogger(String sessionName, String pattern, String filename, String interval) {
         if (filename.contains("_SESSION_")) {
@@ -256,7 +234,7 @@ public class ActivityMetrics {
         }
         Pattern compiledPattern = Pattern.compile(pattern);
         File logfile = new File(filename);
-        long intervalMillis = Unit.msFor(interval).orElseThrow(() -> new RuntimeException("Unable to parse interval spec:'" + interval + "'"));
+        long intervalMillis = Unit.msFor(interval).orElseThrow(() -> new RuntimeException("Unable to parse interval spec:'" + interval + '\''));
 
         HistoIntervalLogger histoIntervalLogger =
             new HistoIntervalLogger(sessionName, logfile, compiledPattern, intervalMillis);
@@ -268,10 +246,14 @@ public class ActivityMetrics {
     /**
      * Add a histogram stats logger to matching metrics in this JVM instance.
      *
-     * @param sessionName The name for the session to be annotated in the histogram log
-     * @param pattern     A regular expression pattern to filter out metric names for logging
-     * @param filename    A file to log the histogram data in
-     * @param interval    How many seconds to wait between writing each interval histogram
+     * @param sessionName
+     *     The name for the session to be annotated in the histogram log
+     * @param pattern
+     *     A regular expression pattern to filter out metric names for logging
+     * @param filename
+     *     A file to log the histogram data in
+     * @param interval
+     *     How many seconds to wait between writing each interval histogram
      */
     public static void addStatsLogger(String sessionName, String pattern, String filename, String interval) {
         if (filename.contains("_SESSION_")) {
@@ -279,7 +261,7 @@ public class ActivityMetrics {
         }
         Pattern compiledPattern = Pattern.compile(pattern);
         File logfile = new File(filename);
-        long intervalMillis = Unit.msFor(interval).orElseThrow(() -> new RuntimeException("Unable to parse interval spec:" + interval + "'"));
+        long intervalMillis = Unit.msFor(interval).orElseThrow(() -> new RuntimeException("Unable to parse interval spec:" + interval + '\''));
 
         HistoStatsLogger histoStatsLogger =
             new HistoStatsLogger(sessionName, logfile, compiledPattern, intervalMillis, TimeUnit.NANOSECONDS);
@@ -293,14 +275,18 @@ public class ActivityMetrics {
      * get a view to both the enhanced histogram implementation as well as the classic implementation in the
      * same scenario.
      *
-     * @param sessionName The name of the session to be annotated in the classic histogram
-     * @param pattern     A regular expression pattern to filter out metric names for inclusion
-     * @param prefix      The name prefix to add to the classic histograms so that they fit into the existing metrics namespace
-     * @param interval    How frequently to update the histogram
+     * @param sessionName
+     *     The name of the session to be annotated in the classic histogram
+     * @param pattern
+     *     A regular expression pattern to filter out metric names for inclusion
+     * @param prefix
+     *     The name prefix to add to the classic histograms so that they fit into the existing metrics namespace
+     * @param interval
+     *     How frequently to update the histogram
      */
     public static void addClassicHistos(String sessionName, String pattern, String prefix, String interval) {
         Pattern compiledPattern = Pattern.compile(pattern);
-        long intervalMillis = Unit.msFor(interval).orElseThrow(() -> new RuntimeException("Unable to parse interval spec:" + interval + "'"));
+        long intervalMillis = Unit.msFor(interval).orElseThrow(() -> new RuntimeException("Unable to parse interval spec:" + interval + '\''));
 
         ClassicHistoListener classicHistoListener =
             new ClassicHistoListener(get(), sessionName, prefix, compiledPattern, interval, TimeUnit.NANOSECONDS);
@@ -317,7 +303,8 @@ public class ActivityMetrics {
      * This should be called at the end of a process, so that open intervals can be finished, logs closed properly,
      * etc.
      *
-     * @param showChart whether to chart metrics on console
+     * @param showChart
+     *     whether to chart metrics on console
      */
     public static void closeMetrics(boolean showChart) {
         logger.trace("Closing all registered metrics closable objects.");
@@ -351,7 +338,7 @@ public class ActivityMetrics {
     }
 
     public static void removeActivityMetrics(NBNamedElement named) {
-        get().getMetrics().keySet().stream().filter(s -> s.startsWith(named.getName() + "."))
+        get().getMetrics().keySet().stream().filter(s -> s.startsWith(named.getName() + '.'))
             .forEach(get()::remove);
     }
 

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ConvenientSnapshot.java

@@ -22,12 +22,13 @@ import java.io.OutputStream;
 
 public class ConvenientSnapshot extends Snapshot {
 
+    // TODO - Determine if HistorgramSnapshot vs. Snapshot (codahale)
     private final double NS_PER_S = 1000000000.0D;
     private final double NS_PER_MS = 1000000.0D;
     private final double NS_PER_US = 1000.0D;
 
     private final Snapshot snapshot;
-    ConvenientSnapshot(Snapshot snapshot) {
+    public ConvenientSnapshot(Snapshot snapshot) {
         this.snapshot = snapshot;
     }
 

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/DeltaHdrHistogramReservoir.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,11 +18,13 @@ package io.nosqlbench.api.engine.metrics;
 
 import com.codahale.metrics.Reservoir;
 import com.codahale.metrics.Snapshot;
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
 import org.HdrHistogram.Histogram;
 import org.HdrHistogram.HistogramLogWriter;
 import org.HdrHistogram.Recorder;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * A custom wrapping of snapshotting logic on the HdrHistogram. This histogram will always report the last histogram
@@ -32,27 +34,26 @@ import org.apache.logging.log4j.LogManager;
  *
  * <p>This implementation also supports attaching a single log writer. If a log writer is attached, each
  * time an interval is snapshotted internally, the data will also be written to an hdr log via the writer.</p>
- *
  */
-public final class DeltaHdrHistogramReservoir implements Reservoir {
-    private final static Logger logger = LogManager.getLogger(DeltaHdrHistogramReservoir.class);
+public final class DeltaHdrHistogramReservoir implements Reservoir, NBLabeledElement {
+    private static final Logger logger = LogManager.getLogger(DeltaHdrHistogramReservoir.class);
 
     private final Recorder recorder;
     private Histogram lastHistogram;
 
     private Histogram intervalHistogram;
     private long intervalHistogramEndTime = System.currentTimeMillis();
-    private final String metricName;
+    private final NBLabels labels;
     private HistogramLogWriter writer;
 
     /**
      * Create a reservoir with a default recorder. This recorder should be suitable for most usage.
      *
-     * @param name the name to give to the reservoir, for logging purposes
+     * @param labels              the labels to give to the reservoir, for logging purposes
      * @param significantDigits how many significant digits to track in the reservoir
      */
-    public DeltaHdrHistogramReservoir(String name, int significantDigits) {
-        this.metricName = name;
+    public DeltaHdrHistogramReservoir(NBLabels labels, int significantDigits) {
+        this.labels = labels;
         this.recorder = new Recorder(significantDigits);
 
         /*
@@ -106,14 +107,14 @@ public final class DeltaHdrHistogramReservoir implements Reservoir {
         long intervalHistogramStartTime = intervalHistogramEndTime;
         intervalHistogramEndTime = System.currentTimeMillis();
 
-        intervalHistogram.setTag(metricName);
+        intervalHistogram.setTag(this.labels.linearizeValues("name"));
         intervalHistogram.setStartTimeStamp(intervalHistogramStartTime);
         intervalHistogram.setEndTimeStamp(intervalHistogramEndTime);
 
         lastHistogram = intervalHistogram.copy();
-        lastHistogram.setTag(metricName);
+        lastHistogram.setTag(this.labels.linearizeValues("name"));
 
-        if (writer!=null) {
+        if (null != this.writer) {
             writer.outputIntervalHistogram(lastHistogram);
         }
         return lastHistogram;
@@ -121,6 +122,7 @@ public final class DeltaHdrHistogramReservoir implements Reservoir {
 
     /**
      * Write the last results via the log writer.
+     *
      * @param writer the log writer to use
      */
     public void write(HistogramLogWriter writer) {
@@ -128,7 +130,7 @@ public final class DeltaHdrHistogramReservoir implements Reservoir {
     }
 
     public DeltaHdrHistogramReservoir copySettings() {
-        return new DeltaHdrHistogramReservoir(this.metricName, intervalHistogram.getNumberOfSignificantValueDigits());
+        return new DeltaHdrHistogramReservoir(this.labels, intervalHistogram.getNumberOfSignificantValueDigits());
     }
 
     public void attachLogWriter(HistogramLogWriter logWriter) {
@@ -138,4 +140,9 @@ public final class DeltaHdrHistogramReservoir implements Reservoir {
     public Histogram getLastHistogram() {
         return lastHistogram;
     }
+
+    @Override
+    public NBLabels getLabels() {
+        return this.labels;
+    }
 }

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/DeltaHistogramSnapshot.java

@@ -26,7 +26,7 @@ import java.io.PrintWriter;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-final class DeltaHistogramSnapshot extends Snapshot {
+public final class DeltaHistogramSnapshot extends Snapshot {
     private final Histogram histogram;
 
     DeltaHistogramSnapshot(Histogram histogram) {

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/NicerTimer.java

@@ -1,92 +0,0 @@
-/*
- * Copyright (c) 2022 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.nosqlbench.api.engine.metrics;
-
-import com.codahale.metrics.Timer;
-import org.HdrHistogram.Histogram;
-
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.TimeUnit;
-
-public class NicerTimer extends Timer implements DeltaSnapshotter, HdrDeltaHistogramAttachment, TimerAttachment {
-    private final String metricName;
-    private final DeltaHdrHistogramReservoir deltaHdrHistogramReservoir;
-    private long cacheExpiry = 0L;
-    private List<Timer> mirrors;
-
-    public NicerTimer(String metricName, DeltaHdrHistogramReservoir deltaHdrHistogramReservoir) {
-        super(deltaHdrHistogramReservoir);
-        this.metricName = metricName;
-        this.deltaHdrHistogramReservoir = deltaHdrHistogramReservoir;
-    }
-
-    @Override
-    public ConvenientSnapshot getSnapshot() {
-        if (System.currentTimeMillis() >= cacheExpiry) {
-            return new ConvenientSnapshot(deltaHdrHistogramReservoir.getSnapshot());
-        } else {
-            return new ConvenientSnapshot(deltaHdrHistogramReservoir.getLastSnapshot());
-        }
-    }
-
-    public DeltaSnapshotReader getDeltaReader() {
-        return new DeltaSnapshotReader(this);
-    }
-
-    @Override
-    public ConvenientSnapshot getDeltaSnapshot(long cacheTimeMillis) {
-        this.cacheExpiry = System.currentTimeMillis() + cacheTimeMillis;
-        return new ConvenientSnapshot(deltaHdrHistogramReservoir.getSnapshot());
-    }
-
-    @Override
-    public synchronized NicerTimer attachHdrDeltaHistogram() {
-        if (mirrors==null) {
-            mirrors = new CopyOnWriteArrayList<>();
-        }
-        DeltaHdrHistogramReservoir sameConfigReservoir = this.deltaHdrHistogramReservoir.copySettings();
-        NicerTimer mirror = new NicerTimer(this.metricName, sameConfigReservoir);
-        mirrors.add(mirror);
-        return mirror;
-    }
-    @Override
-    public Timer attachTimer(Timer timer) {
-        if (mirrors==null) {
-            mirrors = new CopyOnWriteArrayList<>();
-        }
-        mirrors.add(timer);
-        return timer;
-    }
-
-
-    @Override
-    public Histogram getNextHdrDeltaHistogram() {
-        return this.deltaHdrHistogramReservoir.getNextHdrHistogram();
-    }
-
-    @Override
-    public void update(long duration, TimeUnit unit) {
-        super.update(duration, unit);
-        if (mirrors!=null) {
-            for (Timer mirror : mirrors) {
-                mirror.update(duration,unit);
-            }
-        }
-    }
-
-}

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/TimerAttachment.java

@@ -18,6 +18,7 @@ package io.nosqlbench.api.engine.metrics;
 
 import com.codahale.metrics.Timer;
 
+
 public interface TimerAttachment {
     Timer attachTimer(Timer timer);
 }

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/instruments/NBMetricCounter.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.engine.metrics.instruments;
+
+import com.codahale.metrics.Counter;
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
+
+public class NBMetricCounter extends Counter implements NBLabeledElement {
+
+    private final NBLabels labels;
+
+    public NBMetricCounter(final NBLabels labels) {
+        this.labels = labels;
+    }
+
+    @Override
+    public NBLabels getLabels() {
+        return labels;
+    }
+}

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/instruments/NBMetricGauge.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.engine.metrics.instruments;
+
+import com.codahale.metrics.Gauge;
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
+
+public class NBMetricGauge<T> implements Gauge<T>, NBLabeledElement {
+
+    private final Gauge<? extends T> gauge;
+    private final NBLabels labels;
+
+    public NBMetricGauge(NBLabels labels, Gauge<? extends T> gauge) {
+        this.gauge = gauge;
+        this.labels = labels;
+    }
+
+    @Override
+    public T getValue() {
+        return gauge.getValue();
+    }
+
+    @Override
+    public NBLabels getLabels() {
+        return labels;
+    }
+}

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/instruments/NBMetricHistogram.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,25 +14,34 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.api.engine.metrics;
+package io.nosqlbench.api.engine.metrics.instruments;
 
 import com.codahale.metrics.Histogram;
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
+import io.nosqlbench.api.engine.metrics.*;
 
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 
-public class NicerHistogram extends Histogram implements DeltaSnapshotter, HdrDeltaHistogramAttachment, HistogramAttachment {
+public class NBMetricHistogram extends Histogram implements DeltaSnapshotter, HdrDeltaHistogramAttachment, HistogramAttachment, NBLabeledElement {
 
     private final DeltaHdrHistogramReservoir hdrDeltaReservoir;
-    private long cacheExpiryMillis = 0L;
-    private long cacheTimeMillis = 0L;
-    private final String metricName;
+    private final NBLabels labels;
+    private long cacheExpiryMillis;
+    private long cacheTimeMillis;
     private List<Histogram> mirrors;
 
-    public NicerHistogram(String metricName, DeltaHdrHistogramReservoir hdrHistogramReservoir) {
+    public NBMetricHistogram(NBLabels labels, DeltaHdrHistogramReservoir hdrHistogramReservoir) {
         super(hdrHistogramReservoir);
-        this.metricName = metricName;
+        this.labels = labels;
+        this.hdrDeltaReservoir = hdrHistogramReservoir;
+    }
+
+    public NBMetricHistogram(String name, DeltaHdrHistogramReservoir hdrHistogramReservoir) {
+        super(hdrHistogramReservoir);
+        this.labels = NBLabels.forKV("name",name);
         this.hdrDeltaReservoir = hdrHistogramReservoir;
     }
 
@@ -50,11 +59,11 @@ public class NicerHistogram extends Histogram implements DeltaSnapshotter, HdrDe
     public ConvenientSnapshot getSnapshot() {
         if (System.currentTimeMillis() < cacheExpiryMillis) {
             return new ConvenientSnapshot(hdrDeltaReservoir.getLastSnapshot());
-        } else {
-            return new ConvenientSnapshot(hdrDeltaReservoir.getSnapshot());
         }
+        return new ConvenientSnapshot(hdrDeltaReservoir.getSnapshot());
     }
 
+    @Override
     public ConvenientSnapshot getDeltaSnapshot(long cacheTimeMillis) {
         this.cacheTimeMillis = cacheTimeMillis;
         cacheExpiryMillis = System.currentTimeMillis() + this.cacheTimeMillis;
@@ -63,19 +72,19 @@ public class NicerHistogram extends Histogram implements DeltaSnapshotter, HdrDe
     }
 
     @Override
-    public synchronized NicerHistogram attachHdrDeltaHistogram() {
-        if (mirrors == null) {
+    public synchronized NBMetricHistogram attachHdrDeltaHistogram() {
+        if (null == this.mirrors) {
             mirrors = new CopyOnWriteArrayList<>();
         }
         DeltaHdrHistogramReservoir mirrorReservoir = this.hdrDeltaReservoir.copySettings();
-        NicerHistogram mirror = new NicerHistogram("mirror-" + this.metricName, mirrorReservoir);
+        NBMetricHistogram mirror = new NBMetricHistogram("mirror-" + this.labels.linearizeValues("name"), mirrorReservoir);
         mirrors.add(mirror);
         return mirror;
     }
 
     @Override
     public Histogram attachHistogram(Histogram histogram) {
-        if (mirrors == null) {
+        if (null == this.mirrors) {
             mirrors = new CopyOnWriteArrayList<>();
         }
         mirrors.add(histogram);
@@ -85,7 +94,7 @@ public class NicerHistogram extends Histogram implements DeltaSnapshotter, HdrDe
     @Override
     public void update(long value) {
         super.update(value);
-        if (mirrors != null) {
+        if (null != this.mirrors) {
             for (Histogram mirror : mirrors) {
                 mirror.update(value);
             }
@@ -97,4 +106,8 @@ public class NicerHistogram extends Histogram implements DeltaSnapshotter, HdrDe
         return hdrDeltaReservoir.getNextHdrHistogram();
     }
 
+    @Override
+    public NBLabels getLabels() {
+        return this.labels;
+    }
 }

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/instruments/NBMetricMeter.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.engine.metrics.instruments;
+
+import com.codahale.metrics.Meter;
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
+
+public class NBMetricMeter extends Meter implements NBLabeledElement {
+
+    private final NBLabels labels;
+
+    public NBMetricMeter(NBLabels labels) {
+        this.labels = labels;
+    }
+
+    @Override
+    public NBLabels getLabels() {
+        return labels;
+    }
+}

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/instruments/NBMetricTimer.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2022-2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.engine.metrics.instruments;
+
+import com.codahale.metrics.Timer;
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
+import io.nosqlbench.api.engine.metrics.*;
+import org.HdrHistogram.Histogram;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+
+public class NBMetricTimer extends Timer implements DeltaSnapshotter, HdrDeltaHistogramAttachment, TimerAttachment, NBLabeledElement {
+    private final DeltaHdrHistogramReservoir deltaHdrHistogramReservoir;
+    private long cacheExpiry;
+    private List<Timer> mirrors;
+    private final NBLabels labels;
+
+    public NBMetricTimer(final NBLabels labels, final DeltaHdrHistogramReservoir deltaHdrHistogramReservoir) {
+        super(deltaHdrHistogramReservoir);
+        this.labels = labels;
+        this.deltaHdrHistogramReservoir = deltaHdrHistogramReservoir;
+    }
+
+    @Override
+    public ConvenientSnapshot getSnapshot() {
+        if (System.currentTimeMillis() >= this.cacheExpiry)
+            return new ConvenientSnapshot(this.deltaHdrHistogramReservoir.getSnapshot());
+        return new ConvenientSnapshot(this.deltaHdrHistogramReservoir.getLastSnapshot());
+    }
+
+    @Override
+    public DeltaSnapshotReader getDeltaReader() {
+        return new DeltaSnapshotReader(this);
+    }
+
+    @Override
+    public ConvenientSnapshot getDeltaSnapshot(final long cacheTimeMillis) {
+        cacheExpiry = System.currentTimeMillis() + cacheTimeMillis;
+        return new ConvenientSnapshot(this.deltaHdrHistogramReservoir.getSnapshot());
+    }
+
+    @Override
+    public synchronized NBMetricTimer attachHdrDeltaHistogram() {
+        if (null == mirrors) this.mirrors = new CopyOnWriteArrayList<>();
+        final DeltaHdrHistogramReservoir sameConfigReservoir = deltaHdrHistogramReservoir.copySettings();
+        final NBMetricTimer mirror = new NBMetricTimer(labels, sameConfigReservoir);
+        this.mirrors.add(mirror);
+        return mirror;
+    }
+    @Override
+    public Timer attachTimer(final Timer timer) {
+        if (null == mirrors) this.mirrors = new CopyOnWriteArrayList<>();
+        this.mirrors.add(timer);
+        return timer;
+    }
+
+
+    @Override
+    public Histogram getNextHdrDeltaHistogram() {
+        return deltaHdrHistogramReservoir.getNextHdrHistogram();
+    }
+
+    @Override
+    public void update(final long duration, final TimeUnit unit) {
+        super.update(duration, unit);
+        if (null != mirrors) for (final Timer mirror : this.mirrors) mirror.update(duration, unit);
+    }
+
+    @Override
+    public NBLabels getLabels() {
+        return labels;
+    }
+}

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/reporters/Log4JMetricsReporter.java

@@ -0,0 +1,429 @@
+/*
+ * Copyright (c) 2022-2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.engine.metrics.reporters;
+
+import com.codahale.metrics.*;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.Marker;
+
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This is a Log4J targeted metrics logging reporter, derived from
+ * {@link Slf4jReporter}. This implementation
+ * was built to allow for consolidating internal logging dependencies
+ * to log4j only.
+ */
+public class Log4JMetricsReporter extends ScheduledReporter {
+    /**
+     * Returns a new {@link Builder} for .
+     *
+     * @param registry the registry to report
+     * @return a {@link Builder} instance for a
+     */
+    public static Builder forRegistry(final MetricRegistry registry) {
+        return new Builder(registry);
+    }
+
+    public enum LoggingLevel { TRACE, DEBUG, INFO, WARN, ERROR }
+
+    /**
+     * A builder for {@link Log4JMetricsReporter} instances. Defaults to logging to {@code metrics}, not
+     * using a marker, converting rates to events/second, converting durations to milliseconds, and
+     * not filtering metrics.
+     */
+    public static class Builder {
+        private final MetricRegistry registry;
+        private Logger logger;
+        private LoggingLevel loggingLevel;
+        private Marker marker;
+        private String prefix;
+        private TimeUnit rateUnit;
+        private TimeUnit durationUnit;
+        private MetricFilter filter;
+        private ScheduledExecutorService executor;
+        private boolean shutdownExecutorOnStop;
+
+        private Builder(final MetricRegistry registry) {
+            this.registry = registry;
+            logger = LogManager.getLogger("metrics");
+            marker = null;
+            prefix = "";
+            rateUnit = TimeUnit.SECONDS;
+            durationUnit = TimeUnit.MILLISECONDS;
+            filter = MetricFilter.ALL;
+            loggingLevel = LoggingLevel.INFO;
+            executor = null;
+            shutdownExecutorOnStop = true;
+        }
+
+        /**
+         * Specifies whether or not, the executor (used for reporting) will be stopped with same time with reporter.
+         * Default value is true.
+         * Setting this parameter to false, has the sense in combining with providing external managed executor via {@link #scheduleOn(ScheduledExecutorService)}.
+         *
+         * @param shutdownExecutorOnStop if true, then executor will be stopped in same time with this reporter
+         * @return {@code this}
+         */
+        public Builder shutdownExecutorOnStop(final boolean shutdownExecutorOnStop) {
+            this.shutdownExecutorOnStop = shutdownExecutorOnStop;
+            return this;
+        }
+
+        /**
+         * Specifies the executor to use while scheduling reporting of metrics.
+         * Default value is null.
+         * Null value leads to executor will be auto created on start.
+         *
+         * @param executor the executor to use while scheduling reporting of metrics.
+         * @return {@code this}
+         */
+        public Builder scheduleOn(final ScheduledExecutorService executor) {
+            this.executor = executor;
+            return this;
+        }
+
+        /**
+         * Log metrics to the given logger.
+         *
+         * @param logger an SLF4J {@link Logger}
+         * @return {@code this}
+         */
+        public Builder outputTo(final Logger logger) {
+            this.logger = logger;
+            return this;
+        }
+
+        /**
+         * Mark all logged metrics with the given marker.
+         *
+         * @param marker an SLF4J {@link Marker}
+         * @return {@code this}
+         */
+        public Builder markWith(final Marker marker) {
+            this.marker = marker;
+            return this;
+        }
+
+        /**
+         * Prefix all metric names with the given string.
+         *
+         * @param prefix the prefix for all metric names
+         * @return {@code this}
+         */
+        public Builder prefixedWith(final String prefix) {
+            this.prefix = prefix;
+            return this;
+        }
+
+        /**
+         * Convert rates to the given time unit.
+         *
+         * @param rateUnit a unit of time
+         * @return {@code this}
+         */
+        public Builder convertRatesTo(final TimeUnit rateUnit) {
+            this.rateUnit = rateUnit;
+            return this;
+        }
+
+        /**
+         * Convert durations to the given time unit.
+         *
+         * @param durationUnit a unit of time
+         * @return {@code this}
+         */
+        public Builder convertDurationsTo(final TimeUnit durationUnit) {
+            this.durationUnit = durationUnit;
+            return this;
+        }
+
+        /**
+         * Only report metrics which match the given filter.
+         *
+         * @param filter a {@link MetricFilter}
+         * @return {@code this}
+         */
+        public Builder filter(final MetricFilter filter) {
+            this.filter = filter;
+            return this;
+        }
+
+        /**
+         * Use Logging Level when reporting.
+         *
+         * @param loggingLevel a (@link Slf4jReporter.LoggingLevel}
+         * @return {@code this}
+         */
+        public Builder withLoggingLevel(final LoggingLevel loggingLevel) {
+            this.loggingLevel = loggingLevel;
+            return this;
+        }
+
+        /**
+         * Builds a {@link Log4JMetricsReporter} with the given properties.
+         *
+         * @return a {@link Log4JMetricsReporter}
+         */
+        public Log4JMetricsReporter build() {
+            final LoggerProxy loggerProxy;
+            switch (this.loggingLevel) {
+                case TRACE:
+                    loggerProxy = new TraceLoggerProxy(this.logger);
+                    break;
+                case INFO:
+                    loggerProxy = new InfoLoggerProxy(this.logger);
+                    break;
+                case WARN:
+                    loggerProxy = new WarnLoggerProxy(this.logger);
+                    break;
+                case ERROR:
+                    loggerProxy = new ErrorLoggerProxy(this.logger);
+                    break;
+                default:
+                case DEBUG:
+                    loggerProxy = new DebugLoggerProxy(this.logger);
+                    break;
+            }
+            return new Log4JMetricsReporter(this.registry, loggerProxy, this.marker, this.prefix, this.rateUnit, this.durationUnit, this.filter, this.executor, this.shutdownExecutorOnStop);
+        }
+    }
+
+    private final LoggerProxy loggerProxy;
+    private final Marker marker;
+    private final String prefix;
+
+    private Log4JMetricsReporter(final MetricRegistry registry,
+                                 final LoggerProxy loggerProxy,
+                                 final Marker marker,
+                                 final String prefix,
+                                 final TimeUnit rateUnit,
+                                 final TimeUnit durationUnit,
+                                 final MetricFilter filter,
+                                 final ScheduledExecutorService executor,
+                                 final boolean shutdownExecutorOnStop) {
+        super(registry, "logger-reporter", filter, rateUnit, durationUnit, executor, shutdownExecutorOnStop);
+        this.loggerProxy = loggerProxy;
+        this.marker = marker;
+        this.prefix = prefix;
+    }
+
+    @Override
+    @SuppressWarnings("rawtypes")
+    public void report(final SortedMap<String, Gauge> gauges,
+                       final SortedMap<String, Counter> counters,
+                       final SortedMap<String, Histogram> histograms,
+                       final SortedMap<String, Meter> meters,
+                       final SortedMap<String, Timer> timers) {
+        if (this.loggerProxy.isEnabled(this.marker)) {
+            for (final Map.Entry<String, Gauge> entry : gauges.entrySet())
+                this.logGauge(entry.getKey(), entry.getValue());
+
+            for (final Map.Entry<String, Counter> entry : counters.entrySet())
+                this.logCounter(entry.getKey(), entry.getValue());
+
+            for (final Map.Entry<String, Histogram> entry : histograms.entrySet())
+                this.logHistogram(entry.getKey(), entry.getValue());
+
+            for (final Map.Entry<String, Meter> entry : meters.entrySet())
+                this.logMeter(entry.getKey(), entry.getValue());
+
+            for (final Map.Entry<String, Timer> entry : timers.entrySet())
+                this.logTimer(entry.getKey(), entry.getValue());
+        }
+    }
+
+    private void logTimer(final String name, final Timer timer) {
+        Snapshot snapshot = timer.getSnapshot();
+        this.loggerProxy.log(this.marker,
+                "type={}, name={}, count={}, min={}, max={}, mean={}, stddev={}, median={}, " +
+                        "p75={}, p95={}, p98={}, p99={}, p999={}, mean_rate={}, m1={}, m5={}, " +
+                        "m15={}, rate_unit={}, duration_unit={}",
+                "TIMER",
+            this.prefix(name),
+                timer.getCount(),
+            this.convertDuration(snapshot.getMin()),
+            this.convertDuration(snapshot.getMax()),
+            this.convertDuration(snapshot.getMean()),
+            this.convertDuration(snapshot.getStdDev()),
+            this.convertDuration(snapshot.getMedian()),
+            this.convertDuration(snapshot.get75thPercentile()),
+            this.convertDuration(snapshot.get95thPercentile()),
+            this.convertDuration(snapshot.get98thPercentile()),
+            this.convertDuration(snapshot.get99thPercentile()),
+            this.convertDuration(snapshot.get999thPercentile()),
+            this.convertRate(timer.getMeanRate()),
+            this.convertRate(timer.getOneMinuteRate()),
+            this.convertRate(timer.getFiveMinuteRate()),
+            this.convertRate(timer.getFifteenMinuteRate()),
+            this.getRateUnit(),
+            this.getDurationUnit());
+    }
+
+    private void logMeter(final String name, final Meter meter) {
+        this.loggerProxy.log(this.marker,
+                "type={}, name={}, count={}, mean_rate={}, m1={}, m5={}, m15={}, rate_unit={}",
+                "METER",
+            this.prefix(name),
+                meter.getCount(),
+            this.convertRate(meter.getMeanRate()),
+            this.convertRate(meter.getOneMinuteRate()),
+            this.convertRate(meter.getFiveMinuteRate()),
+            this.convertRate(meter.getFifteenMinuteRate()),
+            this.getRateUnit());
+    }
+
+    private void logHistogram(final String name, final Histogram histogram) {
+        Snapshot snapshot = histogram.getSnapshot();
+        this.loggerProxy.log(this.marker,
+                "type={}, name={}, count={}, min={}, max={}, mean={}, stddev={}, " +
+                        "median={}, p75={}, p95={}, p98={}, p99={}, p999={}",
+                "HISTOGRAM",
+            this.prefix(name),
+                histogram.getCount(),
+                snapshot.getMin(),
+                snapshot.getMax(),
+                snapshot.getMean(),
+                snapshot.getStdDev(),
+                snapshot.getMedian(),
+                snapshot.get75thPercentile(),
+                snapshot.get95thPercentile(),
+                snapshot.get98thPercentile(),
+                snapshot.get99thPercentile(),
+                snapshot.get999thPercentile());
+    }
+
+    private void logCounter(final String name, final Counter counter) {
+        this.loggerProxy.log(this.marker, "type={}, name={}, count={}", "COUNTER", this.prefix(name), counter.getCount());
+    }
+
+    private void logGauge(final String name, final Gauge<?> gauge) {
+        this.loggerProxy.log(this.marker, "type={}, name={}, value={}", "GAUGE", this.prefix(name), gauge.getValue());
+    }
+
+    @Override
+    protected String getRateUnit() {
+        return "events/" + super.getRateUnit();
+    }
+
+    private String prefix(final String... components) {
+        return MetricRegistry.name(this.prefix, components);
+    }
+
+    /* private class to allow logger configuration */
+    abstract static class LoggerProxy {
+        protected final Logger logger;
+
+        protected LoggerProxy(final Logger logger) {
+            this.logger = logger;
+        }
+
+        abstract void log(Marker marker, String format, Object... arguments);
+
+        abstract boolean isEnabled(Marker marker);
+    }
+
+    /* private class to allow logger configuration */
+    private static class DebugLoggerProxy extends LoggerProxy {
+        public DebugLoggerProxy(final Logger logger) {
+            super(logger);
+        }
+
+        @Override
+        public void log(final Marker marker, final String format, final Object... arguments) {
+            this.logger.debug(marker, format, arguments);
+        }
+
+        @Override
+        public boolean isEnabled(final Marker marker) {
+            return this.logger.isDebugEnabled(marker);
+        }
+    }
+
+    /* private class to allow logger configuration */
+    private static class TraceLoggerProxy extends LoggerProxy {
+        public TraceLoggerProxy(final Logger logger) {
+            super(logger);
+        }
+
+        @Override
+        public void log(final Marker marker, final String format, final Object... arguments) {
+            this.logger.trace(marker, format, arguments);
+        }
+
+        @Override
+        public boolean isEnabled(final Marker marker) {
+            return this.logger.isTraceEnabled(marker);
+        }
+    }
+
+    /* private class to allow logger configuration */
+    private static class InfoLoggerProxy extends LoggerProxy {
+        public InfoLoggerProxy(final Logger logger) {
+            super(logger);
+        }
+
+        @Override
+        public void log(final Marker marker, final String format, final Object... arguments) {
+            this.logger.info(marker, format, arguments);
+        }
+
+        @Override
+        public boolean isEnabled(final Marker marker) {
+            return this.logger.isInfoEnabled(marker);
+        }
+    }
+
+    /* private class to allow logger configuration */
+    private static class WarnLoggerProxy extends LoggerProxy {
+        public WarnLoggerProxy(final Logger logger) {
+            super(logger);
+        }
+
+        @Override
+        public void log(final Marker marker, final String format, final Object... arguments) {
+            this.logger.warn(marker, format, arguments);
+        }
+
+        @Override
+        public boolean isEnabled(final Marker marker) {
+            return this.logger.isWarnEnabled(marker);
+        }
+    }
+
+    /* private class to allow logger configuration */
+    private static class ErrorLoggerProxy extends LoggerProxy {
+        public ErrorLoggerProxy(final Logger logger) {
+            super(logger);
+        }
+
+        @Override
+        public void log(final Marker marker, final String format, final Object... arguments) {
+            this.logger.error(marker, format, arguments);
+        }
+
+        @Override
+        public boolean isEnabled(final Marker marker) {
+            return this.logger.isErrorEnabled(marker);
+        }
+    }
+
+}

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/reporters/PromExpositionFormat.java

@@ -0,0 +1,226 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.engine.metrics.reporters;
+
+import com.codahale.metrics.*;
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
+import io.nosqlbench.api.testutils.Perf;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * Format NBMetrics according to the prometheus exposition format.
+ *
+ * @see <a
+ *     href="https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md">prometheus
+ *     exposition format</a>
+ */
+
+public enum PromExpositionFormat {
+;
+    public static String format(final Clock clock, final Metric... metrics) {
+        return PromExpositionFormat.format(clock, new StringBuilder(), metrics).toString();
+    }
+
+    /**
+     * @param clock
+     *     The clock to use for assigning an observation time to each metric value.
+     * @param builder
+     *     A string builder to append to
+     * @param metrics
+     *     zero or more metric which need to be formatted
+     * @return A string representation of the metrics in prometheus exposition format
+     */
+    public static StringBuilder format(final Clock clock, final StringBuilder builder, final Object... metrics) {
+        final StringBuilder buffer = (null != builder) ? builder : new StringBuilder();
+        final Instant instant = clock.instant();
+
+        for (final Object metric : metrics) {
+            NBLabels labels = null;
+
+            if (metric instanceof final NBLabeledElement labeled) labels = labeled.getLabels();
+            else throw new RuntimeException(
+                "Unknown label set for metric type '" + metric.getClass().getCanonicalName() + '\''
+            );
+            final long epochMillis = instant.toEpochMilli();
+
+            if (metric instanceof final Counting counting) {
+                buffer.append("# TYPE ")
+                    .append(labels.modifyValue("name", n -> n+"_total").only("name")).append(" counter\n");
+
+                final long count = counting.getCount();
+                buffer
+                    .append(labels.modifyValue("name", n -> n+"_total"))
+                    .append(' ')
+                    .append(count)
+                    .append(' ')
+                    .append(epochMillis)
+                    .append('\n');
+            }
+            if (metric instanceof final Sampling sampling) {
+                // Use the summary form
+                buffer.append("# TYPE ").append(labels.only("name")).append(" summary\n");
+                final Snapshot snapshot = sampling.getSnapshot();
+                for (final double quantile : new double[]{0.5, 0.75, 0.90, 0.95, 0.98, 0.99, 0.999}) {
+                    final double value = snapshot.getValue(quantile);
+                    buffer
+                        .append(labels.and("quantile", String.valueOf(quantile)))
+                        .append(' ')
+                        .append(value)
+                        .append('\n');
+                }
+                final double snapshotCount =snapshot.size();
+                buffer.append(labels.modifyValue("name",n->n+"_count"))
+                    .append(' ')
+                    .append(snapshotCount)
+                    .append('\n');
+                buffer.append("# TYPE ").append(labels.only("name")).append("_max").append(" gauge\n");
+                final long maxValue = snapshot.getMax();
+                buffer.append(labels.modifyValue("name",n->n+"_max"))
+                    .append(' ')
+                    .append(maxValue)
+                    .append('\n');
+                buffer.append("# TYPE ").append(labels.only("name")).append("_min").append(" gauge\n");
+                final long minValue = snapshot.getMin();
+                buffer.append(labels.modifyValue("name",n->n+"_min"))
+                    .append(' ')
+                    .append(minValue)
+                    .append('\n');
+                buffer.append("# TYPE ").append(labels.only("name")).append("_mean").append(" gauge\n");
+                final double meanValue = snapshot.getMean();
+                buffer.append(labels.modifyValue("name",n->n+"_mean"))
+                    .append(' ')
+                    .append(meanValue)
+                    .append('\n');
+                buffer.append("# TYPE ").append(labels.only("name")).append("_stdev").append(" gauge\n");
+                final double stdDev = snapshot.getStdDev();
+                buffer.append(labels.modifyValue("name",n->n+"_stdev"))
+                    .append(' ')
+                    .append(stdDev)
+                    .append('\n');
+
+            }
+            if (metric instanceof final Gauge gauge) {
+                buffer.append("# TYPE ").append(labels.only("name")).append(" gauge\n");
+                final Object value = gauge.getValue();
+                if (value instanceof final Number number) {
+                    final double doubleValue = number.doubleValue();
+                    buffer.append(labels)
+                        .append(' ')
+                        .append(doubleValue)
+                        .append('\n');
+                } else if (value instanceof final CharSequence sequence) {
+                    final String stringValue = sequence.toString();
+                    buffer.append(labels)
+                        .append(' ')
+                        .append(stringValue)
+                        .append('\n');
+                } else if (value instanceof final String stringValue) {
+                    buffer.append(labels)
+                        .append(' ')
+                        .append(stringValue)
+                        .append('\n');
+                } else throw new RuntimeException(
+                        "Unknown label set for metric type '" + metric.getClass().getCanonicalName() + '\''
+                    );
+            }
+            if (metric instanceof final Metered meter) {
+                buffer.append("# TYPE ").append(labels.modifyValue("name",n->n+"_1mRate").only("name")).append(" gauge\n");
+                final double oneMinuteRate = meter.getOneMinuteRate();
+                buffer.append(labels.modifyValue("name",n->n+"_1mRate"))
+                    .append(' ')
+                    .append(oneMinuteRate)
+                    .append('\n');
+
+                buffer.append("# TYPE ").append(labels.modifyValue("name",n->n+"_5mRate").only("name")).append(" gauge\n");
+                final double fiveMinuteRate = meter.getFiveMinuteRate();
+                buffer.append(labels.modifyValue("name",n->n+"_5mRate"))
+                    .append(' ')
+                    .append(fiveMinuteRate)
+                    .append('\n');
+
+                buffer.append("# TYPE ").append(labels.modifyValue("name",n->n+"_15mRate").only("name")).append(" gauge\n");
+                final double fifteenMinuteRate = meter.getFifteenMinuteRate();
+                buffer.append(labels.modifyValue("name",n->n+"_15mRate"))
+                    .append(' ')
+                    .append(fifteenMinuteRate)
+                    .append('\n');
+
+                buffer.append("# TYPE ").append(labels.modifyValue("name",n->n+"_meanRate").only("name")).append(" gauge\n");
+                final double meanRate = meter.getMeanRate();
+                buffer.append(labels.modifyValue("name",n->n+"_meanRate"))
+                    .append(' ')
+                    .append(meanRate)
+                    .append('\n');
+
+            }
+
+        }
+
+        return buffer;
+
+
+    }
+
+    public static String labels(final Map<String, String> labels, final String... additional) {
+        final StringBuilder sb = new StringBuilder("{");
+        for (final String labelName : labels.keySet()) {
+            if ("name".equals(labelName)) continue;
+            sb.append(labelName)
+                .append("=\"")
+                .append(labels.get(labelName))
+                .append('"')
+                .append(',');
+        }
+        sb.setLength(sb.length() - 1);
+
+//        if (additional.length > 0) {
+            for (int i = 0; i < additional.length; i += 2)
+                sb.append(',')
+                    .append(additional[i])
+                    .append("=\"")
+                    .append(additional[i + 1])
+                    .append('"');
+//        }
+
+        sb.append('}');
+        return sb.toString();
+    }
+
+    private static void writeEscapedHelp(final Writer writer, final String s) throws IOException {
+        for (int i = 0; i < s.length(); i++) {
+            final char c = s.charAt(i);
+            switch (c) {
+                case '\\':
+                    writer.append("\\\\");
+                    break;
+                case '\n':
+                    writer.append("\\n");
+                    break;
+                default:
+                    writer.append(c);
+            }
+        }
+    }
+
+}

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/reporters/PromPushReporter.java

@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.engine.metrics.reporters;
+
+import com.codahale.metrics.*;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpClient.Redirect;
+import java.net.http.HttpClient.Version;
+import java.net.http.HttpRequest;
+import java.net.http.HttpRequest.BodyPublishers;
+import java.net.http.HttpResponse;
+import java.net.http.HttpResponse.BodyHandler;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.concurrent.TimeUnit;
+
+public class PromPushReporter extends ScheduledReporter {
+    private static final Logger logger = LogManager.getLogger(PromPushReporter.class);
+    private HttpClient client;
+    private final URI uri;
+
+    public PromPushReporter(
+        final String targetUriSpec,
+        MetricRegistry registry,
+        String name,
+        MetricFilter filter,
+        TimeUnit rateUnit,
+        TimeUnit durationUnit
+    ) {
+        super(registry, name, filter, rateUnit, durationUnit);
+        uri = URI.create(targetUriSpec);
+    }
+
+    @Override
+    public synchronized void report(
+        SortedMap<String, Gauge> gauges,
+        SortedMap<String, Counter> counters,
+        SortedMap<String, Histogram> histograms,
+        SortedMap<String, Meter> meters,
+        SortedMap<String, Timer> timers
+    ) {
+        final java.time.Clock nowclock = Clock.fixed(Instant.now(), ZoneId.systemDefault());
+
+        StringBuilder sb = new StringBuilder(1024*1024); // 1M pre-allocated to reduce heap churn
+
+        int total=0;
+        for(final SortedMap smap : new SortedMap[]{gauges,counters,histograms,meters,timers})
+            for (final Object metric : smap.values()) {
+                sb = PromExpositionFormat.format(nowclock, sb, metric);
+                total++;
+            }
+        PromPushReporter.logger.debug("formatted {} metrics in prom expo format", total);
+        final String exposition = sb.toString();
+
+        final double backoffRatio=1.5;
+        final double maxBackoffSeconds=10;
+        double backOff = 1.0;
+
+        final int maxRetries = 5;
+        int remainingRetries = maxRetries;
+        final List<Exception> errors = new ArrayList<>();
+        boolean succeeded=false;
+
+        while (0 < remainingRetries) {
+            remainingRetries--;
+            final HttpClient client = getCachedClient();
+            final HttpRequest request = HttpRequest.newBuilder().uri(uri).POST(BodyPublishers.ofString(exposition)).build();
+            final BodyHandler<String> handler = HttpResponse.BodyHandlers.ofString();
+            HttpResponse<String> response = null;
+            try {
+                response = client.send(request, handler);
+                final int status = response.statusCode();
+                if ((200 > status) || (300 <= status)) {
+                    final String errmsg = "status " + response.statusCode() + " while posting metrics to '" + this.uri + '\'';
+                    throw new RuntimeException(errmsg);
+                }
+                PromPushReporter.logger.debug("posted {} metrics to prom push endpoint '{}'", total, this.uri);
+                succeeded=true;
+                break;
+            } catch (final Exception e) {
+                errors.add(e);
+                try {
+                    Thread.sleep((int)backOff * 1000L);
+                } catch (final InterruptedException ignored) {
+                }
+                backOff = Math.min(maxBackoffSeconds,backOff*backoffRatio);
+            }
+        }
+        if (!succeeded) {
+            PromPushReporter.logger.error("Failed to send push prom metrics after {} tries. Errors follow:", maxRetries);
+            for (final Exception error : errors) PromPushReporter.logger.error(error);
+        }
+    }
+
+    private synchronized HttpClient getCachedClient() {
+        if (null == client) this.client = this.getNewClient();
+        return this.client;
+    }
+
+    private synchronized HttpClient getNewClient() {
+        this.client = HttpClient.newBuilder()
+            .followRedirects(Redirect.NORMAL)
+            .connectTimeout(Duration.ofSeconds(60))
+            .version(Version.HTTP_2)
+            .build();
+        return this.client;
+    }
+}

nb-api/src/main/java/io/nosqlbench/api/labels/Labeled.java

@@ -1,73 +0,0 @@
-/*
- * Copyright (c) 2022 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.nosqlbench.api.labels;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-public interface Labeled {
-    Map<String, String> getLabels();
-
-    default Map<String, String> getLabelsAnd(String... keyvalues) {
-        LinkedHashMap<String, String> map = new LinkedHashMap<>(getLabels());
-        for (int idx = 0; idx < keyvalues.length; idx+=2) {
-            map.put(keyvalues[idx],keyvalues[idx+1]);
-        }
-        return map;
-    }
-
-    default Map<String, String> getLabelsAnd(Map<String,String> extra) {
-        LinkedHashMap<String,String> map = new LinkedHashMap<>(getLabels());
-        map.putAll(extra);
-        return map;
-    }
-
-    static MapLabels forMap(Map<String,String> labels) {
-        return new MapLabels(labels);
-    }
-
-    class MapLabels implements Labeled {
-        private final Map<String, String> labels;
-
-        public MapLabels(Map<String,String> labels) {
-            this.labels = labels;
-        }
-
-        @Override
-        public Map<String, String> getLabels() {
-            return labels;
-        }
-    }
-
-    default String linearized(Map<String,String> and) {
-        StringBuilder sb= new StringBuilder();
-        Map<String, String> allLabels = this.getLabelsAnd(and);
-        ArrayList<String> sortedLabels = new ArrayList<>(allLabels.keySet());
-        Collections.sort(sortedLabels);
-        for (String label : sortedLabels) {
-            sb.append(label).append(":").append(allLabels.get(label)).append((","));
-        }
-        sb.setLength(sb.length()-",".length());
-        return sb.toString();
-    }
-
-    default String linearized(String... and) {
-        return linearized(getLabelsAnd(and));
-    }
-}

nb-api/src/main/java/io/nosqlbench/api/labels/MutableLabels.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,10 +16,15 @@
 
 package io.nosqlbench.api.labels;
 
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
+
 import java.util.HashMap;
 import java.util.Map;
 
-public class MutableLabels extends HashMap<String, String> implements Labeled {
+public class MutableLabels extends HashMap<String, String> implements NBLabeledElement {
+
+    private NBLabels labels;
 
     public static MutableLabels fromMaps(Map<String, String> entries) {
         MutableLabels mutableLabels = new MutableLabels();
@@ -29,7 +34,7 @@ public class MutableLabels extends HashMap<String, String> implements Labeled {
 
 
     @Override
-    public Map<String, String> getLabels() {
-        return this;
+    public NBLabels getLabels() {
+        return this.labels;
     }
 }

nb-api/src/test/java/io/nosqlbench/api/config/MapLabelsTest.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.config;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class MapLabelsTest {
+
+    @Test
+    public void testLinearizeValues() {
+        final MapLabels l1 = new MapLabels(Map.of("key-a", "value-a", "key-c", "value-c"));
+        final String result = l1.linearizeValues('_', "key-a", "[key-b]", "key-c");
+        assertThat(result).isEqualTo("value-a_value-c");
+    }
+
+}

nb-api/src/test/java/io/nosqlbench/api/config/NBLabeledElementTest.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.config;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class NBLabeledElementTest {
+
+    @Test
+    public void testBasicNameScenario() {
+        final NBLabels labels = NBLabels.forKV("name", "testname","label1","labelvalue1");
+        assertThat(labels.linearize("name")).isEqualTo("testname{label1=\"labelvalue1\"}");
+    }
+
+}

nb-api/src/test/java/io/nosqlbench/api/engine/metrics/reporters/PromExpositionFormatTest.java

@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.engine.metrics.reporters;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import io.nosqlbench.api.config.NBLabels;
+import io.nosqlbench.api.engine.metrics.DeltaHdrHistogramReservoir;
+import io.nosqlbench.api.engine.metrics.instruments.*;
+import org.junit.jupiter.api.Test;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class PromExpositionFormatTest {
+
+    private final Clock nowclock = Clock.fixed(Instant.now(), ZoneId.systemDefault());
+
+    @Test
+    public void testLabelFormat() {
+        assertThat(
+            NBLabels.forMap(Map.of("name","namefoo","property2","value2")).linearize("name")
+        ).isEqualTo("""
+            namefoo{property2="value2"}""");
+    }
+    @Test
+    public void testCounterFormat() {
+        Counter counter = new NBMetricCounter(NBLabels.forKV("name","counter_test_2342", "origin","mars"));
+        counter.inc(23423L);
+
+        String buffer = PromExpositionFormat.format(nowclock, counter);
+        assertThat(buffer).matches(Pattern.compile("""
+            # TYPE counter_test_2342_total counter
+            counter_test_2342_total\\{origin="mars"} \\d+ \\d+
+            """));
+    }
+
+    @Test
+    public void testHistogramFormat() {
+
+        DeltaHdrHistogramReservoir hdr = new DeltaHdrHistogramReservoir(NBLabels.forKV("name","mynameismud","label3","value3"),3);
+
+        for (long i = 0; 1000 > i; i++) {
+            hdr.update(i * 37L);
+        }
+        NBMetricHistogram nbHistogram = new NBMetricHistogram(NBLabels.forKV("name","mynameismud","label3", "value3"), hdr);
+        String formatted = PromExpositionFormat.format(nowclock, nbHistogram);
+
+        assertThat(formatted).matches(Pattern.compile("""
+            # TYPE mynameismud_total counter
+            mynameismud_total\\{label3="value3"} 0 \\d+
+            # TYPE mynameismud summary
+            mynameismud\\{label3="value3",quantile="0.5"} 18463.0
+            mynameismud\\{label3="value3",quantile="0.75"} 27727.0
+            mynameismud\\{label3="value3",quantile="0.9"} 33279.0
+            mynameismud\\{label3="value3",quantile="0.95"} 35135.0
+            mynameismud\\{label3="value3",quantile="0.98"} 36223.0
+            mynameismud\\{label3="value3",quantile="0.99"} 36607.0
+            mynameismud\\{label3="value3",quantile="0.999"} 36927.0
+            mynameismud_count\\{label3="value3"} 1000.0
+            # TYPE mynameismud_max gauge
+            mynameismud_max\\{label3="value3"} 36991
+            # TYPE mynameismud_min gauge
+            mynameismud_min\\{label3="value3"} 0
+            # TYPE mynameismud_mean gauge
+            mynameismud_mean\\{label3="value3"} 18481.975
+            # TYPE mynameismud_stdev gauge
+            mynameismud_stdev\\{label3="value3"} 10681.018083421426
+            """));
+    }
+
+    @Test
+    public void testTimerFormat() {
+
+        DeltaHdrHistogramReservoir hdr = new DeltaHdrHistogramReservoir(NBLabels.forKV("name","monsieurmarius","label4","value4"),3);
+        NBMetricTimer nbMetricTimer = new NBMetricTimer(NBLabels.forKV("name","monsieurmarius","label4", "value4"), hdr);
+        for (long i = 0; 1000 > i; i++)
+            nbMetricTimer.update(i * 37L, TimeUnit.NANOSECONDS);
+
+        String formatted = PromExpositionFormat.format(nowclock, nbMetricTimer);
+
+        assertThat(formatted).matches(Pattern.compile("""
+            # TYPE monsieurmarius_total counter
+            monsieurmarius_total\\{label4="value4"} 1000 \\d+
+            # TYPE monsieurmarius summary
+            monsieurmarius\\{label4="value4",quantile="0.5"} 18463.0
+            monsieurmarius\\{label4="value4",quantile="0.75"} 27727.0
+            monsieurmarius\\{label4="value4",quantile="0.9"} 33279.0
+            monsieurmarius\\{label4="value4",quantile="0.95"} 35135.0
+            monsieurmarius\\{label4="value4",quantile="0.98"} 36223.0
+            monsieurmarius\\{label4="value4",quantile="0.99"} 36607.0
+            monsieurmarius\\{label4="value4",quantile="0.999"} 36927.0
+            monsieurmarius_count\\{label4="value4"} 1000.0
+            # TYPE monsieurmarius_max gauge
+            monsieurmarius_max\\{label4="value4"} 36991
+            # TYPE monsieurmarius_min gauge
+            monsieurmarius_min\\{label4="value4"} 0
+            # TYPE monsieurmarius_mean gauge
+            monsieurmarius_mean\\{label4="value4"} 18481.975
+            # TYPE monsieurmarius_stdev gauge
+            monsieurmarius_stdev\\{label4="value4"} \\d+\\.\\d+
+            # TYPE monsieurmarius_1mRate gauge
+            monsieurmarius_1mRate\\{label4="value4"} 0.0
+            # TYPE monsieurmarius_5mRate gauge
+            monsieurmarius_5mRate\\{label4="value4"} 0.0
+            # TYPE monsieurmarius_15mRate gauge
+            monsieurmarius_15mRate\\{label4="value4"} 0.0
+            # TYPE monsieurmarius_meanRate gauge
+            monsieurmarius_meanRate\\{label4="value4"} \\d+\\.\\d+
+            """));
+    }
+
+    @Test
+    public void testMeterFormat() {
+        NBMetricMeter nbMetricMeter = new NBMetricMeter(NBLabels.forKV("name","eponine","label5", "value5"));
+        String formatted = PromExpositionFormat.format(nowclock, nbMetricMeter);
+
+        assertThat(formatted).matches(Pattern.compile("""
+            # TYPE eponine_total counter
+            eponine_total\\{label5="value5"} 0 \\d+
+            # TYPE eponine_1mRate gauge
+            eponine_1mRate\\{label5="value5"} 0.0
+            # TYPE eponine_5mRate gauge
+            eponine_5mRate\\{label5="value5"} 0.0
+            # TYPE eponine_15mRate gauge
+            eponine_15mRate\\{label5="value5"} 0.0
+            # TYPE eponine_meanRate gauge
+            eponine_meanRate\\{label5="value5"} 0.0
+            """));
+    }
+
+    @Test
+    public void testGaugeFormat() {
+        Gauge cosetteGauge = () -> 1500;
+        NBMetricGauge nbMetricGauge = new NBMetricGauge(NBLabels.forKV("name","cosette","label6", "value6"), cosetteGauge);
+        String formatted = PromExpositionFormat.format(nowclock, nbMetricGauge);
+
+        assertThat(formatted).matches(Pattern.compile("""
+            # TYPE cosette gauge
+            cosette\\{label6="value6"} 1500.0
+            """));
+
+        Gauge cosetteGauge2 = () -> "2000.0";
+        NBMetricGauge nbMetricGauge2 = new NBMetricGauge(NBLabels.forKV("name","cosette2","label7", "value7"), cosetteGauge2);
+        String formatted2 = PromExpositionFormat.format(nowclock, nbMetricGauge2);
+
+        assertThat(formatted2).matches(Pattern.compile("""
+            # TYPE cosette2 gauge
+            cosette2\\{label7="value7"} 2000.0
+            """));
+
+        final int number = 3000;
+        final CharSequence charSequence = Integer.toString(number);
+        Gauge cosetteGauge3 = () -> charSequence;
+        NBMetricGauge nbMetricGauge3 = new NBMetricGauge(NBLabels.forKV("name","cosette3","label8", "value8"), cosetteGauge3);
+        String formatted3 = PromExpositionFormat.format(nowclock, nbMetricGauge3);
+
+        assertThat(formatted3).matches(Pattern.compile("""
+            # TYPE cosette3 gauge
+            cosette3\\{label8="value8"} 3000
+            """));
+    }
+}