nosqlbench-1478 sanitize all labels which are used according to the openmetrics exposition format...

adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/util/AmqpAdapterMetrics.java

@@ -51,7 +51,7 @@ public class AmqpAdapterMetrics {
 
     public AmqpAdapterMetrics(final AmqpBaseOpDispenser amqpBaseOpDispenser, final NBLabeledElement labeledParent) {
         this.amqpBaseOpDispenser = amqpBaseOpDispenser;
-        labels=labeledParent.getLabels().and("name", AmqpAdapterMetrics.class.getSimpleName());
+        labels=labeledParent.getLabels().andTypes("name", AmqpAdapterMetrics.class.getSimpleName());
     }
 
     public void initS4JAdapterInstrumentation() {

engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/ratelimits/HybridRateLimiter.java

@@ -152,9 +152,9 @@ public class HybridRateLimiter implements RateLimiter {
 
 
     protected void init(final NBLabeledElement activityDef) {
-        delayGauge = ActivityMetrics.gauge(activityDef, this.label + ".waittime", new RateLimiters.WaitTimeGauge(this));
+        delayGauge = ActivityMetrics.gauge(activityDef, this.label + "_waittime", new RateLimiters.WaitTimeGauge(this));
-        avgRateGauge = ActivityMetrics.gauge(activityDef, this.label + ".config.cyclerate", new RateLimiters.RateGauge(this));
+        avgRateGauge = ActivityMetrics.gauge(activityDef, this.label + "_config_cyclerate", new RateLimiters.RateGauge(this));
-        burstRateGauge = ActivityMetrics.gauge(activityDef, this.label + ".config.burstrate", new RateLimiters.BurstRateGauge(this));
+        burstRateGauge = ActivityMetrics.gauge(activityDef, this.label + "_config_burstrate", new RateLimiters.BurstRateGauge(this));
     }
 
     @Override

engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java

@@ -46,7 +46,7 @@ public class NBCLIScenarioParser {
 
     private final static Logger logger = LogManager.getLogger("SCENARIOS");
     private static final String SEARCH_IN = "activities";
-    public static final String WORKLOAD_SCENARIO_STEP = "WORKLOAD_SCENARIO_STEP";
+    public static final String WORKLOAD_SCENARIO_STEP = "STEP";
 
     public static boolean isFoundWorkload(String workload, String... includes) {
         Optional<Content<?>> found = NBIO.all()
@@ -190,8 +190,9 @@ public class NBCLIScenarioParser {
                     buildingCmd.put("alias", "alias=" + WORKLOAD_SCENARIO_STEP);
                 }
 
+                // TODO: simplify this
                 String alias = buildingCmd.get("alias");
-                for (String token : new String[]{"WORKLOAD", "SCENARIO", "STEP"}) {
+                for (String token : new String[]{"STEP"}) {
                     if (!alias.contains(token)) {
                         logger.warn("Your alias template '" + alias + "' does not contain " + token + ", which will " +
                             "cause your metrics to be combined under the same name. It is strongly advised that you " +
@@ -206,6 +207,7 @@ public class NBCLIScenarioParser {
                 alias = alias.replaceAll("STEP", sanitize(stepName));
                 alias = (alias.startsWith("alias=") ? alias : "alias=" + alias);
                 buildingCmd.put("alias", alias);
+                buildingCmd.put("labels","labels=workload:"+sanitize(workloadToken));
 
                 logger.debug(() -> "rebuilt command: " + String.join(" ", buildingCmd.values()));
                 buildCmdBuffer.addAll(buildingCmd.values());
@@ -219,7 +221,12 @@ public class NBCLIScenarioParser {
     public static String sanitize(String word) {
         String sanitized = word;
         sanitized = sanitized.replaceAll("\\..+$", "");
-        sanitized = sanitized.replaceAll("[^a-zA-Z0-9]+", "");
+        sanitized = sanitized.replaceAll("-","_");
+        sanitized = sanitized.replaceAll("[^a-zA-Z0-9_]+", "");
+
+        if (!word.equals(sanitized)) {
+            logger.warn("The identifier or value '" + word + "' was sanitized to '" + sanitized + "' to be compatible with monitoring systems. You should probably change this to make diagnostics easier.");
+        }
         return sanitized;
     }
 
@@ -326,7 +333,7 @@ public class NBCLIScenarioParser {
 
                 OpsDocList stmts = null;
                 try {
-                    stmts = OpsLoader.loadContent(content, Map.of());
+                    stmts = OpsLoader.loadContent(content, new LinkedHashMap<>());
                     if (stmts.getStmtDocs().size() == 0) {
                         logger.warn("Encountered yaml with no docs in '" + referenced + "'");
                         continue;

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -16,6 +16,8 @@
 
 package io.nosqlbench.engine.cli;
 
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 import io.nosqlbench.api.annotations.Annotation;
 import io.nosqlbench.api.annotations.Layer;
 import io.nosqlbench.api.config.NBLabeledElement;
@@ -242,14 +244,30 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
                     " these urls in your browser: http://<host>:3000 and http://<host>:9090";
             NBCLI.logger.warn(warn);
             graphiteMetricsAddress = "localhost";
-        } else if (null != dockerMetricsAt) graphiteMetricsAddress = dockerMetricsAt;
+        } else if (null != dockerMetricsAt) {
+            graphiteMetricsAddress = dockerMetricsAt;
+        }
 
-        if (null != graphiteMetricsAddress) {
+        if (annotatorsConfig == null || annotatorsConfig.isBlank()) {
-            reportGraphiteTo = graphiteMetricsAddress + ":9109";
+            List<Map<String, String>> annotatorsConfigs = new ArrayList<>();
-            annotatorsConfig = "[{type:'log',level:'info'},{type:'grafana',baseurl:'http://" + graphiteMetricsAddress + ":3000" +
+            annotatorsConfigs.add(Map.of(
-                "/'," +
+                    "type", "log",
-                "tags:'appname:nosqlbench',timeoutms:5000,onerror:'warn'}]";
+                    "level", "info"
-        } else annotatorsConfig = "[{type:'log',level:'info'}]";
+            ));
+
+            if (null != graphiteMetricsAddress) {
+                reportGraphiteTo = graphiteMetricsAddress + ":9109";
+                annotatorsConfigs.add(Map.of(
+                        "type", "grafana",
+                        "baseurl", "http://" + graphiteMetricsAddress + ":3000",
+                        "tags", "appname:nosqlbench",
+                        "timeoutms", "5000",
+                        "onerror", "warn"
+                ));
+            }
+            Gson gson = new GsonBuilder().create();
+            annotatorsConfig = gson.toJson(annotatorsConfigs);
+        }
 
         final NBCLIOptions options = new NBCLIOptions(args);
         NBCLI.logger = LogManager.getLogger("NBCLI");
@@ -377,12 +395,12 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
         NBCLI.logger.debug("initializing annotators with config:'{}'", annotatorsConfig);
         Annotators.init(annotatorsConfig);
         Annotators.recordAnnotation(
-            Annotation.newBuilder()
+                Annotation.newBuilder()
-                .session(sessionName)
+                        .element(this)
-                .now()
+                        .now()
-                .layer(Layer.CLI)
+                        .layer(Layer.Session)
-                .detail("cli", String.join("\n", args))
+                        .detail("cli", String.join("\n", args))
-                .build()
+                        .build()
         );
 
         if ((null != reportPromPushTo) || (null != reportGraphiteTo) || (null != options.wantsReportCsvTo())) {
@@ -428,17 +446,17 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
         }
 
         final Scenario scenario = new Scenario(
-            sessionName,
+                sessionName,
-            options.getScriptFile(),
+                options.getScriptFile(),
-            options.getScriptingEngine(),
+                options.getScriptingEngine(),
-            options.getProgressSpec(),
+                options.getProgressSpec(),
-            options.wantsStackTraces(),
+                options.wantsStackTraces(),
-            options.wantsCompileScript(),
+                options.wantsCompileScript(),
-            options.getReportSummaryTo(),
+                options.getReportSummaryTo(),
-            String.join("\n", args),
+                String.join("\n", args),
-            options.getLogsDirectory(),
+                options.getLogsDirectory(),
-            Maturity.Unspecified,
+                Maturity.Unspecified,
-            this);
+                this);
 
         final ScriptBuffer buffer = new BasicScriptBuffer()
                 .add(options.getCommands()

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java

@@ -16,20 +16,19 @@
 
 package io.nosqlbench.engine.cli;
 
+import io.nosqlbench.api.config.NBLabelSpec;
+import io.nosqlbench.api.config.NBLabels;
 import io.nosqlbench.api.engine.util.Unit;
 import io.nosqlbench.api.errors.BasicError;
 import io.nosqlbench.api.logging.NBLogLevel;
-import io.nosqlbench.api.system.NBEnvironment;
+import io.nosqlbench.api.system.NBStatePath;
 import io.nosqlbench.engine.api.metrics.IndicatorMode;
 import io.nosqlbench.engine.cli.Cmd.CmdType;
 import io.nosqlbench.engine.core.lifecycle.scenario.Scenario.Engine;
 import io.nosqlbench.nb.annotations.Maturity;
 
 import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.attribute.PosixFilePermissions;
 import java.security.InvalidParameterException;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -44,14 +43,13 @@ public class NBCLIOptions {
 
 
     private static final String NB_STATE_DIR = "--statedir";
-    private static final String NB_STATEDIR_PATHS = "$NBSTATEDIR:$PWD/.nosqlbench:$HOME/.nosqlbench";
     public static final String ARGS_FILE_DEFAULT = "$NBSTATEDIR/argsfile";
     private static final String INCLUDE = "--include";
 
     private static final String userHome = System.getProperty("user.home");
 
 
-    private static final Map<String,String> DEFAULT_LABELS=Map.of("appname","nosqlbench");
+    private static final Map<String, String> DEFAULT_LABELS = Map.of("appname", "nosqlbench");
     private static final String METRICS_PREFIX = "--metrics-prefix";
     private static final String ANNOTATE_EVENTS = "--annotate";
     private static final String ANNOTATORS_CONFIG = "--annotators";
@@ -136,7 +134,7 @@ public class NBCLIOptions {
     //    private static final String DEFAULT_CONSOLE_LOGGING_PATTERN = "%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n";
 
 
-    private final Map<String,String> labels = new LinkedHashMap<>(DEFAULT_LABELS);
+    private NBLabels labels = NBLabels.forKV();
     private final List<Cmd> cmdList = new ArrayList<>();
     private int logsMax;
     private boolean wantsVersionShort;
@@ -151,7 +149,8 @@ public class NBCLIOptions {
     private int reportInterval = 10;
     private String metricsPrefix = "nosqlbench";
     private String wantsMetricsForActivity;
-    private String sessionName = "";
+    private String sessionName = "SESSIONCODE";
+    //    private String sessionName = "scenario_%tY%tm%td_%tH%tM%tS_%tL";
     private boolean showScript;
     private NBLogLevel consoleLevel = NBLogLevel.WARN;
     private final List<String> histoLoggerConfigs = new ArrayList<>();
@@ -185,9 +184,8 @@ public class NBCLIOptions {
     private String[] annotateEvents = {"ALL"};
     private String dockerMetricsHost;
     private String annotatorsConfig = "";
-    private String statedirs = NBCLIOptions.NB_STATEDIR_PATHS;
+    private String statedirs = NBStatePath.NB_STATEDIR_PATHS;
     private Path statepath;
-    private final List<String> statePathAccesses = new ArrayList<>();
     private final String hdrForChartFileName = NBCLIOptions.DEFAULT_CHART_HDR_LOG_NAME;
     private String dockerPromRetentionDays = "3650d";
     private String reportSummaryTo = NBCLIOptions.REPORT_SUMMARY_TO_DEFAULT;
@@ -210,8 +208,8 @@ public class NBCLIOptions {
         return this.annotatorsConfig;
     }
 
-    public Map<String,String> getLabelMap() {
+    public NBLabels getLabelMap() {
-        return Collections.unmodifiableMap(this.labels);
+        return this.labels;
     }
 
     public String getChartHdrFileName() {
@@ -305,8 +303,7 @@ public class NBCLIOptions {
                     nonincludes.addLast(arglist.removeFirst());
             }
         }
-        statedirs = (null != this.statedirs) ? statedirs : NBCLIOptions.NB_STATEDIR_PATHS;
+        this.statepath = NBStatePath.initialize(statedirs);
-        setStatePath();
 
         arglist = nonincludes;
         nonincludes = new LinkedList<>();
@@ -336,7 +333,7 @@ public class NBCLIOptions {
                 case NBCLIArgsFile.ARGS_FILE_REQUIRED:
                 case NBCLIArgsFile.ARGS_PIN:
                 case NBCLIArgsFile.ARGS_UNPIN:
-                    if (null == this.statepath) this.setStatePath();
+                    this.statepath = NBStatePath.initialize(statedirs);
                     arglist = argsfile.process(arglist);
                     break;
                 case NBCLIOptions.ANSI:
@@ -487,63 +484,13 @@ public class NBCLIOptions {
     }
 
     private void setLabels(String labeldata) {
-        this.labels.clear();
+        this.labels = NBLabels.forKV();
         addLabels(labeldata);
     }
 
     private void addLabels(String labeldata) {
-        Map<String,String> newLabels = parseLabels(labeldata);
+        NBLabels newLabels = NBLabelSpec.parseLabels(labeldata);
-        this.labels.putAll(newLabels);
+        this.labels = this.labels.and(newLabels);
-    }
-
-    private Map<String, String> parseLabels(String labeldata) {
-        Map<String,String> setLabelsTo = new LinkedHashMap<>();
-        for (String component : labeldata.split("[,; ]")) {
-            String[] parts = component.split("\\W", 2);
-            if (parts.length!=2) {
-                throw new BasicError("Unable to parse labels to set:" + labeldata);
-            }
-            setLabelsTo.put(parts[0],parts[1]);
-        }
-        return setLabelsTo;
-    }
-
-
-    private Path setStatePath() {
-        if (0 < statePathAccesses.size())
-            throw new BasicError("The state dir must be set before it is used by other\n" +
-                    " options. If you want to change the statedir, be sure you do it before\n" +
-                    " dependent options. These parameters were called before this --statedir:\n" +
-                    this.statePathAccesses.stream().map(s -> "> " + s).collect(Collectors.joining("\n")));
-        if (null != this.statepath) return statepath;
-
-        final List<String> paths = NBEnvironment.INSTANCE.interpolateEach(":", this.statedirs);
-        Path selected = null;
-
-        for (final String pathName : paths) {
-            final Path path = Path.of(pathName);
-            if (Files.exists(path)) {
-                if (Files.isDirectory(path)) {
-                    selected = path;
-                    break;
-                }
-                System.err.println("ERROR: possible state dir path is not a directory: '" + path + '\'');
-            }
-        }
-        if (null == selected) selected = Path.of(paths.get(paths.size() - 1));
-
-        if (!Files.exists(selected)) try {
-            Files.createDirectories(
-                    selected,
-                    PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxrwx---"))
-            );
-        } catch (final IOException e) {
-            throw new BasicError("Could not create state directory at '" + selected + "': " + e.getMessage());
-        }
-
-        NBEnvironment.INSTANCE.put(NBEnvironment.NBSTATEDIR, selected.toString());
-
-        return selected;
     }
 
     private void parseAllOptions(final String[] args) {
@@ -846,14 +793,10 @@ public class NBCLIOptions {
 
     public String getProgressSpec() {
         final ProgressSpec spec = this.parseProgressSpec(progressSpec);// sanity check
-        //                System.err.println("Console is already logging info or more, so progress data on console is " +
-        //                        "suppressed.");
         if (IndicatorMode.console == spec.indicatorMode)
             if (consoleLevel.isGreaterOrEqualTo(NBLogLevel.INFO)) spec.indicatorMode = IndicatorMode.logonly;
-            else //                System.err.println("Command line includes script calls, so progress data on console is " +
+            else if (cmdList.stream().anyMatch(cmd -> CmdType.script == cmd.getCmdType()))
-                //                        "suppressed.");
+                spec.indicatorMode = IndicatorMode.logonly;
-                if (cmdList.stream().anyMatch(cmd -> CmdType.script == cmd.getCmdType()))
-                    spec.indicatorMode = IndicatorMode.logonly;
         return spec.toString();
     }
 

engine-clients/src/main/java/io/nosqlbench/engine/clients/grafana/GrafanaTokenAuthenticator.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.clients.grafana;
+
+import io.nosqlbench.api.apps.BundledApp;
+import io.nosqlbench.api.system.NBEnvironment;
+import io.nosqlbench.nb.annotations.Service;
+import picocli.CommandLine;
+
+@Service(value = BundledApp.class,selector = "grafana-apikey")
+@CommandLine.Command(
+    name="gafana-apikey",
+    description = "create and cache a grafana apikey for a given grafana server"
+)
+public class GrafanaTokenAuthenticator implements BundledApp {
+
+    @CommandLine.Parameters
+    private final String keyfile = NBEnvironment.INSTANCE.get("apikeyfile");
+    @Override
+    public int applyAsInt(String[] value) {
+        return 0;
+    }
+}

engine-clients/src/main/java/io/nosqlbench/engine/clients/grafana/annotator/GrafanaMetricsAnnotator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,12 +21,12 @@ import io.nosqlbench.api.annotations.Annotator;
 import io.nosqlbench.api.config.params.ParamsParser;
 import io.nosqlbench.api.config.standard.*;
 import io.nosqlbench.api.errors.BasicError;
+import io.nosqlbench.api.errors.OnError;
 import io.nosqlbench.api.metadata.SystemId;
 import io.nosqlbench.engine.clients.grafana.GrafanaClient;
 import io.nosqlbench.engine.clients.grafana.GrafanaClientConfig;
 import io.nosqlbench.engine.clients.grafana.transfer.GAnnotation;
 import io.nosqlbench.nb.annotations.Service;
-import io.nosqlbench.api.errors.OnError;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -36,6 +36,7 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.regex.Pattern;
 
 @Service(value = Annotator.class, selector = "grafana")
 public class GrafanaMetricsAnnotator implements Annotator, NBConfigurable {
@@ -58,9 +59,12 @@ public class GrafanaMetricsAnnotator implements Annotator, NBConfigurable {
             ga.setTime(annotation.getStart());
             ga.setTimeEnd(annotation.getEnd());
 
-            annotation.getLabels().forEach((k, v) -> {
+            annotation.getLabels().onlyTypes().asMap().forEach((k, v) -> {
                 ga.getTags().add(k + ":" + v);
             });
+            annotation.getLabels().onlyInstances().asMap().forEach((k,v)->{
+                ga.addText(" " + k + ":" + v);
+            });
             ga.getTags().add("layer:" + annotation.getLayer().toString());
 
             if (annotation.getStart() == annotation.getEnd()) {
@@ -69,15 +73,12 @@ public class GrafanaMetricsAnnotator implements Annotator, NBConfigurable {
                 ga.getTags().add("span:interval");
             }
 
-            Map<String, String> labels = annotation.getLabels();
+            Map<String, String> labels = annotation.getLabels().asMap();
 
             Optional.ofNullable(labels.get("alertId"))
                 .map(Integer::parseInt).ifPresent(ga::setAlertId);
 
-            ga.setText(annotation.toString());
+            ga.addText(annotation.toString());
-
-            annotation.getSession();
-
 
             // Target
             Optional.ofNullable(labels.get("type"))
@@ -175,7 +176,8 @@ public class GrafanaMetricsAnnotator implements Annotator, NBConfigurable {
     public NBConfigModel getConfigModel() {
         return ConfigModel.of(this.getClass())
             .add(Param.required("baseurl", String.class)
-                .setDescription("The base url of the grafana node, like http://localhost:3000/"))
+                .setDescription("The base url of the grafana node, like http://localhost:3000/")
+                    .setRegex(Pattern.compile("http.+\\\\/")))
             .add(Param.defaultTo("apikeyfile", "$NBSTATEDIR/grafana/grafana_apikey")
                 .setDescription("The file that contains the api key, supersedes apikey"))
             .add(Param.optional("apikey", String.class)

engine-clients/src/main/java/io/nosqlbench/engine/clients/grafana/transfer/GAnnotation.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -121,8 +121,8 @@ public class GAnnotation {
         return text;
     }
 
-    public void setText(String text) {
+    public void addText(String text) {
-        this.text = text;
+        this.text = this.text == null ? text : this.text + "\n" + text;
     }
 
     public String getMetric() {

engine-clients/src/test/java/io/nosqlbench/engine/clients/grafana/annotator/GrafanaMetricsAnnotatorTest.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.clients.grafana.annotator;
+
+import io.nosqlbench.api.annotations.Annotation;
+import io.nosqlbench.api.annotations.Layer;
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
+import io.nosqlbench.api.system.NBStatePath;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+public class GrafanaMetricsAnnotatorTest implements NBLabeledElement {
+
+
+    @Test
+    @Disabled
+    public void testPost() {
+        String ipaddr="CHANGEME";
+        NBStatePath.initialize();
+        GrafanaMetricsAnnotator ganno = new GrafanaMetricsAnnotator();
+        ganno.applyConfig(ganno.getConfigModel().apply(Map.of(
+            "baseurl","http://"+ipaddr+":3000/",
+            "tags","appname:nosqlbench",
+            "timeoutms","5000",
+            "onerror","warn"
+        )));
+        ganno.recordAnnotation(Annotation.newBuilder()
+            .element(this)
+            .now()
+            .layer(Layer.Session)
+            .build());
+    }
+
+    @Override
+    public NBLabels getLabels() {
+        return NBLabels.forMap(Map.of("testlabelname","testlabelvalue"));
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java

@@ -15,6 +15,8 @@
  */
 package io.nosqlbench.engine.core.lifecycle.activity;
 
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
 import io.nosqlbench.engine.api.activityapi.core.*;
 import io.nosqlbench.engine.api.activityimpl.MotorState;
 import io.nosqlbench.api.annotations.Annotation;
@@ -51,7 +53,7 @@ import java.util.stream.Collectors;
  * This allows the state tracking to work consistently for all observers.</p>
  */
 
-public class ActivityExecutor implements ActivityController, ParameterMap.Listener, ProgressCapable, Callable<ExecutionResult> {
+public class ActivityExecutor implements NBLabeledElement, ActivityController, ParameterMap.Listener, ProgressCapable, Callable<ExecutionResult> {
 
     // TODO Encapsulate valid state transitions to be only modifiable within the appropriate type view.
 
@@ -99,12 +101,9 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
         logger.info(() -> "stopped: " + this.getActivityDef().getAlias() + " with " + motors.size() + " slots");
 
         Annotators.recordAnnotation(Annotation.newBuilder()
-            .session(sessionId)
+            .element(this)
             .interval(this.startedAt, this.stoppedAt)
             .layer(Layer.Activity)
-            .label("alias", getActivityDef().getAlias())
-            .label("driver", getActivityDef().getActivityType())
-            .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none"))
             .detail("params", getActivityDef().toString())
             .build()
         );
@@ -126,12 +125,9 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
         logger.info(() -> "stopped: " + this.getActivityDef().getAlias() + " with " + motors.size() + " slots");
 
         Annotators.recordAnnotation(Annotation.newBuilder()
-            .session(sessionId)
+                .element(this)
             .interval(this.startedAt, this.stoppedAt)
             .layer(Layer.Activity)
-            .label("alias", getActivityDef().getAlias())
-            .label("driver", getActivityDef().getActivityType())
-            .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none"))
             .detail("params", getActivityDef().toString())
             .build()
         );
@@ -505,12 +501,9 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
 
         logger.info(() -> "starting activity " + activity.getAlias() + " for cycles " + activity.getCycleSummary());
         Annotators.recordAnnotation(Annotation.newBuilder()
-            .session(sessionId)
+            .element(this)
             .now()
             .layer(Layer.Activity)
-            .label("alias", getActivityDef().getAlias())
-            .label("driver", getActivityDef().getActivityType())
-            .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none"))
             .detail("params", getActivityDef().toString())
             .build()
         );
@@ -533,4 +526,8 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
     }
 
 
+    @Override
+    public NBLabels getLabels() {
+        return activity.getLabels();
+    }
 }

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/Scenario.java

@@ -138,7 +138,7 @@ public class Scenario implements Callable<ExecutionMetricsResult>, NBLabeledElem
     }
 
     public static Scenario forTesting(final String name, final Engine engine, final String reportSummaryTo, final Maturity minMaturity) {
-        return new Scenario(name, null, engine, "console:10s", true, true, reportSummaryTo, "", Path.of("logs"), minMaturity, NBLabeledElement.forKV("test-name", "name"));
+        return new Scenario(name, null, engine, "console:10s", true, true, reportSummaryTo, "", Path.of("logs"), minMaturity, NBLabeledElement.forKV("test_name", "name"));
     }
 
     public Scenario setLogger(final Logger logger) {
@@ -249,7 +249,7 @@ public class Scenario implements Callable<ExecutionMetricsResult>, NBLabeledElem
         this.startedAtMillis = System.currentTimeMillis();
         Annotators.recordAnnotation(
             Annotation.newBuilder()
-                .session(scenarioName)
+                .element(this)
                 .now()
                 .layer(Layer.Scenario)
                 .detail("engine", engine.toString())
@@ -348,10 +348,10 @@ public class Scenario implements Callable<ExecutionMetricsResult>, NBLabeledElem
 
         // We report the scenario state via annotation even for short runs
         final Annotation annotation = Annotation.newBuilder()
-            .session(scenarioName)
+            .element(this)
             .interval(startedAtMillis, this.endedAtMillis)
             .layer(Layer.Scenario)
-            .label("state", state.toString())
+//            .labels("state", state.toString())
             .detail("command_line", commandLine)
             .build();
 

engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/ScenarioController.java

@@ -68,15 +68,6 @@ public class ScenarioController implements NBLabeledElement {
      * @param activityDef string in alias=value1;driver=value2;... format
      */
     public synchronized void start(ActivityDef activityDef) {
-        Annotators.recordAnnotation(Annotation.newBuilder()
-            .session(scenario.getScenarioName())
-            .now()
-            .layer(Layer.Activity)
-            .label("alias", activityDef.getAlias())
-            .detail("command", "start")
-            .detail("params", activityDef.toString())
-            .build());
-
         doStartActivity(activityDef);
     }
 
@@ -84,6 +75,14 @@ public class ScenarioController implements NBLabeledElement {
     private synchronized ActivityRuntimeInfo doStartActivity(ActivityDef activityDef) {
         if (!this.activityInfoMap.containsKey(activityDef.getAlias())) {
             Activity activity = this.activityLoader.loadActivity(activityDef, this);
+
+            Annotators.recordAnnotation(Annotation.newBuilder()
+                .element(activity)
+                .now()
+                .layer(Layer.Activity)
+                .detail("params", activityDef.toString())
+                .build());
+
             ActivityExecutor executor = new ActivityExecutor(activity, this.scenario.getScenarioName());
             Future<ExecutionResult> startedActivity = activitiesExecutor.submit(executor);
             ActivityRuntimeInfo activityRuntimeInfo = new ActivityRuntimeInfo(activity, startedActivity, executor);
@@ -125,14 +124,6 @@ public class ScenarioController implements NBLabeledElement {
      * @param activityDef A definition for an activity to run
      */
     public synchronized void run(ActivityDef activityDef, long timeoutMs) {
-        Annotators.recordAnnotation(Annotation.newBuilder()
-            .session(this.scenario.getScenarioName())
-            .now()
-            .layer(Layer.Activity)
-            .label("alias", activityDef.getAlias())
-            .detail("command", "run")
-            .detail("params", activityDef.toString())
-            .build());
 
         doStartActivity(activityDef);
         awaitActivity(activityDef, timeoutMs);
@@ -179,14 +170,6 @@ public class ScenarioController implements NBLabeledElement {
      * @param activityDef An activity def, including at least the alias parameter.
      */
     public synchronized void stop(ActivityDef activityDef) {
-        Annotators.recordAnnotation(Annotation.newBuilder()
-            .session(this.scenario.getScenarioName())
-            .now()
-            .layer(Layer.Activity)
-            .label("alias", activityDef.getAlias())
-            .detail("command", "stop")
-            .detail("params", activityDef.toString())
-            .build());
 
         ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(activityDef.getAlias());
         if (null == runtimeInfo) {
@@ -196,6 +179,14 @@ public class ScenarioController implements NBLabeledElement {
         scenariologger.debug("STOP {}", activityDef.getAlias());
 
         runtimeInfo.stopActivity();
+        Annotators.recordAnnotation(Annotation.newBuilder()
+            .element(runtimeInfo.getActivity())
+            .now()
+            .layer(Layer.Activity)
+            .detail("command", "stop")
+            .detail("params", activityDef.toString())
+            .build());
+
     }
 
     /**
@@ -240,20 +231,21 @@ public class ScenarioController implements NBLabeledElement {
      * @param activityDef An activity def, including at least the alias parameter.
      */
     public synchronized void forceStop(ActivityDef activityDef) {
-        Annotators.recordAnnotation(Annotation.newBuilder()
-            .session(this.scenario.getScenarioName())
-            .now()
-            .layer(Layer.Activity)
-            .label("alias", activityDef.getAlias())
-            .detail("command", "forceStop")
-            .detail("params", activityDef.toString())
-            .build());
 
         ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(activityDef.getAlias());
+
         if (null == runtimeInfo) {
             throw new RuntimeException("could not force stop missing activity:" + activityDef);
         }
 
+        Annotators.recordAnnotation(Annotation.newBuilder()
+            .element(runtimeInfo.getActivity())
+            .now()
+            .layer(Layer.Activity)
+            .detail("command", "forceStop")
+            .detail("params", activityDef.toString())
+            .build());
+
         scenariologger.debug("FORCE STOP {}", activityDef.getAlias());
 
         runtimeInfo.forceStopActivity();

nb-api/src/main/java/io/nosqlbench/api/annotations/Annotation.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,6 +16,9 @@
 
 package io.nosqlbench.api.annotations;
 
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
+
 import java.util.Map;
 
 /**
@@ -30,11 +33,7 @@ import java.util.Map;
  * NoSQLBench. It is up to the downstream consumers to map these
  * to concrete fields or identifiers as appropriate.
  */
-public interface Annotation {
+public interface Annotation extends NBLabeledElement {
-    /**
-     * @return The named session that the annotation is associated with
-     */
-    String getSession();
 
     /**
      * If this is the same as {@link #getEnd()}, then the annotation is
@@ -74,7 +73,7 @@ public interface Annotation {
      *
      * @return The labels map
      */
-    Map<String, String> getLabels();
+    NBLabels getLabels();
 
     /**
      * The details are an ordered map of all the content that you would want the user to see.
@@ -83,15 +82,15 @@ public interface Annotation {
      */
     Map<String, String> getDetails();
 
-    static AnnotationBuilderFacets.WantsSession newBuilder() {
+    static AnnotationBuilderFacets.WantsLabeledElement newBuilder() {
         return new AnnotationBuilder();
     }
 
     /**
-     * This should return {@link Span#interval} if the span of time is not an instant, and
+     * This should return {@link Temporal#interval} if the span of time is not an instant, and
-     * {@link Span#instant}, otherwise.
+     * {@link Temporal#instant}, otherwise.
      */
-    Span getSpan();
+    Temporal getTemporal();
 
     String asJson();
 

nb-api/src/main/java/io/nosqlbench/api/annotations/AnnotationBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 package io.nosqlbench.api.annotations;
 
+import io.nosqlbench.api.config.NBLabeledElement;
+
 import java.time.ZoneId;
 import java.util.LinkedHashMap;
 import java.util.TimeZone;
@@ -24,15 +26,15 @@ public class AnnotationBuilder implements AnnotationBuilderFacets.All {
     private String session;
     private long start;
     private long end;
-    private final LinkedHashMap<String, String> labels = new LinkedHashMap<>();
     private final LinkedHashMap<String, String> details = new LinkedHashMap<>();
     private Layer layer;
     private final TimeZone timezone = TimeZone.getTimeZone(ZoneId.of("GMT"));
 
+    private NBLabeledElement element;
+
     @Override
     public AnnotationBuilder layer(Layer layer) {
         this.layer = layer;
-        this.label("layer", layer.toString());
         return this;
     }
 
@@ -68,12 +70,6 @@ public class AnnotationBuilder implements AnnotationBuilderFacets.All {
     }
 
 
-    @Override
-    public AnnotationBuilder label(String name, String value) {
-        this.labels.put(name, value);
-        return this;
-    }
-
     @Override
     public AnnotationBuilderFacets.WantsMoreDetailsOrBuild detail(String name, String value) {
         this.details.put(name, value);
@@ -82,14 +78,13 @@ public class AnnotationBuilder implements AnnotationBuilderFacets.All {
 
     @Override
     public Annotation build() {
-        return new MutableAnnotation(timezone, session, layer, start, end, labels, details).asReadOnly();
+        return new MutableAnnotation(timezone, session, layer, start, end, element, details).asReadOnly();
 
     }
 
     @Override
-    public AnnotationBuilderFacets.WantsInterval session(String session) {
+    public AnnotationBuilderFacets.WantsInterval element(NBLabeledElement element) {
-        this.session = session;
+        this.element = element;
         return this;
     }
-
 }

nb-api/src/main/java/io/nosqlbench/api/annotations/AnnotationBuilderFacets.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,17 +16,19 @@
 
 package io.nosqlbench.api.annotations;
 
+import io.nosqlbench.api.config.NBLabeledElement;
+
 public interface AnnotationBuilderFacets {
 
     interface All extends
-            WantsSession, WantsInterval, WantsLayer, WantsLabels, WantsMoreDetailsOrBuild, WantsMoreLabelsOrDetails {
+        WantsLabeledElement, WantsInterval, WantsLayer, WantsMoreDetailsOrBuild {
     }
 
-    interface WantsSession {
+    interface WantsLabeledElement {
         /**
          * The session is the global name of a NoSQLBench process which run a scenario. It is required.
          */
-        WantsInterval session(String session);
+        WantsInterval element(NBLabeledElement element);
     }
 
     interface WantsInterval {
@@ -50,22 +52,11 @@ public interface AnnotationBuilderFacets {
     }
 
     interface WantsLayer {
-        WantsMoreLabelsOrDetails layer(Layer layer);
+        WantsMoreDetailsOrBuild layer(Layer layer);
-    }
-
-    interface WantsLabels {
-        WantsMoreLabelsOrDetails label(String name, String value);
-    }
-
-    interface WantsMoreLabelsOrDetails {
-        WantsMoreLabelsOrDetails label(String name, String value);
-
-        WantsMoreDetailsOrBuild detail(String name, String value);
     }
 
     interface WantsMoreDetailsOrBuild {
         WantsMoreDetailsOrBuild detail(String name, String value);
-
         Annotation build();
     }
 

nb-api/src/main/java/io/nosqlbench/api/annotations/Layer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@ public enum Layer {
      * Events which describe command line arguments, such as parsing,
      * named scenario mapping, or critical errors
      */
-    CLI,
+    Session,
 
     /**
      * Events which describe scenario execution, such as parameters,

nb-api/src/main/java/io/nosqlbench/api/annotations/MutableAnnotation.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,12 +19,16 @@ package io.nosqlbench.api.annotations;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.annotations.Expose;
+import io.nosqlbench.api.config.NBLabeledElement;
+import io.nosqlbench.api.config.NBLabels;
 
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
-import java.util.*;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.TimeZone;
 
 public class MutableAnnotation implements Annotation {
 
@@ -41,13 +45,11 @@ public class MutableAnnotation implements Annotation {
     @Expose
     private long end = 0L;
 
-    @Expose
-    private Map<String, String> labels = new LinkedHashMap<>();
-
     @Expose
     private Map<String, String> details = new LinkedHashMap<>();
 
     private final ZoneId zoneid = ZoneId.of("GMT");
+    private NBLabeledElement element;
 
     public MutableAnnotation(
             TimeZone timezone,
@@ -55,50 +57,40 @@ public class MutableAnnotation implements Annotation {
             Layer layer,
             long start,
             long end,
-            LinkedHashMap<String, String> labels,
+            NBLabeledElement element,
             LinkedHashMap<String, String> details) {
-        setLabels(labels);
+        setElement(element);
         setSession(session);
         setLayer(layer);
         setStart(start);
         setEnd(end);
         setDetails(details);
-        labels.put("appname", "nosqlbench");
+    }
+
+    private void setElement(NBLabeledElement element) {
+        this.element = element;
     }
 
     public void setSession(String sessionName) {
         this.session = sessionName;
-        this.labels.put("session", sessionName);
     }
 
     public void setStart(long intervalStart) {
         this.start = intervalStart;
-        this.labels.put("span", getSpan().toString());
     }
 
     public void setEnd(long intervalEnd) {
         this.end = intervalEnd;
-        this.labels.put("span", getSpan().toString());
-    }
-
-    public void setLabels(Map<String, String> labels) {
-        this.labels = labels;
     }
 
     public void setLayer(Layer layer) {
         this.layer = layer;
-        this.labels.put("layer", layer.toString());
     }
 
     public void setDetails(Map<String, String> details) {
         this.details = details;
     }
 
-    @Override
-    public String getSession() {
-        return session;
-    }
-
     @Override
     public long getStart() {
         return start;
@@ -115,11 +107,8 @@ public class MutableAnnotation implements Annotation {
     }
 
     @Override
-    public Map<String, String> getLabels() {
+    public NBLabels getLabels() {
-//        if (!labels.containsKey("span")) {
+        return element.getLabels();
-//            labels.put("span",getSpan().toString());
-//        }
-        return labels;
     }
 
     @Override
@@ -130,7 +119,6 @@ public class MutableAnnotation implements Annotation {
     @Override
     public String toString() {
         StringBuilder sb = new StringBuilder();
-        sb.append("session: ").append(getSession()).append("\n");
 
         ZonedDateTime startTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(getStart()), zoneid);
         ZonedDateTime endTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(getStart()), zoneid);
@@ -144,17 +132,17 @@ public class MutableAnnotation implements Annotation {
         }
         sb.append("]\n");
 
-        sb.append("span:").append(getSpan()).append("\n");
+        sb.append("span:").append(getTemporal()).append("\n");
         sb.append("details:\n");
         formatMap(sb, getDetails());
         sb.append("labels:\n");
-        formatMap(sb, getLabels());
+        formatMap(sb, getLabels().asMap());
         return sb.toString();
     }
 
     private void formatMap(StringBuilder sb, Map<String, String> details) {
         details.forEach((k, v) -> {
-            sb.append(" ").append(k).append(": ");
+            sb.append(" ").append(k).append(":");
             if (v.contains("\n")) {
                 sb.append("\n");
 
@@ -164,7 +152,7 @@ public class MutableAnnotation implements Annotation {
                 }
 //                Arrays.stream(lines).sequential().map(s -> "  "+s+"\n").forEach(sb::append);
             } else {
-                sb.append(v).append("\n");
+                sb.append(" ").append(v).append("\n");
             }
         });
     }
@@ -173,8 +161,8 @@ public class MutableAnnotation implements Annotation {
         return this;
     }
 
-    public Span getSpan() {
+    public Temporal getTemporal() {
-        return (getStart() == getEnd()) ? Span.instant : Span.interval;
+        return (getStart() == getEnd()) ? Temporal.instant : Temporal.interval;
     }
 
     public String asJson() {

nb-api/src/main/java/io/nosqlbench/api/annotations/Temporal.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 
 package io.nosqlbench.api.annotations;
 
-public enum Span {
+public enum Temporal {
     /**
      * A span of time of size zero.
      */

nb-api/src/main/java/io/nosqlbench/api/config/MapLabels.java

@@ -16,33 +16,65 @@
 
 package io.nosqlbench.api.config;
 
+import org.jetbrains.annotations.NotNull;
+
 import java.util.*;
 import java.util.function.Function;
+import java.util.regex.Pattern;
 
 public class MapLabels implements NBLabels {
     private final Map<String,String> labels;
+    private String[] instanceFields = new String[0];
 
-    public MapLabels(final Map<String, String> labels) {
+    public MapLabels(final Map<String, String> labels, String... instanceFields) {
+        verifyValidNamesAndValues(labels);
+//        verifyValidValues(labels);
         this.labels = Collections.unmodifiableMap(labels);
+        this.instanceFields = instanceFields;
     }
 
-    public MapLabels(final Map<String,String> parentLabels, final Map<String,String> childLabels) {
+
-        final Map<String,String> combined = new LinkedHashMap<>();
+    public MapLabels(final Map<String,String> parentLabels, final Map<String,String> childLabels, String... instanceFields) {
-        parentLabels.forEach(combined::put);
+        final Map<String, String> combined = new LinkedHashMap<>(parentLabels);
         childLabels.forEach((k,v) -> {
             if (combined.containsKey(k))
                 throw new RuntimeException("Can't overlap label keys (for instance " + k + ") between parent and child elements. parent:" + parentLabels + ", child:" + childLabels);
             combined.put(k,v);
         });
+        verifyValidNamesAndValues(combined);
+//        verifyValidValues(combined);
+        this.instanceFields = instanceFields;
         labels=Collections.unmodifiableMap(combined);
     }
 
+    private final Pattern validNamesPattern = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]+");
+    private void verifyValidNamesAndValues(Map<String, String> labels) {
+        labels.forEach((label,value) -> {
+            if (!validNamesPattern.matcher(label).matches()) {
+                throw new RuntimeException("Invalid label name '" + label + "', only a-z,A-Z,_ are allowed as the initial character, and a-z,A-Z,0-9,_ are allowed after.");
+            }
+//            if (!validNamesPattern.matcher(value).matches()) {
+//                throw new RuntimeException("Invalid label value '" + value + "', only a-z,A-Z,_ are allowed as the initial character, and a-z,A-Z,0-9,_ are allowed after.");
+//            }
+        });
+    }
+
+    private void verifyValidValues(Map<String, String> labels) {
+        for (String value : labels.values()) {
+            if (!validNamesPattern.matcher(value).matches()) {
+                throw new RuntimeException("Invalid label value '" + value + "', only a-z,A-Z,_ are allowed as the initial character, and a-z,A-Z,0-9,_ are allowed after.");
+            }
+        }
+    }
+
+
+
     @Override
     public String linearizeValues(final char delim, final String... included) {
         final StringBuilder sb = new StringBuilder();
         final List<String> includedNames = new ArrayList<>();
         if (0 < included.length) Collections.addAll(includedNames, included);
-        else this.labels.keySet().forEach(includedNames::add);
+        else includedNames.addAll(this.labels.keySet());
 
         for (String includedName : includedNames) {
             final boolean optional= includedName.startsWith("[") && includedName.endsWith("]");
@@ -65,7 +97,7 @@ public class MapLabels implements NBLabels {
 
         final List<String> includedNames = new ArrayList<>();
         if (0 < included.length) Collections.addAll(includedNames, included);
-        else this.labels.keySet().forEach(includedNames::add);
+        else includedNames.addAll(this.labels.keySet());
         String rawName = null;
         if (null != bareName) {
             rawName = this.labels.get(bareName);
@@ -92,14 +124,28 @@ public class MapLabels implements NBLabels {
     }
 
     @Override
-    public NBLabels and(final String... labelsAndValues) {
+    public MapLabels andTypes(final String... labelsAndValues) {
-        if (0 != (labelsAndValues.length % 2))
+        final Map<String, String> childLabels = getStringStringMap(labelsAndValues);
-            throw new RuntimeException("Must provide even number of keys and values: " + Arrays.toString(labelsAndValues));
-        final Map<String,String> childLabels = new LinkedHashMap<>();
-        for (int i = 0; i < labelsAndValues.length; i+=2) childLabels.put(labelsAndValues[i], labelsAndValues[i + 1]);
         return new MapLabels(labels,childLabels);
     }
 
+    @Override
+    public MapLabels and(NBLabels labels) {
+        return new MapLabels(this.labels,labels.asMap(), concat(this.instanceFields,labels.getInstanceFields()));
+    }
+
+
+    @Override
+    public MapLabels andInstances(final String... labelsAndValues) {
+        final Map<String, String> childLabels = getStringStringMap(labelsAndValues);
+        String[] childInstanceFields = getNamesArray(labelsAndValues);
+        return new MapLabels(this.labels,childLabels,concat(this.instanceFields,getNamesArray(labelsAndValues)));
+    }
+
+    @Override
+    public MapLabels andInstances(Map<String, String> instanceLabelsAndValues) {
+        return new MapLabels(this.labels,instanceLabelsAndValues,instanceLabelsAndValues.keySet().toArray(new String[0]));
+    }
     @Override
     public NBLabels modifyName(final String nameToModify, final Function<String, String> transform) {
         if (!this.labels.containsKey(nameToModify))
@@ -134,7 +180,7 @@ public class MapLabels implements NBLabels {
     }
 
     @Override
-    public String only(final String name) {
+    public String valueOf(final String name) {
         if (!this.labels.containsKey(name))
             throw new RuntimeException("The specified key does not exist: '" + name + '\'');
         final String only = labels.get(name);
@@ -148,7 +194,55 @@ public class MapLabels implements NBLabels {
     }
 
     @Override
-    public NBLabels and(final Map<String, String> moreLabels) {
+    public NBLabels onlyTypes() {
+        Map<String,String> typesOnlyMap = new LinkedHashMap<>(this.labels);
+        for (String instanceField : this.instanceFields) {
+            typesOnlyMap.remove(instanceField);
+        }
+        return new MapLabels(typesOnlyMap);
+    }
+
+    @Override
+    public NBLabels onlyInstances() {
+        Map<String,String> instancesOnlyMap = new LinkedHashMap<>();
+        for (String instanceField : this.instanceFields) {
+            instancesOnlyMap.put(instanceField,this.labels.get(instanceField));
+        }
+        return new MapLabels(instancesOnlyMap);
+    }
+
+    @Override
+    public String[] getInstanceFields() {
+        return instanceFields;
+    }
+
+    @Override
+    public NBLabels andTypes(final Map<String, String> moreLabels) {
         return new MapLabels(this.labels, moreLabels);
     }
+
+    private String[] concat(String[] a, String[] b) {
+        String[] c = new String[a.length+b.length];
+        System.arraycopy(a,0,c,0,a.length);
+        System.arraycopy(b,0,c,a.length,b.length);
+        return c;
+    }
+
+    private static String[] getNamesArray(final String... labelsAndValues) {
+        String[] keys = new String[labelsAndValues.length>>1];
+        for (int i = 0; i < keys.length; i++) {
+            keys[i]=labelsAndValues[i<<1];
+        }
+        return keys;
+    }
+    @NotNull
+    private static Map<String, String> getStringStringMap(String[] labelsAndValues) {
+        if (0 != (labelsAndValues.length % 2))
+            throw new RuntimeException("Must provide even number of keys and values: " + Arrays.toString(labelsAndValues));
+        final Map<String,String> childLabels = new LinkedHashMap<>();
+        for (int i = 0; i < labelsAndValues.length; i+=2) childLabels.put(labelsAndValues[i], labelsAndValues[i + 1]);
+        return childLabels;
+    }
+
+
 }

nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java

@@ -30,7 +30,6 @@ import java.io.File;
 import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.ServiceLoader;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
@@ -106,7 +105,8 @@ public class ActivityMetrics {
      * @return the timer, perhaps a different one if it has already been registered
      */
     public static Timer timer(NBLabeledElement parent, String metricFamilyName, int hdrdigits) {
-        final NBLabels labels = parent.getLabels().and("name",metricFamilyName);
+        final NBLabels labels = parent.getLabels().andTypes("name",sanitize(metricFamilyName));
+
 
         Timer registeredTimer = (Timer) register(labels, () ->
             new NBMetricTimer(labels,
@@ -134,7 +134,7 @@ public class ActivityMetrics {
      * @return the histogram, perhaps a different one if it has already been registered
      */
     public static Histogram histogram(NBLabeledElement labeled, String metricFamilyName, int hdrdigits) {
-        final NBLabels labels = labeled.getLabels().and("name", metricFamilyName);
+        final NBLabels labels = labeled.getLabels().andTypes("name", sanitize(metricFamilyName));
         return (Histogram) register(labels, () ->
             new NBMetricHistogram(
                 labels,
@@ -157,7 +157,7 @@ public class ActivityMetrics {
      * @return the counter, perhaps a different one if it has already been registered
      */
     public static Counter counter(NBLabeledElement parent, String metricFamilyName) {
-        final NBLabels labels = parent.getLabels().and("name",metricFamilyName);
+        final NBLabels labels = parent.getLabels().andTypes("name",metricFamilyName);
         return (Counter) register(labels, () -> new NBMetricCounter(labels));
     }
 
@@ -173,7 +173,7 @@ public class ActivityMetrics {
      * @return the meter, perhaps a different one if it has already been registered
      */
     public static Meter meter(NBLabeledElement parent, String metricFamilyName) {
-        final NBLabels labels = parent.getLabels().and("name",metricFamilyName);
+        final NBLabels labels = parent.getLabels().andTypes("name",sanitize(metricFamilyName));
         return (Meter) register(labels, () -> new NBMetricMeter(labels));
     }
 
@@ -191,7 +191,7 @@ public class ActivityMetrics {
 
     @SuppressWarnings("unchecked")
     public static <T> Gauge<T> gauge(NBLabeledElement parent, String metricFamilyName, Gauge<T> gauge) {
-        final NBLabels labels = parent.getLabels().and("name",metricFamilyName);
+        final NBLabels labels = parent.getLabels().andTypes("name",sanitize(metricFamilyName));
 
         return (Gauge<T>) register(labels, () -> new NBMetricGauge(labels,gauge));
     }
@@ -342,4 +342,16 @@ public class ActivityMetrics {
             .forEach(get()::remove);
     }
 
+    public static String sanitize(String word) {
+        String sanitized = word;
+        sanitized = sanitized.replaceAll("\\..+$", "");
+        sanitized = sanitized.replaceAll("-","_");
+        sanitized = sanitized.replaceAll("[^a-zA-Z0-9_]+", "");
+
+        if (!word.equals(sanitized)) {
+            logger.warn("The identifier or value '" + word + "' was sanitized to '" + sanitized + "' to be compatible with monitoring systems. You should probably change this to make diagnostics easier.");
+        }
+        return sanitized;
+    }
+
 }

nb-api/src/test/java/io/nosqlbench/api/config/MapLabelsTest.java

@@ -21,14 +21,40 @@ import org.junit.jupiter.api.Test;
 import java.util.Map;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 public class MapLabelsTest {
 
     @Test
     public void testLinearizeValues() {
-        final MapLabels l1 = new MapLabels(Map.of("key-a", "value-a", "key-c", "value-c"));
+        final MapLabels l1 = new MapLabels(Map.of("key_a", "value_a", "key_c", "value_c"));
-        final String result = l1.linearizeValues('_', "key-a", "[key-b]", "key-c");
+        final String result = l1.linearizeValues('_', "key_a", "[key_b]", "key_c");
-        assertThat(result).isEqualTo("value-a_value-c");
+        assertThat(result).isEqualTo("value_a_value_c");
     }
 
+    @Test
+    public void testInstances() {
+        final MapLabels l1 = new MapLabels(Map.of("key_a", "value_a", "key_c", "value_c"),"key_c");
+        NBLabels typesOnly = l1.onlyTypes();
+        assertThat(typesOnly.linearizeValues()).isEqualTo("value_a");
+    }
+
+    @Test
+    public void testInstanceCombination() {
+        final MapLabels l1 = new MapLabels(Map.of("key_a", "value_a"),Map.of("key_c", "value_c"),"key_c");
+        final MapLabels l2 = new MapLabels(Map.of("key_dog", "value_dog"),Map.of( "key_cat", "value_cat"),"key_dog");
+        final MapLabels l3 = l1.and(l2);
+        assertThat(l3.linearizeValues()).matches("value_a.value_c.value_dog.value_cat");
+
+        assertThat(l3.onlyTypes().linearizeValues()).matches("value_a.value_cat");
+        assertThat(l3.onlyInstances().linearizeValues()).matches("value_c.value_dog");
+    }
+
+
+    @Test
+    public void testInvalidCharacters() {
+        assertThatThrownBy(() -> new MapLabels(Map.of("a-b","c-d"))).isOfAnyClassIn(RuntimeException.class);
+    }
+
+
 }

nb-api/src/test/java/io/nosqlbench/nb/api/annotations/AnnotationBuilderTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ package io.nosqlbench.nb.api.annotations;
 
 import io.nosqlbench.api.annotations.Annotation;
 import io.nosqlbench.api.annotations.Layer;
+import io.nosqlbench.api.config.NBLabeledElement;
 import org.junit.jupiter.api.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -30,36 +31,31 @@ public class AnnotationBuilderTest {
     public void testBasicAnnotation() {
 
         Annotation an1 = Annotation.newBuilder()
-                .session("test-session")
+            .element(NBLabeledElement.forKV("test_element","value"))
                 .at(time)
                 .layer(Layer.Scenario)
-                .label("labelka", "labelvb")
-                .label("labelkc", "labelvd")
                 .detail("detailk1", "detailv1")
                 .detail("detailk2", "detailv21\ndetailv22")
                 .detail("detailk3", "v1\nv2\nv3\n")
                 .build();
 
         String represented = an1.toString();
-        assertThat(represented).isEqualTo("session: test-session\n" +
+        assertThat(represented).isEqualTo(
-                "[2020-09-13T12:26:40Z]\n" +
+                """
-                "span:instant\n" +
+                [2020-09-13T12:26:40Z]
-                "details:\n" +
+                span:instant
-                " detailk1: detailv1\n" +
+                details:
-                " detailk2: \n" +
+                 detailk1: detailv1
-                "  detailv21\n" +
+                 detailk2:
-                "  detailv22\n" +
+                  detailv21
-                " detailk3: \n" +
+                  detailv22
-                "  v1\n" +
+                 detailk3:
-                "  v2\n" +
+                  v1
-                "  v3\n" +
+                  v2
-                "labels:\n" +
+                  v3
-                " layer: Scenario\n" +
+                labels:
-                " labelka: labelvb\n" +
+                 test_element: value
-                " labelkc: labelvd\n" +
+                """);
-                " session: test-session\n" +
-                " span: instant\n" +
-                " appname: nosqlbench\n");
 
     }