add missing files from previous commit

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/RunnableOp.java

@@ -0,0 +1,4 @@
+package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes;
+
+public interface RunnableOp extends Op, Runnable {
+}

adapters-api/src/main/java/io/nosqlbench/engine/api/templating/DriverAdapterDecorators.java

@@ -0,0 +1,15 @@
+package io.nosqlbench.engine.api.templating;
+
+/**
+ * This type simply captures (by extension) any optional decorator
+ * interfaces which may be implemented by a {@link io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter}.
+ * Thus, it is mostly for documentation.
+ *
+ * Decorator interfaces are used within NoSQLBench where implementations are truly optional,
+ * and thus would cloud the view of a developer implementing strictly to requirements.
+ *
+ * You can find any such decorator interfaces specific to driver adapters by looking for
+ * all implementations of this type.
+ */
+public interface DriverAdapterDecorators {
+}

adapters-api/src/main/java/io/nosqlbench/engine/api/templating/OpTemplateSupplier.java

@@ -0,0 +1,18 @@
+package io.nosqlbench.engine.api.templating;
+
+import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
+import io.nosqlbench.nb.api.config.standard.NBConfiguration;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * An Op Template Supplier can provide its own source of op templates instead
+ * of relying on the built-in mechanism. By default, the built-in mechanism
+ * will read op definitions from parameters first, then any ops (statements)
+ * from yaml files provided in the workload= or yaml= activity parameters.
+ */
+public interface OpTemplateSupplier extends DriverAdapterDecorators {
+
+    Optional<List<OpTemplate>> loadOpTemplates(NBConfiguration cfg);
+}

adapters-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedOp.java

@@ -0,0 +1,492 @@
+package io.nosqlbench.engine.api.templating;
+
+import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
+import io.nosqlbench.engine.api.templating.binders.ArrayBinder;
+import io.nosqlbench.engine.api.templating.binders.ListBinder;
+import io.nosqlbench.engine.api.templating.binders.OrderedMapBinder;
+import io.nosqlbench.nb.api.config.fieldreaders.DynamicFieldReader;
+import io.nosqlbench.nb.api.config.fieldreaders.StaticFieldReader;
+import io.nosqlbench.nb.api.config.standard.*;
+import io.nosqlbench.nb.api.errors.BasicError;
+import io.nosqlbench.nb.api.errors.OpConfigError;
+import io.nosqlbench.virtdata.core.bindings.DataMapper;
+import io.nosqlbench.virtdata.core.bindings.VirtData;
+import io.nosqlbench.virtdata.core.templates.BindPoint;
+import io.nosqlbench.virtdata.core.templates.CapturePoint;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
+import io.nosqlbench.virtdata.core.templates.StringBindings;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.*;
+import java.util.function.Function;
+import java.util.function.LongFunction;
+
+/**
+ * Parse an OpTemplate into a ParsedOp, which can dispense object maps
+ */
+public class ParsedOp implements LongFunction<Map<String, ?>>, StaticFieldReader, DynamicFieldReader {
+
+    private final static Logger logger = LogManager.getLogger(ParsedOp.class);
+
+    /**
+     * The fields which are statically assigned
+     **/
+    private final Map<String, Object> statics = new LinkedHashMap<>();
+
+    /**
+     * The fields which are dynamic, and must be realized via functions.
+     * This map contains keys which identify the field names, and values, which may be null or undefined.
+     */
+    private final Map<String, LongFunction<?>> dynamics = new LinkedHashMap<>();
+
+    /**
+     * The names of payload values in the result of the operation which should be saved.
+     * The keys in this map represent the name of the value as it would be found in the native
+     * representation of a result. If the values are defined, then each one represents the name
+     * that the found value should be saved as instead of the original name.
+     */
+    private final List<List<CapturePoint>> captures = new ArrayList<>();
+    private final int mapsize;
+
+    /**
+     * A prototype of the fully generated map, to be used as the starting point
+     * when rendering the full map with dynamic values.
+     */
+    private final LinkedHashMap<String, Object> protomap = new LinkedHashMap<>();
+    private final OpTemplate ot;
+    private final NBConfiguration activityCfg;
+
+    /**
+     * Create a parsed command from an Op template.
+     *
+     * @param ot   An OpTemplate representing an operation to be performed in a native driver.
+     * @param activityCfg The activity configuration, used for reading config parameters
+     */
+    public ParsedOp(OpTemplate ot, NBConfiguration activityCfg) {
+        this(ot, activityCfg, List.of());
+    }
+
+    /**
+     * Create a parsed command from an Op template. This version is exactly like
+     * {@link ParsedOp (OpTemplate,NBConfiguration)} except that it allows
+     * preprocessors. Preprocessors are all applied to the the op template before
+     * it is applied to the parsed command fields, allowing you to combine or destructure
+     * fields from more tha one representation into a single canonical representation
+     * for processing.
+     *
+     * @param opTemplate            The OpTemplate as provided by a user via YAML, JSON, or API (data structure)
+     * @param activityCfg          The activity configuration, used to resolve nested config parameters
+     * @param preprocessors Map->Map transformers.
+     */
+    public ParsedOp(OpTemplate opTemplate, NBConfiguration activityCfg, List<Function<Map<String, Object>, Map<String, Object>>> preprocessors) {
+        this.ot = opTemplate;
+        this.activityCfg = activityCfg;
+
+        Map<String, Object> map = opTemplate.getOp().orElseThrow();
+        for (Function<Map<String, Object>, Map<String, Object>> preprocessor : preprocessors) {
+            map = preprocessor.apply(map);
+        }
+
+        applyTemplateFields(map, opTemplate.getBindings());
+        mapsize = statics.size() + dynamics.size();
+    }
+
+    // For now, we only allow bind points to reference bindings, not other op template
+    // fields. This seems like the saner and less confusing approach, so implementing
+    // op field references should be left until it is requested if at all
+    private void applyTemplateFields(Map<String, Object> map, Map<String, String> bindings) {
+        map.forEach((k, v) -> {
+            if (v instanceof CharSequence) {
+                ParsedTemplate pt = ParsedTemplate.of(((CharSequence) v).toString(), bindings);
+                this.captures.add(pt.getCaptures());
+                switch (pt.getType()) {
+                    case literal:
+                        statics.put(k, ((CharSequence) v).toString());
+                        protomap.put(k, ((CharSequence) v).toString());
+                        break;
+                    case bindref:
+                        String spec = pt.asBinding().orElseThrow().getBindspec();
+                        Optional<DataMapper<Object>> mapper = VirtData.getOptionalMapper(spec);
+                        dynamics.put(k, mapper.orElseThrow());
+                        protomap.put(k, null);
+                        break;
+                    case concat:
+                        StringBindings sb = new StringBindings(pt);
+                        dynamics.put(k, sb);
+                        protomap.put(k, null);
+                        break;
+                }
+            } else {
+                // Eventually, nested and mixed static dynamic structure could be supported, but
+                // it would be complex to implement and also not that efficient, so let's just copy
+                // structure for now
+                statics.put(k, v);
+                protomap.put(k, v);
+            }
+        });
+
+    }
+
+    public String getName() {
+        return ot.getName();
+    }
+
+    public Map<String, Object> getStaticPrototype() {
+        return statics;
+    }
+
+    public Map<String, LongFunction<?>> getDynamicPrototype() {
+        return dynamics;
+    }
+
+    @Override
+    public Map<String, Object> apply(long value) {
+        LinkedHashMap<String, Object> map = new LinkedHashMap<>(protomap);
+        dynamics.forEach((k, v) -> {
+            map.put(k, v.apply(value));
+        });
+        return map;
+    }
+
+    @Override
+    public boolean isDefinedDynamic(String field) {
+        return dynamics.containsKey(field);
+    }
+
+
+    /**
+     * @param field The field name to look for in the static field map.
+     * @return true if and only if the named field is present in the static field map.
+     */
+    public boolean isStatic(String field) {
+        return statics.containsKey(field);
+    }
+
+    public boolean isStatic(String field, Class<?> type) {
+        return statics.containsKey(field) && type.isAssignableFrom(field.getClass());
+    }
+
+    /**
+     * @param fields Names of fields to look for in the static field map.
+     * @return true if and only if all provided field names are present in the static field map.
+     */
+    @Override
+    public boolean isDefined(String... fields) {
+        for (String field : fields) {
+            if (!statics.containsKey(field)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Get the static value for the provided name, cast to the required type.
+     *
+     * @param field    Name of the field to get
+     * @param classOfT The type of the field to return. If actual type is not compatible to a cast to this type, then a
+     *                 casting error will be thrown.
+     * @param <T>      The parameter type of the return type, used at compile time only to qualify asserted return type
+     * @return A value of type T, or null
+     */
+    @Override
+    public <T> T getStaticValue(String field, Class<T> classOfT) {
+        return (T) statics.get(field);
+    }
+
+    /**
+     * Get the static value for the provided name, cast to the required type, where the type is inferred
+     * from the calling context.
+     *
+     * @param field Name of the field to get
+     * @param <T>   The parameter type of the return type. used at compile time only to quality return type.
+     * @return A value of type T, or null
+     */
+    @Override
+    public <T> T getStaticValue(String field) {
+        return (T) statics.get(field);
+    }
+
+    public Optional<ParsedTemplate> getStmtAsTemplate() {
+        return ot.getParsed();
+    }
+
+    /**
+     * Get the named static field value, or return the provided default, but throw an exception if
+     * the named field is dynamic.
+     *
+     * @param name         The name of the field value to return.
+     * @param defaultValue A value to return if the named value is not present in static nor dynamic fields.
+     * @param <T>          The type of the field to return.
+     * @return The value
+     * @throws RuntimeException if the field name is only present in the dynamic fields.
+     */
+    @Override
+    public <T> T getStaticValueOr(String name, T defaultValue) {
+        if (statics.containsKey(name)) {
+            return (T) statics.get(name);
+        } else if (dynamics.containsKey(name)) {
+            throw new BasicError("static field '" + name + "' was defined dynamically. This may be supportable if the driver developer" +
+                "updates the op mapper to support this field as a dynamic field, but it is not yet supported.");
+        } else {
+            return defaultValue;
+        }
+    }
+
+    /**
+     * Get the specified parameter by the user using the defined field which is closest to the op
+     * template. This is the standard way of getting parameter values which can be specified at the
+     * op template, op param, or activity level.
+     *
+     * @param name         The name of the configuration param
+     * @param defaultValue the default value to return if the value is not defined anywhere in
+     *                     (op fields, op params, activity params)
+     * @param <T>          The type of the value to return
+     * @return A configuration value
+     * @throws io.nosqlbench.nb.api.config.standard.NBConfigError if the named field is defined dynamically,
+     *                                                            as in this case, it is presumed that the parameter is not supported unless it is defined statically.
+     */
+    public <T> T getStaticConfigOr(String name, T defaultValue) {
+        if (statics.containsKey(name)) {
+            return NBTypeConverter.convertOr(statics.get(name), defaultValue);
+        } else if (ot.getParams().containsKey(name)) {
+            return NBTypeConverter.convertOr(ot.getParams().get(name), defaultValue);
+        } else if (activityCfg.getMap().containsKey(name)) {
+            return NBTypeConverter.convertOr(activityCfg.get("name"), defaultValue);
+        } else if (dynamics.containsKey(name)) {
+            throw new NBConfigError("static config field '" + name + "' was defined dynamically. This may be supportable if the driver developer" +
+                "updates the op mapper to support this field as a dynamic field, but it is not yet supported.");
+        } else {
+            return defaultValue;
+        }
+    }
+
+    public <T> Optional<T> getOptionalStaticConfig(String name, Class<T> type) {
+        if (statics.containsKey(name)) {
+            return Optional.of(NBTypeConverter.convert(statics.get(name), type));
+        } else if (ot.getParams().containsKey(name)) {
+            return Optional.of(NBTypeConverter.convert(ot.getParams().get(name), type));
+        } else if (activityCfg.getMap().containsKey(name)) {
+            return Optional.of(NBTypeConverter.convert(activityCfg.get("name"), type));
+        } else if (dynamics.containsKey("name")) {
+            throw new NBConfigError("static config field '" + name + "' was defined dynamically. This may be supportable if the driver developer" +
+                "updates the op mapper to support this field as a dynamic field, but it is not yet supported.");
+        } else {
+            return Optional.empty();
+        }
+    }
+
+
+    /**
+     * Works exactly like {@link #getStaticConfigOr(String, Object)}, except that dynamic values
+     * at the op field level will be generated on a per-input basis. This is a shortcut method for
+     * allowing configuration values to be accessed dynamically where it makes sense.
+     */
+    public <T> T getConfigOr(String name, T defaultValue, long input) {
+        if (statics.containsKey(name)) {
+            return NBTypeConverter.convertOr(statics.get(name), defaultValue);
+        } else if (dynamics.containsKey(name)) {
+            return NBTypeConverter.convertOr(dynamics.get(name).apply(input), defaultValue);
+        } else if (ot.getParams().containsKey(name)) {
+            return NBTypeConverter.convertOr(ot.getParams().get(name), defaultValue);
+        } else if (activityCfg.getMap().containsKey(name)) {
+            return NBTypeConverter.convertOr(activityCfg.get("name"), defaultValue);
+        } else return defaultValue;
+
+    }
+
+
+    /**
+     * Return an optional value for the named field. This is an {@link Optional} form of {@link #getStaticValue}.
+     *
+     * @param field    Name of the field to get
+     * @param classOfT The type of field to return. If the actual type is not compatible to a cast to this type,
+     *                 then a casting error will be thrown.
+     * @param <T>      The parameter type of the return
+     * @return An optional value, empty unless the named value is defined in the static field map.
+     */
+    @Override
+    public <T> Optional<T> getOptionalValue(String field, Class<T> classOfT) {
+        return Optional.ofNullable(getStaticValue(field, classOfT));
+    }
+
+    public <T> Optional<T> getStaticValueOptionally(String field) {
+        return Optional.ofNullable(getStaticValue(field));
+    }
+
+    /**
+     * Get the named field value for a given long input. This uses parameter type inference -- The casting
+     * to the return type will be based on the type of any assignment or casting on the caller's side.
+     * Thus, if the actual type is not compatable to a cast to the needed return type, a casting error will
+     * be thrown.
+     *
+     * @param field The name of the field to get.
+     * @param input The seed value, or cycle value for which to generate the value.
+     * @param <T>   The parameter type of the returned value. Inferred from usage context.
+     * @return The value.
+     */
+    @Override
+    public <T> T get(String field, long input) {
+        if (statics.containsKey(field)) {
+            return (T) statics.get(field);
+        }
+        if (dynamics.containsKey(field)) {
+            return (T) dynamics.get(field).apply(input);
+        }
+        return null;
+    }
+
+    public Set<String> getDefinedNames() {
+        HashSet<String> nameSet = new HashSet<>(statics.keySet());
+        nameSet.addAll(dynamics.keySet());
+        return nameSet;
+    }
+
+    public <V> LongFunction<V> getAsFunction(String name, Class<? extends V> type) {
+        if (isStatic(name)) {
+            V value = getStaticValue(name);
+            return (cycle) -> value;
+        } else if (isDefinedDynamic(name)) {
+            Object testValue = dynamics.get(name).apply(0L);
+            if (type.isAssignableFrom(testValue.getClass())) {
+                return (LongFunction<V>) dynamics.get(name);
+            } else {
+                throw new OpConfigError(
+                    "function for '" + name + "' yielded a " + testValue.getClass().getCanonicalName()
+                        + " type, which is not assignable to " + type.getClass().getCanonicalName() + "'");
+            }
+        } else {
+            throw new OpConfigError("No op field named '" + name + "' was found. If this field has a reasonable" +
+                " default value, consider using getAsFunctionOr(...) and documenting the default.");
+        }
+    }
+
+    @Override
+    public <V> LongFunction<V> getAsFunctionOr(String name, V defaultValue) {
+        if (isStatic(name)) {
+            V value = getStaticValue(name);
+            return l -> value;
+        } else if (isDefinedDynamic(name)) {
+            return l -> get(name, l);
+        } else {
+            return l -> defaultValue;
+        }
+    }
+
+    public <V> LongFunction<V> getAsCachedFunctionOr(String fieldname, String defaultValue, Function<String, V> init) {
+        if (isStatic(fieldname)) {
+            V value = getStaticValue(fieldname);
+            if (value instanceof String) {
+                V defaultObject = init.apply((String) value);
+                return l -> defaultObject;
+            } else {
+                throw new OpConfigError("Unable to compose string to object cache with non-String value of type " + defaultValue.getClass().getCanonicalName());
+            }
+        } else if (isDefinedDynamic(fieldname)) {
+            LongFunction<V> f = l -> get(fieldname, l);
+            V testValue = f.apply(0);
+            if (testValue instanceof String) {
+                LongFunction<String> fs = l -> (String) get(fieldname, l);
+                ObjectCache<V> oc = new ObjectCache<>(init);
+                return l -> oc.apply(fs.apply(l));
+            } else {
+                throw new OpConfigError(
+                    "Unable to compose string func to obj cache with non-String function of type " + f.getClass().getCanonicalName()
+                );
+            }
+        } else {
+            throw new OpConfigError(
+                "Unable to compose string func to obj cache with no defined static nor dynamic field named " + fieldname
+            );
+        }
+    }
+
+    public boolean isDefined(String field) {
+        return statics.containsKey(field) || dynamics.containsKey(field);
+    }
+
+    @Override
+    public boolean isDefined(String field, Class<?> type) {
+        if (statics.containsKey(field)) {
+            if (type.isAssignableFrom(statics.get(field).getClass())) {
+                return true;
+            } else {
+                throw new OpConfigError("field " + field + " was defined, but not as the requested type " + type.getCanonicalName());
+            }
+        } else if (dynamics.containsKey(field)) {
+            Object testObject = dynamics.get(field).apply(0L);
+            if (type.isAssignableFrom(testObject.getClass())) {
+                return true;
+            } else {
+                throw new OpConfigError("field " + field + " was defined as a function, but not one that returns the" +
+                    " requested type " + testObject.getClass().getCanonicalName());
+            }
+        }
+        return false;
+    }
+
+    public boolean isDefinedAll(String... fields) {
+        for (String field : fields) {
+            if (!statics.containsKey(field) && !dynamics.containsKey(field)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public void assertDefinedStatic(String... fields) {
+        for (String field : fields) {
+            if (!statics.containsKey(field)) {
+                Set<String> missing = new HashSet<>();
+                for (String readoutfield : fields) {
+                    if (!statics.containsKey(readoutfield)) {
+                        missing.add(readoutfield);
+                    }
+                }
+                throw new OpConfigError("Fields " + missing + " are required to be defined with static values for this type of operation.");
+            }
+        }
+    }
+
+    public LongFunction<List<Object>> newListBinder(String... fields) {
+        return new ListBinder(this, fields);
+    }
+
+    public LongFunction<List<Object>> newListBinder(List<String> fields) {
+        return new ListBinder(this, fields);
+    }
+
+    public LongFunction<Map<String, Object>> newOrderedMapBinder(String... fields) {
+        return new OrderedMapBinder(this, fields);
+
+    }
+
+    public LongFunction<Object[]> newArrayBinder(String... fields) {
+        return new ArrayBinder(this, fields);
+    }
+
+    public LongFunction<Object[]> newArrayBinder(List<String> fields) {
+        return new ArrayBinder(this, fields);
+    }
+
+    public LongFunction<Object[]> newArrayBinderFromBindPoints(List<BindPoint> bindPoints) {
+        return new ArrayBinder(bindPoints);
+    }
+
+    public LongFunction<?> getMapper(String field) {
+        LongFunction<?> mapper = dynamics.get(field);
+        return mapper;
+    }
+
+    public int getSize() {
+        return this.mapsize;
+    }
+
+    public boolean isUndefined(String field) {
+        return !(statics.containsKey(field) || dynamics.containsKey(field));
+    }
+
+
+}

driver-directapi/src/main/java/io/nosqlbench/driver/direct/CallMapper.java

@@ -0,0 +1,11 @@
+package io.nosqlbench.driver.direct;
+
+import io.nosqlbench.engine.api.activityimpl.OpDispenser;
+
+public class CallMapper implements OpDispenser<DirectCall> {
+
+    @Override
+    public DirectCall apply(long value) {
+        return null;
+    }
+}

driver-directapi/src/main/java/io/nosqlbench/driver/direct/DirectCall.java

@@ -0,0 +1,26 @@
+package io.nosqlbench.driver.direct;
+
+import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op;
+
+import java.lang.reflect.Method;
+
+public class DirectCall implements Op,Runnable {
+    private final Method method;
+    private final Object[] args;
+    private final Object instance;
+
+    public DirectCall(Method method, Object instance, Object[] args) {
+        this.method = method;
+        this.instance = instance;
+        this.args = args;
+    }
+
+    @Override
+    public void run() {
+        try {
+            method.invoke(instance,args);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

driver-directapi/src/main/java/io/nosqlbench/driver/direct/DirectCallAdapter.java

@@ -0,0 +1,34 @@
+package io.nosqlbench.driver.direct;
+
+import io.nosqlbench.engine.api.activityimpl.OpMapper;
+import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
+import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
+import io.nosqlbench.nb.annotations.Service;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+
+/**
+ * This activity type driver allows you to dynamically map any available
+ * Java API which is exposed to the NoSQLBench runtime, executing methods
+ * on this API by name, (optionally) storing named results, and re-using
+ * these named results as arguments to subsequent calls.
+ *
+ * It supports static method dispatch, instance methods, and per-thread
+ * object scoping.
+ */
+@Service(value = DriverAdapter.class, selector = "directapi")
+public class DirectCallAdapter extends BaseDriverAdapter<DirectCall,Void> {
+
+    @Override
+    public List<Function<String, Optional<Map<String, Object>>>> getOpStmtRemappers() {
+        return List.of(new DirectCallStmtParser());
+    }
+
+    @Override
+    public OpMapper<DirectCall> getOpMapper() {
+        return new DirectOpMapper();
+    }
+}

driver-directapi/src/main/java/io/nosqlbench/driver/direct/DirectCallStmtParser.java

@@ -0,0 +1,44 @@
+package io.nosqlbench.driver.direct;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class DirectCallStmtParser implements Function<String, Optional<Map<String, Object>>> {
+
+    @Override
+    public Optional<Map<String, Object>> apply(String s) {
+        Pattern stmtPattern = Pattern.compile(
+            "(?<package>[a-z](\\.[a-z]+)?)?(?<class>[A-Z]\\w+)(\\.(?<staticfield>\\w+))?(\\.(?<method>\\w+))\\((?<args>.+)\\)"
+        );
+        Matcher matcher = stmtPattern.matcher(s);
+        if (matcher.matches()) {
+            LinkedHashMap<String, Object> map = new LinkedHashMap<>();
+            List.of("package","class","staticfield","method").forEach(n -> {
+               if (matcher.group(n)!=null) {
+                   map.put(n,matcher.group(n));
+               }
+            });
+            if (matcher.group("args")!=null) {
+                String args = matcher.group("args");
+                String[] argsplit = args.split(",");
+                for (int i = 0; i < argsplit.length; i++) {
+                    String val = argsplit[i];
+                    if (val.startsWith("\\") && val.endsWith("\\")) {
+                        val = val.substring(1,val.length()-2);
+                    } else if (val.startsWith("'") && val.endsWith("'")) {
+                        val = val.substring(1,val.length()-2);
+                    }
+                    map.put("_arg"+i,argsplit[i]);
+                }
+            }
+            return Optional.of(map);
+        } else {
+            return Optional.empty();
+        }
+    }
+}

driver-directapi/src/main/java/io/nosqlbench/driver/direct/DirectOpMapper.java

@@ -0,0 +1,79 @@
+package io.nosqlbench.driver.direct;
+
+import io.nosqlbench.engine.api.activityimpl.OpDispenser;
+import io.nosqlbench.engine.api.activityimpl.OpMapper;
+import io.nosqlbench.engine.api.templating.ParsedOp;
+import io.nosqlbench.nb.api.errors.OpConfigError;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.LongFunction;
+import java.util.stream.Collectors;
+
+public class DirectOpMapper implements OpMapper<DirectCall> {
+
+    @Override
+    public OpDispenser<DirectCall> apply(ParsedOp cmd) {
+
+        String pkg = cmd.getOptionalValue("package", String.class).orElse("java.lang");
+        String cls = cmd.getStaticValue("class");
+        String fq = pkg + "." + cls;
+        Class<?> clazz = null;
+        Object instance = null;
+        try {
+            clazz = Class.forName(fq);
+
+            Class<?> finalClazz = clazz;
+            Optional<Field> staticfield =
+                cmd.getOptionalValue("staticfield", String.class)
+                    .map(name -> {
+                        try {
+                            return finalClazz.getDeclaredField(name);
+                        } catch (Exception e) {
+                            e.printStackTrace();
+                        }
+                        return null;
+                    });
+            if (staticfield.isPresent()) {
+                Field sfield = staticfield.get();
+                if ((sfield.getModifiers() | Modifier.STATIC) > 0) {
+                    instance = sfield.get(null);
+                    clazz = instance.getClass();
+
+                } else {
+                    throw new OpConfigError("staticfield '" + cmd.getStaticValue("staticfield", String.class) + "' is not static");
+                }
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        String methodName = cmd.getStaticValue("method");
+
+        Map<String, Object> protomap = cmd.apply(0L);
+        List<Class<?>> protoargs = new ArrayList<>();
+        List<String> argnames = protomap.keySet().stream()
+            .filter(n -> n.startsWith("_"))
+            .collect(Collectors.toList());
+
+        LongFunction<List<Object>> argsbinder = cmd.newListBinder(argnames);
+        List<Object> args = argsbinder.apply(0L);
+        List<Class<?>> types = args.stream().map(Object::getClass).collect(Collectors.toList());
+
+        Class<?>[] argTypes = types.toArray(new Class<?>[0]);
+
+        Method method = null;
+        try {
+            method = clazz.getMethod(methodName, argTypes);
+            return new StaticMethodOpDispenser(method, instance, cmd.newArrayBinder(argnames));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+}

driver-directapi/src/main/java/io/nosqlbench/driver/direct/DynamicCallDispenser.java

@@ -0,0 +1,16 @@
+package io.nosqlbench.driver.direct;
+
+import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
+
+import java.util.function.LongFunction;
+
+public class DynamicCallDispenser implements LongFunction<DirectCall> {
+
+    public DynamicCallDispenser(OpTemplate opTemplate) {
+    }
+
+    @Override
+    public DirectCall apply(long value) {
+        return null;
+    }
+}

driver-directapi/src/main/java/io/nosqlbench/driver/direct/StaticMethodOpDispenser.java

@@ -0,0 +1,24 @@
+package io.nosqlbench.driver.direct;
+
+import io.nosqlbench.engine.api.activityimpl.OpDispenser;
+
+import java.lang.reflect.Method;
+import java.util.function.LongFunction;
+
+public class StaticMethodOpDispenser implements OpDispenser<DirectCall> {
+    private final LongFunction<Object[]> argsfunc;
+    private final Method method;
+    private final Object instance;
+
+    public StaticMethodOpDispenser(Method method, Object instance, LongFunction<Object[]> argsfunc) {
+        this.method = method;
+        this.instance = instance;
+        this.argsfunc = argsfunc;
+    }
+
+    @Override
+    public DirectCall apply(long value) {
+        Object[] args = argsfunc.apply(value);
+        return new DirectCall(method, instance, args);
+    }
+}

driver-directapi/src/main/java/io/nosqlbench/driver/direct/optypes/DynamicMethodCall.java

@@ -0,0 +1,73 @@
+package io.nosqlbench.driver.direct.optypes;
+
+import io.nosqlbench.virtdata.library.basics.core.threadstate.SharedState;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.Parameter;
+import java.util.Map;
+
+public class DynamicMethodCall implements Runnable {
+
+    private final static Logger logger = LogManager.getLogger(DynamicMethodCall.class);
+
+    private final Map<String, Object> callinfo;
+
+    public DynamicMethodCall(Map<String,Object> callinfo) {
+        this.callinfo = callinfo;
+    }
+
+    // At this point, class and method should have been set, and args optionally
+    private void callMethod() {
+        String className = callinfo.get("class").toString();
+        String methodName = callinfo.get("method").toString();
+
+        Class<?> clazz;
+        Method method;
+        try {
+            clazz = Class.forName(className);
+            method = clazz.getMethod(methodName);
+
+            Object instance = null;
+            if (!Modifier.isStatic(method.getModifiers())) {
+                if (callinfo.containsKey("instance")) {
+                    String instanceName = callinfo.get("instance").toString();
+                    instance = SharedState.tl_ObjectMap.get().get(instanceName);
+                }
+            }
+
+            Parameter[] parameters = method.getParameters();
+            Object[] args = new Object[parameters.length];
+            for (int i = 0; i < args.length; i++) {
+                String posname = "arg" + i;
+                if (callinfo.containsKey(posname)) {
+                    args[i] = callinfo.get(posname);
+                } else if (parameters[i].isNamePresent()) {
+                    String argname = parameters[i].getName();
+                    if (callinfo.containsKey(argname)) {
+                        args[i]=callinfo.get(argname);
+                    } else {
+                        throw new RuntimeException("could not find arg named '" + posname + "', nor '" + argname + "' in op template for method " + method.toGenericString());
+                    }
+                }
+            }
+            Object result = method.invoke(instance, args);
+
+            if (callinfo.containsKey("save")) {
+                String saveAs = callinfo.get("save").toString();
+                SharedState.tl_ObjectMap.get().put(saveAs,result);
+            }
+
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+
+    @Override
+    public void run() {
+        callMethod();
+    }
+}

driver-directapi/src/main/java/io/nosqlbench/driver/direct/optypes/ObjectCall.java

@@ -0,0 +1,5 @@
+package io.nosqlbench.driver.direct.optypes;
+
+public class ObjectCall {
+
+}

driver-directapi/src/main/resources/META-INF/services/javax.annotation.processing.Processor

@@ -0,0 +1 @@
+io.nosqlbench.nb.annotations.ServiceProcessor

driver-directapi/src/main/resources/directapi.md

@@ -0,0 +1,26 @@
+# Direct Driver
+
+This is a unique type of NoSQLBench driver which assumes no particular
+runtime API, instead relying on runtime reflection to find and invoke
+the methods as specified in the op template.
+
+Presently, therea re two
+```yaml
+statements:
+    - "java.lang.System.out.println(\"Testing\");"
+    - "System.out.println(\"Testing\");"
+    - op: "System.out.println(\"Testing\");"
+    - op:
+          class: java.lang.System
+          field: out
+          method: println
+          _arg0: Testing
+    - op:
+          class: java.lang.System
+          staticfield: out
+          method: println
+          _x: Testing
+    - op:
+          object: myobj
+    - myobj=System.out.println("testing");
+```

driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/TestYamlLoader.java

@@ -0,0 +1,18 @@
+package io.nosqlbench.driver.pulsar;
+
+import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
+import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.jupiter.api.Test;
+
+public class TestYamlLoader {
+
+    private final static Logger logger = LogManager.getLogger(TestYamlLoader.class);
+
+    @Test
+    public void loadAvroYaml() {
+        RawStmtsLoader sl = new RawStmtsLoader();
+        RawStmtsDocList rsdl = sl.loadPath(logger, "activities/pulsar_client_avro.yaml");
+    }
+}

driver-stdout/src/main/java/io/nosqlbench/adapters/stdout/StdoutDriverAdapter.java

@@ -0,0 +1,37 @@
+package io.nosqlbench.adapters.stdout;
+
+import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
+import io.nosqlbench.engine.api.activityimpl.OpMapper;
+import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
+import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache;
+import io.nosqlbench.engine.api.templating.OpTemplateSupplier;
+import io.nosqlbench.nb.api.config.standard.NBConfigModel;
+import io.nosqlbench.nb.api.config.standard.NBConfiguration;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+
+public class StdoutDriverAdapter extends BaseDriverAdapter<StdoutOp, StdoutSpace> implements OpTemplateSupplier {
+
+    @Override
+    public OpMapper<StdoutOp> getOpMapper() {
+        DriverSpaceCache<? extends StdoutSpace> ctxCache = getSpaceCache();
+        return new StdoutOpMapper(ctxCache);
+    }
+
+    @Override
+    public Function<String, ? extends StdoutSpace> getSpaceInitializer(NBConfiguration cfg) {
+        return (s) -> new StdoutSpace(cfg);
+    }
+
+    @Override
+    public NBConfigModel getConfigModel() {
+        return StdoutSpace.getConfigModel();
+    }
+
+    @Override
+    public Optional<List<OpTemplate>> loadOpTemplates(NBConfiguration cfg) {
+        throw new RuntimeException("implement me");
+    }
+}

driver-stdout/src/main/java/io/nosqlbench/adapters/stdout/StdoutOp.java

@@ -0,0 +1,19 @@
+package io.nosqlbench.adapters.stdout;
+
+import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.RunnableOp;
+
+public class StdoutOp implements RunnableOp {
+
+    private final StdoutSpace ctx;
+    private final String text;
+
+    public StdoutOp(StdoutSpace ctx, String text) {
+        this.ctx = ctx;
+        this.text = text;
+    }
+
+    @Override
+    public void run() {
+        ctx.write(text);
+    }
+}

driver-stdout/src/main/java/io/nosqlbench/adapters/stdout/StdoutOpDispenser.java

@@ -0,0 +1,24 @@
+package io.nosqlbench.adapters.stdout;
+
+import io.nosqlbench.engine.api.activityimpl.OpDispenser;
+import io.nosqlbench.engine.api.templating.ParsedOp;
+
+import java.util.function.LongFunction;
+
+public class StdoutOpDispenser implements OpDispenser<StdoutOp> {
+
+    private final LongFunction<StdoutSpace> ctxfunc;
+    private final LongFunction<?> objectFunction;
+
+    public StdoutOpDispenser(ParsedOp cmd, LongFunction<StdoutSpace> ctxfunc) {
+        objectFunction = cmd.getAsFunction("stmt", Object.class);
+        this.ctxfunc = ctxfunc;
+    }
+
+    @Override
+    public StdoutOp apply(long value) {
+        StdoutSpace ctx = ctxfunc.apply(value);
+        String output = objectFunction.apply(value).toString();
+        return new StdoutOp(ctx,output);
+    }
+}

driver-stdout/src/main/java/io/nosqlbench/adapters/stdout/StdoutOpMapper.java

@@ -0,0 +1,25 @@
+package io.nosqlbench.adapters.stdout;
+
+import io.nosqlbench.engine.api.activityimpl.OpDispenser;
+import io.nosqlbench.engine.api.activityimpl.OpMapper;
+import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache;
+import io.nosqlbench.engine.api.templating.ParsedOp;
+
+import java.util.function.LongFunction;
+
+public class StdoutOpMapper implements OpMapper<StdoutOp> {
+
+    private final DriverSpaceCache<? extends StdoutSpace> ctxcache;
+
+    public StdoutOpMapper(DriverSpaceCache<? extends StdoutSpace> ctxcache) {
+        this.ctxcache = ctxcache;
+    }
+
+    @Override
+    public OpDispenser<StdoutOp> apply(ParsedOp cmd) {
+        LongFunction<String> spacefunc = cmd.getAsFunctionOr("space", "default");
+        LongFunction<StdoutSpace> ctxfunc = (cycle) -> ctxcache.get(spacefunc.apply(cycle));
+        return new StdoutOpDispenser(cmd,ctxfunc);
+    }
+
+}

driver-stdout/src/main/java/io/nosqlbench/adapters/stdout/StdoutSpace.java

@@ -0,0 +1,77 @@
+package io.nosqlbench.adapters.stdout;
+
+import io.nosqlbench.nb.api.config.standard.*;
+
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
+import java.io.Writer;
+
+public class StdoutSpace {
+
+    Writer writer;
+    private PrintWriter console;
+
+    public StdoutSpace(NBConfiguration cfg) {
+        String filename = cfg.get("filename");
+        this.writer = createPrintWriter(filename);
+    }
+
+    public void write(String text) {
+        try {
+            writer.write(text);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected Writer createPrintWriter(String filename) {
+        PrintWriter pw;
+        if (filename.equalsIgnoreCase("stdout")) {
+            pw = getConsoleOut();
+        } else {
+            try {
+                pw = new PrintWriter(filename);
+                pw.print("");
+            } catch (FileNotFoundException e) {
+                e.printStackTrace();
+                throw new RuntimeException("Error initializing printwriter:" + e, e);
+            }
+        }
+        return pw;
+    }
+
+    public static NBConfigModel getConfigModel() {
+        return ConfigModel.of(StdoutSpace.class)
+            .add(
+                Param.defaultTo("filename","stdout")
+                    .setDescription("this is the name of the output file. If 'stdout', output is sent to stdout, not a file.")
+            )
+            .add(
+                Param.defaultTo("newline",true)
+                    .setDescription("whether to automatically add a missing newline to the end of any output\n")
+            )
+            .add(
+                Param.optional("format")
+                    .setRegex("csv|readout|json|inlinejson|assignments|diag")
+                    .setDescription("Which format to use.\n" +
+                        "If provided, the format will override any statement formats provided by the YAML")
+            )
+            .add(
+                Param.defaultTo("bindings","doc")
+                    .setDescription("This is a simple way to specify a filter for the names of bindings that you want to use.\n" +
+                        "If this is 'doc', then all the document level bindings are used. If it is any other value, it is taken\n" +
+                        "as a pattern (regex) to subselect a set of bindings by name. You can simply use the name of a binding\n" +
+                        "here as well.")
+
+            )
+            .asReadOnly();
+    }
+
+    public synchronized PrintWriter getConsoleOut() {
+        if (this.console == null) {
+            this.console = new PrintWriter(System.out);
+        }
+        return this.console;
+    }
+
+}

driver-stdout/src/main/java/io/nosqlbench/adapters/stdout/TemplateFormat.java

@@ -0,0 +1,113 @@
+/*
+ *
+ *    Copyright 2016 jshook
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ * /
+ */
+
+package io.nosqlbench.adapters.stdout;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public enum TemplateFormat {
+
+    csv("value1,value2,..."),
+    assignments("varname1=value1 varname2=value2 ..."),
+    readout("varname1       : value1\nvarname2:      : value2\n..."),
+    json("{\n varname1: value1,\n varname2: value2,\n ...\n}\n"),
+    inlinejson("{varname1:value1, varname2:value2, ...}");
+
+    private final String example;
+
+    TemplateFormat(String example) {
+        this.example = example;
+    }
+
+    public String format(boolean addNewlineSeparator, List<String> fieldNames) {
+        return this.format(addNewlineSeparator, fieldNames, null);
+    }
+
+    public String format(
+            boolean addNewlineSeparator,
+            List<String> fieldNames,
+            Function<String, String> fieldNameAdapter) {
+
+        if (fieldNameAdapter != null) {
+            fieldNames = fieldNames.stream()
+                    .map(fieldNameAdapter)
+                    .collect(Collectors.toCollection(ArrayList::new));
+        }
+        String template = "";
+        switch (this) {
+            case csv:
+                template = fieldNames
+                        .stream().map(s -> "{" + s + "}")
+                        .collect(Collectors.joining(","));
+                break;
+            case assignments:
+                template = fieldNames
+                        .stream().map(s -> s + "={" + s + "}")
+                        .collect(Collectors.joining(" "));
+                break;
+            case readout:
+                int maxlen = fieldNames.stream().mapToInt(String::length).max().orElse(0);
+                template = fieldNames
+                        .stream().map((field) -> String.format("%" + maxlen + "s : {%s}", field, field))
+                        .collect(Collectors.joining("\n"));
+                break;
+            case json:
+                template = fieldNames
+                        .stream().map(s -> "\"" + s + "\":\"{" + s + "}\"")
+                        .collect(Collectors.joining(",\n ", "{\n ", "\n}"));
+                break;
+            case inlinejson:
+                template = fieldNames
+                        .stream().map(s -> "\""+ s + "\":\"{" + s + "}\"")
+                        .collect(Collectors.joining(", ", "{", "}"));
+                break;
+            default:
+                throw new RuntimeException("No supported format was found for " + this);
+        }
+
+        if (addNewlineSeparator) {
+            template = withSeparator(template);
+        }
+        return template;
+    }
+
+    /**
+     * Ensure that the statement ends with a newline.
+     * If there are newlines within the statement, then ensure that it ends with a double newline.
+     * @param raw statement possibly without newlines
+     * @return statement with appropriate newlines
+     */
+    private String withSeparator(String raw) {
+        int pos = raw.indexOf("\n");
+        if (pos>0 && pos<raw.length()-1) {
+            if (raw.endsWith("\n\n")) {
+                return raw;
+            }
+            if (raw.endsWith("\n")) {
+                return raw + "\n";
+            }
+            return raw+"\n\n";
+        }
+        if (!raw.endsWith("\n")) {
+            return raw+"\n";
+        }
+        return raw;
+    }
+}

drivers-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java

@@ -0,0 +1,65 @@
+package io.nosqlbench.engine.api.templating;
+
+import io.nosqlbench.engine.api.activityconfig.yaml.OpData;
+import io.nosqlbench.nb.api.config.standard.ConfigModel;
+import io.nosqlbench.nb.api.config.standard.Param;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.LongFunction;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ParsedOpTest {
+
+    ParsedOp pc = new ParsedOp(
+        new OpData().applyFields(
+            Map.of(
+                "op", Map.of(
+                    "stmt", "test",
+                    "dyna1", "{dyna1}",
+                    "dyna2", "{{NumberNameToString()}}",
+                    "identity", "{{Identity()}}"
+                ),
+                "bindings", Map.of(
+                    "dyna1", "NumberNameToString()"
+                )
+            )
+        ),
+        ConfigModel.of(ParsedOpTest.class)
+            .add(Param.defaultTo("testcfg","testval"))
+            .asReadOnly()
+            .apply(Map.of())
+    );
+
+    @Test
+    public void testParsedOp() {
+        Map<String, Object> m1 = pc.apply(0);
+        assertThat(m1).containsEntry("stmt", "test");
+        assertThat(m1).containsEntry("dyna1","zero");
+        assertThat(m1).containsEntry("dyna2","zero");
+        assertThat(m1).containsEntry("identity", 0L);
+    }
+
+    @Test
+    public void testNewListBinder() {
+        LongFunction<List<Object>> lb = pc.newListBinder("dyna1", "identity", "dyna2", "identity");
+        List<Object> objects = lb.apply(1);
+        assertThat(objects).isEqualTo(List.of("one",1L,"one",1L));
+    }
+
+    @Test
+    public void testNewMapBinder() {
+        LongFunction<Map<String, Object>> mb = pc.newOrderedMapBinder("dyna1", "identity", "dyna2");
+        Map<String, Object> objects = mb.apply(2);
+        assertThat(objects).isEqualTo(Map.<String,Object>of("dyna1","two","identity",2L,"dyna2","two"));
+    }
+
+    @Test
+    public void testNewAryBinder() {
+        LongFunction<Object[]> ab = pc.newArrayBinder("dyna1", "dyna1", "identity", "identity");
+        Object[] objects = ab.apply(3);
+        assertThat(objects).isEqualTo(new Object[]{"three","three",3L,3L});
+    }
+}

engine-clients/src/main/java/io/nosqlbench/engine/clients/grafana/analyzer/test.json

@@ -0,0 +1,859 @@
+{
+  "meta": {
+    "isSnapshot": true,
+    "type": "snapshot",
+    "canSave": false,
+    "canEdit": false,
+    "canAdmin": false,
+    "canStar": false,
+    "slug": "",
+    "url": "",
+    "expires": "2070-12-05T06:28:42Z",
+    "created": "2020-12-17T06:28:42Z",
+    "updated": "0001-01-01T00:00:00Z",
+    "updatedBy": "",
+    "createdBy": "",
+    "version": 0,
+    "hasAcl": false,
+    "isFolder": false,
+    "folderId": 0,
+    "folderTitle": "",
+    "folderUrl": "",
+    "provisioned": false,
+    "provisionedExternalId": ""
+  },
+  "dashboard": {
+    "annotations": {
+      "list": [
+        {
+          "datasource": "-- Grafana --",
+          "enable": true,
+          "hide": false,
+          "iconColor": "#96D98D",
+          "limit": 1000.0,
+          "name": "ShowMatching",
+          "showIn": 0.0,
+          "tags": [
+            "appname:nosqlbench",
+            "span:$span",
+            "layer:$layer"
+          ],
+          "type": "tags"
+        },
+        {
+          "builtIn": 1.0,
+          "datasource": "-- Grafana --",
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "limit": 100.0,
+          "name": "Annotations \u0026 Alerts",
+          "showIn": 0.0,
+          "type": "dashboard"
+        },
+        {
+          "datasource": "-- Grafana --",
+          "enable": false,
+          "hide": false,
+          "iconColor": "rgba(255, 96, 96, 1)",
+          "limit": 1000.0,
+          "matchAny": true,
+          "name": "ShowAll",
+          "showIn": 0.0,
+          "tags": [
+            "appname:nosqlbench"
+          ],
+          "type": "tags"
+        }
+      ]
+    },
+    "description": "Basic Dashboard with Annotations for NoSQLBench 4",
+    "editable": true,
+    "graphToolTip": 0,
+    "id": 1,
+    "iteration": 1607542677976,
+    "links": [],
+    "panels": [
+      {
+        "collapsed": true,
+        "gridPos": {
+          "h": "1",
+          "w": "24",
+          "x": "0",
+          "y": "0"
+        },
+        "id": 130,
+        "panels": [
+          {
+            "collapsed": false,
+            "description": "",
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "4",
+              "w": "4",
+              "x": "0",
+              "y": "1"
+            },
+            "id": 132,
+            "options": {
+              "content": "This selects the specific NoSQLBench alias, which includes workload, named scenario, and step components.\n",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "targets": [
+              {
+                "queryType": "randomWalk",
+                "refId": "A"
+              }
+            ],
+            "title": "Alias",
+            "type": "text"
+          },
+          {
+            "collapsed": false,
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "4",
+              "w": "4",
+              "x": "4",
+              "y": "1"
+            },
+            "id": 133,
+            "options": {
+              "content": "This chooses an Annotation Layer to focus on. Each layer is a different shell of execution within NoSQLBench.   \n",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "targets": [
+              {
+                "queryType": "randomWalk",
+                "refId": "A"
+              }
+            ],
+            "title": "AnLayer",
+            "type": "text"
+          },
+          {
+            "collapsed": false,
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "4",
+              "w": "4",
+              "x": "8",
+              "y": "1"
+            },
+            "id": 134,
+            "options": {
+              "content": "This chooses an Annotation Layer to focus on. An instant is a point in time. An Interval is defined by two instants representing start and end times.",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "targets": [
+              {
+                "queryType": "randomWalk",
+                "refId": "A"
+              }
+            ],
+            "title": "AnSpan",
+            "type": "text"
+          }
+        ],
+        "title": "NoSQLBench Controls Guide",
+        "type": "row"
+      },
+      {
+        "collapsed": true,
+        "gridPos": {
+          "h": "1",
+          "w": "24",
+          "x": "0",
+          "y": "1"
+        },
+        "id": 115,
+        "panels": [
+          {
+            "collapsed": false,
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "2",
+              "w": "24",
+              "x": "0",
+              "y": "2"
+            },
+            "id": 128,
+            "options": {
+              "content": "The descriptions below are in the same position as the metric they desscribe.\nYou can collapse this row when you no longer need the descriptions.\nSome of the descriptions have additional details if you scroll with\nyour mouse wheel.\n\n\n",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "title": "",
+            "type": "text"
+          },
+          {
+            "collapsed": false,
+            "description": "",
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "3",
+              "w": "6",
+              "x": "0",
+              "y": "4"
+            },
+            "id": 117,
+            "options": {
+              "content": "These two metrics show the 1-minute averaged ops per second for all ops (successes and errors) and succesful ops (no errors during execution) separately.\n\nWhen there are no errors, these metrics should be the same. When there are errors, there will be a difference. In that case, you can look at the error metrics to learn more about what is happening.\n\nBy comparing these two metrics for any activity, you have a quick first-glance sanity check that your tests are configured properly and\nthat there are no serious configuration or resource issues. In addition, this is the primary throughput metric.\n\n\n\n\n\n",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "title": "Ops and Successful Ops",
+            "type": "text"
+          },
+          {
+            "collapsed": false,
+            "description": "",
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "3",
+              "w": "6",
+              "x": "6",
+              "y": "4"
+            },
+            "id": 118,
+            "options": {
+              "content": "When an exception is caught and counted in a scenario, a specially named metric is emited which includes a simplified name of the error type.\nEach of these metrics is created as needed in the NoSQLBench runtime.\n\nThese metrics are formated with a name pattern of `errorcounts.NAME` where NAME is the simple name\nof the exception class which was caught during the operation. For example, For an activity\nnamed foo (with alias\u003dfoo), you would expect an exception named \u0027FooTimeoutException\u0027 to be shown\nwith a metric name of ...`foo.errorcounts.FooTimeoutException`\n\n\n\n\n",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "title": "Error Counts",
+            "type": "text"
+          },
+          {
+            "collapsed": false,
+            "description": "",
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "3",
+              "w": "6",
+              "x": "12",
+              "y": "4"
+            },
+            "id": 119,
+            "options": {
+              "content": "The service time distribution is an HDR histogram that represents the time elapsed from when an operation is submitted for execution to when its result has been received.\n\nFrom the vantage point of the NoSQLBench runtime, this is a client-side metric. Thus, the service time captures the processing time that an application would see, including driver logic, wire time (transport), and server-side processing. \n\nThis metric does not include the waittime or the responsetime metrics. These metrics are only meaningful (and provided) when a `cyclerate\u003d` is provided to an activity. When computing the responsetime metric, the servicetime is added to the waittime for a given operation.\n",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "title": "Service Time Distribution",
+            "type": "text"
+          },
+          {
+            "collapsed": false,
+            "description": "",
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "3",
+              "w": "6",
+              "x": "18",
+              "y": "4"
+            },
+            "id": 120,
+            "options": {
+              "content": "The tries distribution is an HDR histogram which represents how many times an operation was submitted. In a well balanced system,\ntries should be 1 across the board.\n\nEvery operation which is executed within NoSQLBench should have a `tries` metric. For example, with CQL, the number of times an operation is submitted\nbefore it is succesful is ideally 1. If you are overloading your target system, thus forcing resource contention, you may see operations timeout.\n\nBy default, NoSQLBench will try up to 10 times to submit operations before giving up and throwing an error for an operation.\n\nThe tries metric can be used as a low-noise indicator of system saturation. For example, if you are running a system marginally\nbeyond its capacity, the tries for operations will go above 1 at the higher percentiles such as P99. If you increase the load even further, more retries\nwill be needed and less work will be completed, thus showing higher retries at even lower percentiles, like P95, for example.\n\nAs such, you can use the tries metric as an indicator of relative saturation.",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "title": "Op Tries Distribution",
+            "type": "text"
+          },
+          {
+            "collapsed": false,
+            "description": "",
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "3",
+              "w": "6",
+              "x": "0",
+              "y": "7"
+            },
+            "id": 121,
+            "options": {
+              "content": "This panel shows the P75 service times for the internal stages of NoSQLBench processing. It provides a sanity check to ensure that the client processing time is low and predictable. \n\nIf there are spikes in this data, then you are likely trying to run your workloads on insufficient client hardware. In order to ensure high fidelity results in the other metrics, the client\nneeds to be capable of driving the workload without saturating or introducing signifcant local resource contention.\n\n- **read-input** - the time it takes for a worker thread to acquire a stride (range of cycles) for execution.\n- **bind** - the time it takes to convert a cycle value into a set of fields for us in an operation.\n- **execute** - the time it takes to submit work to a protocol-specific driver.",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "title": "P75 NB Internals",
+            "type": "text"
+          },
+          {
+            "collapsed": false,
+            "description": "",
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "3",
+              "w": "6",
+              "x": "6",
+              "y": "7"
+            },
+            "id": 122,
+            "options": {
+              "content": "This panel shows the P99 service times for the internal stages of NoSQLBench processing. It provides a sanity check to ensure that the client processing time is low and predictable. \n\nIf there are spikes in this data, then you are likely trying to run your workloads on insufficient client hardware. In order to ensure high fidelity results in the other metrics, the client\nneeds to be capable of driving the workload without saturating or introducing signifcant local resource contention.\n\n- **read-input** - the time it takes for a worker thread to acquire a stride (range of cycles) for execution.\n- **bind** - the time it takes to convert a cycle value into a set of fields for us in an operation.\n- **execute** - the time it takes to submit work to a protocol-specific driver.",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "title": "P99 NB Internals",
+            "type": "text"
+          },
+          {
+            "collapsed": false,
+            "description": "",
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "3",
+              "w": "6",
+              "x": "12",
+              "y": "7"
+            },
+            "id": 123,
+            "options": {
+              "content": "This panel simply breaks out the service time range in a simpler view. This is a good metric to look at when you want to know what the best and worse case value is for any given histogram interval.\n\nThese values come from a discrete HDR histogram reservoir. They are the actual best and worst service times, unaffected by time-decaying reservoir logic.\n",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "title": "Service Time Range",
+            "type": "text"
+          },
+          {
+            "collapsed": false,
+            "description": "",
+            "fieldConfig": {
+              "defaults": {
+                "custom": {}
+              },
+              "overrides": []
+            },
+            "gridPos": {
+              "h": "3",
+              "w": "6",
+              "x": "18",
+              "y": "7"
+            },
+            "id": 124,
+            "options": {
+              "content": "This is a simple visual reference for the cycle count within each activity. This makes it easy to see the relative progress of an activity over time.",
+              "mode": "markdown"
+            },
+            "pluginVersion": "7.3.4",
+            "title": "Cycle Count",
+            "type": "text"
+          }
+        ],
+        "title": "NoSQLBench Metrics Guide",
+        "type": "row"
+      },
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": "1",
+          "w": "24",
+          "x": "0",
+          "y": "2"
+        },
+        "id": 91,
+        "panels": [],
+        "title": "NoSQLBench Metrics",
+        "type": "row"
+      },
+      {
+        "collapsed": false,
+        "datasource": "prometheus",
+        "description": "",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {},
+            "links": []
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": "6",
+          "w": "6",
+          "x": "0",
+          "y": "3"
+        },
+        "id": 95,
+        "options": {
+          "alertThreshold": "true"
+        },
+        "pluginVersion": "7.3.4",
+        "targets": [
+          {
+            "expr": "result{type\u003d\"avg_rate\",avg_of\u003d\"1m\",alias\u003d~\"$alias\"}",
+            "interval": "",
+            "legendFormat": "{{alias}}-allops",
+            "refId": "C"
+          },
+          {
+            "expr": "result{type\u003d\"avg_rate\",avg_of\u003d\"1m\",alias\u003d~\"$alias\"}",
+            "interval": "",
+            "legendFormat": "{{alias}}-allops",
+            "refId": "B"
+          }
+        ],
+        "title": "Ops and Successful Ops",
+        "type": "graph"
+      },
+      {
+        "collapsed": false,
+        "datasource": "prometheus",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {},
+            "links": []
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": "6",
+          "w": "6",
+          "x": "6",
+          "y": "3"
+        },
+        "id": 93,
+        "options": {
+          "alertThreshold": "true"
+        },
+        "pluginVersion": "7.3.4",
+        "targets": [
+          {
+            "expr": "{__name__\u003d~\"errorcounts.*\",alias\u003d~\"$alias\"}",
+            "interval": "",
+            "legendFormat": "{{alias}}-{{error}}",
+            "refId": "B"
+          }
+        ],
+        "title": "Error Counts",
+        "type": "graph"
+      },
+      {
+        "collapsed": false,
+        "datasource": "prometheus",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {},
+            "links": []
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": "6",
+          "w": "6",
+          "x": "12",
+          "y": "3"
+        },
+        "id": 97,
+        "options": {
+          "alertThreshold": "true"
+        },
+        "pluginVersion": "7.3.4",
+        "targets": [
+          {
+            "expr": "result_success{type\u003d\"pctile\",alias\u003d~\"$Alias\"}",
+            "hide": "false",
+            "interval": "",
+            "legendFormat": "{{alias}}-p{{pctile}}",
+            "refId": "B"
+          },
+          {
+            "expr": "result_success{type\u003d\"pctile\",alias\u003d~\"$alias\"}",
+            "hide": "false",
+            "interval": "",
+            "legendFormat": "{{alias}}-p{{pctile}}",
+            "refId": "C"
+          }
+        ],
+        "title": "service time distribution",
+        "type": "graph"
+      },
+      {
+        "collapsed": false,
+        "datasource": "prometheus",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {},
+            "links": []
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": "6",
+          "w": "6",
+          "x": "18",
+          "y": "3"
+        },
+        "id": 98,
+        "options": {
+          "alertThreshold": "true"
+        },
+        "pluginVersion": "7.3.4",
+        "targets": [
+          {
+            "expr": "tries{type\u003d\"pctile\",alias\u003d~\"$alias\"}",
+            "interval": "",
+            "legendFormat": "{{alias}}-p{{pctile}}",
+            "refId": "A"
+          }
+        ],
+        "title": "op tries distribution",
+        "type": "graph"
+      },
+      {
+        "collapsed": false,
+        "datasource": "prometheus",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {},
+            "links": []
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": "6",
+          "w": "6",
+          "x": "0",
+          "y": "9"
+        },
+        "id": 99,
+        "options": {
+          "alertThreshold": "true"
+        },
+        "pluginVersion": "7.3.4",
+        "targets": [
+          {
+            "expr": "{__name__\u003d~\"read_input|bind|execute\",type\u003d\"pctile\",pctile\u003d\"75\",alias\u003d~\"$alias\"}",
+            "interval": "",
+            "legendFormat": "{{alias}}-{{__name__}}-p{{pctile}}",
+            "refId": "B"
+          }
+        ],
+        "title": "p75 client overhead",
+        "type": "graph"
+      },
+      {
+        "collapsed": false,
+        "datasource": "prometheus",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {},
+            "links": []
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": "6",
+          "w": "6",
+          "x": "6",
+          "y": "9"
+        },
+        "id": 111,
+        "options": {
+          "alertThreshold": "true"
+        },
+        "pluginVersion": "7.3.4",
+        "targets": [
+          {
+            "expr": "{__name__\u003d~\"read_input|bind|execute\",type\u003d\"pctile\",pctile\u003d\"99\",alias\u003d~\"$alias\"}",
+            "format": "time_series",
+            "hide": "false",
+            "instant": "false",
+            "interval": "",
+            "intervalFactor": "1",
+            "legendFormat": "{{alias}}-{{__name__}}-p{{pctile}}",
+            "refId": "C"
+          }
+        ],
+        "title": "p99 client overhead",
+        "type": "graph"
+      },
+      {
+        "collapsed": false,
+        "datasource": "prometheus",
+        "description": "",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {},
+            "links": []
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": "6",
+          "w": "6",
+          "x": "12",
+          "y": "9"
+        },
+        "id": 109,
+        "options": {
+          "alertThreshold": "true"
+        },
+        "pluginVersion": "7.3.4",
+        "targets": [
+          {
+            "expr": "result_success{type\u003d\"pctile\",pctile\u003d\"0\",alias\u003d~\"$alias\"}",
+            "interval": "",
+            "legendFormat": "{{alias}}-min",
+            "refId": "B"
+          },
+          {
+            "expr": "result_success{type\u003d\"pctile\",pctile\u003d\"100\",alias\u003d~\"$alias\"}",
+            "interval": "",
+            "legendFormat": "{{alias}}-max",
+            "refId": "A"
+          }
+        ],
+        "title": "service time range",
+        "type": "graph"
+      },
+      {
+        "collapsed": false,
+        "datasource": "prometheus",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {},
+            "links": []
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": "6",
+          "w": "6",
+          "x": "18",
+          "y": "9"
+        },
+        "id": 113,
+        "options": {
+          "alertThreshold": "true"
+        },
+        "pluginVersion": "7.3.4",
+        "targets": [
+          {
+            "expr": "cycles_servicetime{type\u003d\"counter\",alias\u003d~\"$alias\"}",
+            "interval": "",
+            "legendFormat": "{{alias}}-count",
+            "refId": "C"
+          }
+        ],
+        "title": "Cycle Count",
+        "type": "graph"
+      }
+    ],
+    "refresh": "1m",
+    "schemaVersion": 26,
+    "style": "dark",
+    "tags": [
+      "NoSQLBench"
+    ],
+    "templating": {
+      "list": [
+        {
+          "allValue": ".*",
+          "current": {
+            "selected": true,
+            "text": [
+              "All"
+            ],
+            "value": [
+              "$__all"
+            ]
+          },
+          "datasource": "prometheus",
+          "definition": "{appname\u003d\"nosqlbench\"}",
+          "hide": 0.0,
+          "includeAll": true,
+          "label": "Alias",
+          "multi": true,
+          "name": "alias",
+          "options": [],
+          "query": "{appname\u003d\"nosqlbench\"}",
+          "refresh": 2.0,
+          "regex": "/.*alias\u003d\"([^\"]+)\".*/",
+          "skipUrlSync": false,
+          "sort": 1.0,
+          "tagValuesQuery": "",
+          "tags": [],
+          "tagsQuery": "",
+          "type": "query",
+          "useTags": false
+        },
+        {
+          "allValue": " ",
+          "current": {
+            "selected": true,
+            "text": "Activity",
+            "value": "Activity"
+          },
+          "hide": 0.0,
+          "includeAll": true,
+          "label": "AnLayer",
+          "multi": false,
+          "name": "layer",
+          "options": [
+            {
+              "selected": false,
+              "text": "All",
+              "value": "$__all"
+            },
+            {
+              "selected": false,
+              "text": "NONE",
+              "value": "NONE"
+            },
+            {
+              "selected": false,
+              "text": "CLI",
+              "value": "CLI"
+            },
+            {
+              "selected": true,
+              "text": "Scenario",
+              "value": "Scenario"
+            },
+            {
+              "selected": false,
+              "text": "Script",
+              "value": "Script"
+            },
+            {
+              "selected": false,
+              "text": "Activity",
+              "value": "Activity"
+            }
+          ],
+          "query": "NONE,CLI,Scenario,Script,Activity",
+          "queryValue": "",
+          "skipUrlSync": false,
+          "type": "custom"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "interval",
+            "value": "interval"
+          },
+          "hide": 0.0,
+          "includeAll": false,
+          "label": "AnSpan",
+          "multi": false,
+          "name": "span",
+          "options": [
+            {
+              "selected": false,
+              "text": "NONE",
+              "value": "NONE"
+            },
+            {
+              "selected": false,
+              "text": "interval",
+              "value": "interval"
+            },
+            {
+              "selected": true,
+              "text": "instant",
+              "value": "instant"
+            }
+          ],
+          "query": "NONE,interval,instant",
+          "queryValue": "",
+          "skipUrlSync": false,
+          "type": "custom"
+        }
+      ]
+    },
+    "time": {
+      "from": "2020-12-16T23:19:50.623Z",
+      "to": "2020-12-16T23:35:45.691Z"
+    },
+    "timepicker": {
+      "refresh_intervals": [
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "time_options": [
+        "5m",
+        "15m",
+        "1h",
+        "6h",
+        "12h",
+        "24h",
+        "2d",
+        "7d",
+        "30d"
+      ]
+    },
+    "timezone": "",
+    "title": "NB4 Dashboard",
+    "uid": "aIIX1f6Wz",
+    "version": 1
+  }
+}

nb-api/src/main/java/io/nosqlbench/nb/api/config/fieldreaders/DynamicFieldReader.java

@@ -0,0 +1,20 @@
+package io.nosqlbench.nb.api.config.fieldreaders;
+
+import java.util.function.LongFunction;
+
+/**
+ * An interface which captures the semantics and patterns of
+ * reading field values that are rendered functionally.
+ * This interface is meant to help standardize the user
+ * interfaces for reading configuration and fields across
+ * the NB codebase.
+ * See also {@link StaticFieldReader}
+ * and {@link EnvironmentReader}
+ */
+public interface DynamicFieldReader {
+    boolean isDefinedDynamic(String field);
+
+    <T> T get(String field, long input);
+
+    <V> LongFunction<V> getAsFunctionOr(String name, V defaultValue);
+}

virtdata-lib-realer/src/main/java/io/nosqlbench/virtdata/library/realer/CountryNames.java

@@ -0,0 +1,23 @@
+package io.nosqlbench.virtdata.library.realer;
+
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.Example;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+import io.nosqlbench.virtdata.library.basics.shared.distributions.CSVSampler;
+
+import java.util.function.LongFunction;
+
+/**
+ * Return a valid country name.
+ */
+@Categories(Category.premade)
+@ThreadSafeMapper
+public class CountryNames extends CSVSampler implements LongFunction<String> {
+
+    @Example("CountryNames()")
+    public CountryNames() {
+        super("COUNTRY_NAME","n/a","name","countries.csv");
+    }
+
+}

virtdata-lib-realer/src/main/java/io/nosqlbench/virtdata/library/realer/TimeZonesByDensity.java

@@ -0,0 +1,19 @@
+package io.nosqlbench.virtdata.library.realer;
+
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.Example;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+import io.nosqlbench.virtdata.library.basics.shared.distributions.CSVSampler;
+
+/**
+ * Return a state name, weighted by population density.
+ */
+@ThreadSafeMapper
+@Categories(Category.premade)
+public class TimeZonesByDensity extends CSVSampler {
+    @Example("TimezonesByDensity")
+    public TimeZonesByDensity() {
+        super("timezone","population","simplemaps/uszips");
+    }
+}

virtdata-lib-realer/src/main/java/io/nosqlbench/virtdata/library/realer/ZipCodesByDensity.java

@@ -0,0 +1,19 @@
+package io.nosqlbench.virtdata.library.realer;
+
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.Example;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+import io.nosqlbench.virtdata.library.basics.shared.distributions.CSVSampler;
+
+/**
+ * Return a zip code, weighted by population density.
+ */
+@ThreadSafeMapper
+@Categories(Category.premade)
+public class ZipCodesByDensity extends CSVSampler {
+    @Example("ZipCodesByDensity")
+    public ZipCodesByDensity() {
+        super("zip","population","simplemaps/uszips");
+    }
+}

virtdata-realdata/src/main/resources/data/simplemaps/license.txt

@@ -0,0 +1,21 @@
+This license is a legal document designed to protect your rights and the rights of the Pareto Software, LLC, the owner of Simplemaps.com. Please read it carefully. Purchasing or downloading a data product constitutes acceptance of this license.
+
+Description of Product and Parties: This license is a contract between you (hereafter, the Customer) and Pareto Software, LLC (hereafter, the Provider) regarding the use and/or sale of an collection of geographic data (hereafter, the Database).
+
+Ownership of Database: All rights to the Database are owned by the Provider. The Database is a cleaned and curated collection of geographic facts and the Provider retains all rights to the Database afforded by the law. Ownership of any intellectual property generated by the Provider while performing custom modifications to the Database for a Customer (with or without payment) is retained by the Provider.
+
+License: Customers who purchase a license are allowed to use the database for projects that benefit their organization or that their organization oversees. Attribution is not required. The Customer is allowed to query the database to power privately or publicly facing applications. The Customer is allowed to make copies and backups of the data. The Customer may not publicly redistribute the Database without prior written permission. Customers can transfer their license to a third-party, at the sole discretion of the Provider, by submitting a request via email.
+
+Free US Zip Code Database: The Provider offers a free version of the US Zip Code Database. This Database is offered free of charge conditional on a link back to https://simplemaps.com/data/us-zips. This backlink must come from a public webpage where the Customer is using the data. If the Customer uses the data internally, the backlink must be placed on the organization's website on a page that can be easily found though links on the root domain. The link must be clearly visible to the human eye. The backlink must be placed before the Customer uses the Database in production.
+
+Free US Cities Database: The Provider offers a free version of the US Cities Database. This Database is offered free of charge conditional on a link back to https://simplemaps.com/data/us-cities. This backlink must come from a public webpage where the Customer is using the data. If the Customer uses the data internally, the backlink must be placed on the organization's website on a page that can be easily found though links on the root domain. The link must be clearly visible to the human eye. The backlink must be placed before the Customer uses the Database in production.
+
+Basic World Cities Database: The Provider offers a Basic World Cities Database free of charge. This database is licensed under the MIT license as described at: https://opensource.org/licenses/MIT.
+
+Comprehensive World Cities Database Density Data: The Comprehensive World Cities Database includes density estimates from The Center for International Earth Science Information Network - CIESIN - Columbia University. 2016. Gridded Population of the World, Version 4 (GPWv4): Population Count. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). http://dx.doi.org/10.7927/H4X63JVC. Accessed June 2017. The density estimates are include under the Creative Commons Attribution 4.0 International License. The Provider places no additional restrictions on the use or distribution of the density data.
+
+Guarantee: The Provider guarantees that for the period of thirty (30) days from the purchase of a License that the Customer shall, upon request, be refunded their actual purchase price within a reasonable period of time. The Customer acknowledges that receipt of a refund constitutes a termination of their License to use the Database. In the event of a Refund, the Customer promises to delete the Database immediately. Refunds after the period of thirty (30) days shall be at the sole discretion of the Provider.
+
+LIMITATION OF LIABILITY: THE DATABASE IS SOLD "AS IS" AND "WITH ALL FAULTS". THE PROVIDER MAKES NO WARRANTY THAT IT IS FREE OF DEFECTS OR IS SUITABLE FOR ANY PARTICULAR PURPOSE. IN NO EVENT SHALL THE PROVIDER BE RESPONSIBLE FOR LOSS OR DAMAGES ARRISING FROM THE INSTALLATION OR USE OF THE DATABASE, INCLUDING BUT NOT LIMITED TO ANY INDIRECT, PUNITIVE, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES. THE CUSTOMER IS SOLELY RESPONSIBLE FOR ENSURING THAT THEIR USE OF THE DATABASE IS IN ACCORDANCE WITH THE LAW OF THEIR JURISDICTION.
+
+PROHIBITION OF ILLEGAL USE: USE OF THE DATABASE WHICH IS CONTRARY TO THE LAW IS PROHIBITED, AND IMMEDIATELY TERMINATES THE CUSTOMER'S LICENSE TO USE THE DATABASE.