Merge pull request #1129 from nosqlbench/nosqlbench-1092-parsedop-captures

nosqlbench-1092-parsedop-captures

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@ import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlPreparedStatement;
 import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
 import io.nosqlbench.engine.api.templating.ParsedOp;
 import io.nosqlbench.api.errors.OpConfigError;
-import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -37,13 +37,13 @@ public class Cqld4PreparedStmtDispenser extends Cqld4BaseOpDispenser {
 
     private final RSProcessors processors;
     private final LongFunction<Statement> stmtFunc;
-    private final ParsedStringTemplate stmtTpl;
+    private final ParsedTemplateString stmtTpl;
     private final LongFunction<Object[]> fieldsF;
     private PreparedStatement preparedStmt;
     private CqlSession boundSession;
 
     public Cqld4PreparedStmtDispenser(
-        DriverAdapter adapter, LongFunction<CqlSession> sessionFunc, ParsedOp op, ParsedStringTemplate stmtTpl, RSProcessors processors) {
+            DriverAdapter adapter, LongFunction<CqlSession> sessionFunc, ParsedOp op, ParsedTemplateString stmtTpl, RSProcessors processors) {
         super(adapter, sessionFunc, op);
         if (op.isDynamic("space")) {
             throw new RuntimeException("Prepared statements and dynamic space values are not supported." +

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4PreparedStmtMapper.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ import io.nosqlbench.engine.api.templating.ParsedOp;
 import io.nosqlbench.engine.api.templating.TypeAndTarget;
 import io.nosqlbench.api.config.params.ParamsParser;
 import io.nosqlbench.api.errors.BasicError;
-import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
 
 import java.util.List;
 import java.util.Map;
@@ -51,7 +51,7 @@ public class CqlD4PreparedStmtMapper implements OpMapper<Cqld4CqlOp> {
 
     public OpDispenser<Cqld4CqlOp> apply(ParsedOp op) {
 
-        ParsedStringTemplate stmtTpl = op.getAsTemplate(target.field).orElseThrow(() -> new BasicError(
+        ParsedTemplateString stmtTpl = op.getAsTemplate(target.field).orElseThrow(() -> new BasicError(
             "No statement was found in the op template:" + op
         ));
 

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4FluentGraphOpMapper.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,7 +31,7 @@ import io.nosqlbench.engine.api.templating.TypeAndTarget;
 import io.nosqlbench.api.errors.OpConfigError;
 import io.nosqlbench.virtdata.core.bindings.Bindings;
 import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
-import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
@@ -63,7 +63,7 @@ public class Cqld4FluentGraphOpMapper implements OpMapper<Op> {
     public OpDispenser<? extends Op> apply(ParsedOp op) {
         GraphTraversalSource g = DseGraph.g;
 
-        ParsedStringTemplate fluent = op.getAsTemplate(target.field).orElseThrow();
+        ParsedTemplateString fluent = op.getAsTemplate(target.field).orElseThrow();
         String scriptBodyWithRawVarRefs = fluent.getPositionalStatement();
 
         CompilerConfiguration compilerConfiguration = new CompilerConfiguration();

adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedOpTemplate.java

@@ -1,129 +0,0 @@
-/*
- * Copyright (c) 2022-2023 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.nosqlbench.engine.api.activityconfig;
-
-import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
-import io.nosqlbench.api.config.params.Element;
-import io.nosqlbench.api.config.params.NBParams;
-import io.nosqlbench.virtdata.core.templates.BindPoint;
-import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Function;
-
-public class ParsedOpTemplate {
-
-    private final OpTemplate optpl;
-    private final ParsedStringTemplate parsed;
-
-    /**
-     * Construct a new ParsedStatement from the provided stmtDef and anchor token.
-     *
-     * @param optpl An existing statement def as read from the YAML API.
-     */
-    public ParsedOpTemplate(OpTemplate optpl) {
-        this.optpl = optpl;
-        String transformed = getStmt();
-        parsed = new ParsedStringTemplate(transformed, optpl.getBindings());
-    }
-
-    public ParsedOpTemplate orError() {
-        if (hasError()) {
-            throw new RuntimeException("Unable to parse statement: " + this);
-        }
-        return this;
-    }
-
-    public String toString() {
-        return parsed.toString();
-    }
-
-    /**
-     * @return true if the parsed statement is not usable.
-     */
-    public boolean hasError() {
-        return parsed.hasError();
-    }
-
-
-    /**
-     * Returns a list of binding names which were referenced
-     * in either <pre>{anchor}</pre> or <pre>?anchor</pre> form,
-     * but which were not present in the provided bindings map.
-     * If any binding names are present in the returned set, then
-     * this binding will not be usable.
-     *
-     * @return A list of binding names which were referenced but not defined*
-     */
-    public Set<String> getMissingBindings() {
-        return parsed.getMissing();
-    }
-
-    /**
-     * Return the statement that can be used as-is by any driver specific version.
-     * This uses the anchor token as provided to yield a version of the statement
-     * which contains positional anchors, but no named bindings.
-     * @param tokenMapper A function which maps the anchor name to the needed form
-     *                    in the callers driver context
-     * @return A driver or usage-specific format of the statement, with anchors
-     */
-    public String getPositionalStatement(Function<String,String> tokenMapper) {
-        return parsed.getPositionalStatement(tokenMapper);
-    }
-
-    /**
-     * @return the statement name from the enclosed {@link OpTemplate}
-     */
-    public String getName() {
-        return optpl.getName();
-    }
-
-    /**
-     * @return the raw statement from the enclosed {@link OpTemplate}
-     */
-    public String getStmt() {
-        return optpl.getStmt().orElseThrow();
-    }
-
-    /**
-     * @return the tags from the enclosed {@link OpTemplate}
-     */
-    public Map<String, String> getTags() {
-        return optpl.getTags();
-    }
-
-    /**
-     * @return the bindings from the enclosed {@link OpTemplate}
-     */
-    public Map<String, String> getBindings() {
-        return optpl.getBindings();
-    }
-
-    /**
-     * @return a params reader from the enclosed {@link OpTemplate} params map
-     */
-    public Element getParamReader() {
-        return NBParams.one(getName(),optpl.getParams());
-    }
-
-    public List<BindPoint> getBindPoints() {
-        return parsed.getBindPoints();
-    }
-
-}

adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java

@@ -23,7 +23,7 @@ import io.nosqlbench.api.config.params.Element;
 import io.nosqlbench.api.config.params.NBParams;
 import io.nosqlbench.api.config.standard.NBTypeConverter;
 import io.nosqlbench.api.errors.OpConfigError;
-import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
 
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -170,9 +170,9 @@ public abstract class OpTemplate implements Tagged {
      * Parse the statement for anchors and return a richer view of the StmtDef which
      * is simpler to use for most statement configuration needs.
      *
-     * @return an optional {@link ParsedStringTemplate}
+     * @return an optional {@link ParsedTemplateString}
      */
-    public Optional<ParsedStringTemplate> getParsed(Function<String, String>... rewriters) {
+    public Optional<ParsedTemplateString> getParsed(Function<String, String>... rewriters) {
         Optional<String> os = getStmt();
         return os.map(s -> {
             String result = s;
@@ -180,11 +180,11 @@ public abstract class OpTemplate implements Tagged {
                 result = rewriter.apply(result);
             }
             return result;
-        }).map(s -> new ParsedStringTemplate(s, getBindings()));
+        }).map(s -> new ParsedTemplateString(s, getBindings()));
     }
 
-    public Optional<ParsedStringTemplate> getParsed() {
+    public Optional<ParsedTemplateString> getParsed() {
-        return getStmt().map(s -> new ParsedStringTemplate(s, getBindings()));
+        return getStmt().map(s -> new ParsedTemplateString(s, getBindings()));
     }
 
     public abstract Optional<Map<String, Object>> getOp();

adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/VariableCapture.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 
 package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes;
 
-import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
 
 import java.util.Map;
 
@@ -24,7 +24,7 @@ import java.util.Map;
  * If an op implements VariableCapture, then it is known to be able to
  * extract variables from its result. Generally speaking, this should
  * be implemented within an Op according to the standard format
- * of {@link ParsedStringTemplate#getCaptures()}. Any op implementing
+ * of {@link ParsedTemplateString#getCaptures()}. Any op implementing
  * this should use the interface below to support interop between adapters
  * and to allow for auto documentation tha the feature is supported for
  * a given adapter.

adapters-api/src/main/java/io/nosqlbench/engine/api/templating/CommandTemplate.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@ import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
 import io.nosqlbench.api.config.params.ParamsParser;
 import io.nosqlbench.api.errors.BasicError;
 import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
-import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
 import io.nosqlbench.virtdata.core.templates.StringBindings;
 import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate;
 import org.apache.logging.log4j.Logger;
@@ -160,7 +160,7 @@ public class CommandTemplate {
         cmd.putAll(params);
 
         cmd.forEach((param, value) -> {
-            ParsedStringTemplate paramTemplate = new ParsedStringTemplate(value, bindings);
+            ParsedTemplateString paramTemplate = new ParsedTemplateString(value, bindings);
             if (paramTemplate.getBindPoints().size() > 0) {
                 BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
                 StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();

adapters-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedOp.java

@@ -16,18 +16,18 @@
 
 package io.nosqlbench.engine.api.templating;
 
+import io.nosqlbench.api.config.fieldreaders.DynamicFieldReader;
+import io.nosqlbench.api.config.fieldreaders.StaticFieldReader;
 import io.nosqlbench.api.config.standard.NBConfigError;
+import io.nosqlbench.api.config.standard.NBConfiguration;
+import io.nosqlbench.api.errors.OpConfigError;
 import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
 import io.nosqlbench.engine.api.templating.binders.ArrayBinder;
 import io.nosqlbench.engine.api.templating.binders.ListBinder;
 import io.nosqlbench.engine.api.templating.binders.OrderedMapBinder;
-import io.nosqlbench.api.config.fieldreaders.DynamicFieldReader;
-import io.nosqlbench.api.config.fieldreaders.StaticFieldReader;
-import io.nosqlbench.api.config.standard.NBConfiguration;
-import io.nosqlbench.api.errors.OpConfigError;
 import io.nosqlbench.virtdata.core.templates.BindPoint;
 import io.nosqlbench.virtdata.core.templates.CapturePoint;
-import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -302,7 +302,7 @@ public class ParsedOp implements LongFunction<Map<String, ?>>, StaticFieldReader
      * representation of a result. If the values are defined, then each one represents the name
      * that the found value should be saved as instead of the original name.
      */
-    private final List<List<CapturePoint>> captures = new ArrayList<>();
+    private final List<CapturePoint> captures = new ArrayList<>();
 
     private final OpTemplate _opTemplate;
     private final NBConfiguration activityCfg;
@@ -426,7 +426,7 @@ public class ParsedOp implements LongFunction<Map<String, ?>>, StaticFieldReader
         return tmap.getStaticValue(field);
     }
 
-    public Optional<ParsedStringTemplate> getAsTemplate(String fieldname) {
+    public Optional<ParsedTemplateString> getAsTemplate(String fieldname) {
         return this.tmap.getAsStringTemplate(fieldname);
     }
 
@@ -916,4 +916,8 @@ public class ParsedOp implements LongFunction<Map<String, ?>>, StaticFieldReader
     public String toString() {
         return this.tmap.toString();
     }
+
+    public List<CapturePoint> getCaptures() {
+        return tmap.getCaptures();
+    }
 }

adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedWorkloadTemplateTest.java

@@ -17,7 +17,7 @@
 package io.nosqlbench.engine.api.activityconfig.yaml;
 
 import io.nosqlbench.engine.api.activityconfig.OpsLoader;
-import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.BeforeAll;
@@ -27,8 +27,8 @@ import java.util.Map;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-public class ParsedOpTemplateTest {
+public class ParsedWorkloadTemplateTest {
-    private static final Logger logger = LogManager.getLogger(ParsedOpTemplateTest.class);
+    private static final Logger logger = LogManager.getLogger(ParsedWorkloadTemplateTest.class);
     private static OpsDocList doclist;
 
     @BeforeAll
@@ -40,13 +40,13 @@ public class ParsedOpTemplateTest {
     public void testBasicParser() {
         OpsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0);
         OpTemplate stmtDef0 = block0.getOps().get(0);
-        ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow();
+        ParsedTemplateString parsed0 = stmtDef0.getParsed().orElseThrow();
         assertThat(parsed0.getMissing()).containsExactly("delta");
         assertThat(parsed0.hasError()).isTrue();
 
         OpsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1);
         OpTemplate stmtDef1 = block1.getOps().get(0);
-        ParsedStringTemplate parsed1 = stmtDef1.getParsed().orElseThrow();
+        ParsedTemplateString parsed1 = stmtDef1.getParsed().orElseThrow();
         assertThat(parsed1.getMissing()).containsExactly();
         assertThat(parsed1.hasError()).isFalse();
     }
@@ -56,12 +56,12 @@ public class ParsedOpTemplateTest {
         OpsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2);
 
         OpTemplate stmtDef0 = block2.getOps().get(0);
-        ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow();
+        ParsedTemplateString parsed0 = stmtDef0.getParsed().orElseThrow();
         assertThat(parsed0.getMissing()).isEmpty();
         assertThat(parsed0.hasError()).isFalse();
 
         OpTemplate stmtDef1 = block2.getOps().get(1);
-        ParsedStringTemplate parsed1 = stmtDef1.getParsed().orElseThrow();
+        ParsedTemplateString parsed1 = stmtDef1.getParsed().orElseThrow();
         assertThat(parsed1.getMissing()).isEmpty();
         assertThat(parsed1.hasError()).isFalse();
     }

virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedSpanType.java

@@ -19,7 +19,7 @@ package io.nosqlbench.engine.api.templating;
 /**
  * The type of a parsed template depends on the structure of the bindings provided.
  */
-public enum BindType {
+public enum ParsedSpanType {
 
     /**
      * A literal template is one which has no bindings that need to be provided to render a specific statement.
@@ -32,7 +32,7 @@ public enum BindType {
      * A bindref template is one which has only a single bind point and no leading or trailing text.
      * It represents a single value which is to be injected, with no clear indication as to whether the
      * value should be in string form or not. These are used when referencing objects by bind point name.
-     * Callers which use rawbind templates where Strings are needed should convert them with {@link Object#toString()}}
+     * Callers which use rawbind templates where Strings are needed should convert them with {@link Object#toString()}
      * Example: <em>{@code {myvalue}}</em>
      */
     bindref,
@@ -40,6 +40,7 @@ public enum BindType {
     /**
      * A string template is one which is neither a literal template nor a bindref template. This includes
      * any template which has any amount of literal text and any template with more than one bind point.
+     * Example: <em>{@code Four {timeunit1} and several {timeunit2} ago}</em>
      */
     concat
 

virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateList.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 package io.nosqlbench.engine.api.templating;
 
+import io.nosqlbench.virtdata.core.templates.CapturePoint;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -25,6 +27,7 @@ public class ParsedTemplateList implements LongFunction<List<?>> {
     private final List<Object> protolist = new ArrayList<>();
     private final int[] dynamic_idx;
     private final LongFunction<?>[] functions;
+    private final List<CapturePoint> captures = new ArrayList<>();
 
     public ParsedTemplateList(List<Object> sublist, Map<String, String> bindings, List<Map<String, Object>> cfgsources) {
 
@@ -34,6 +37,7 @@ public class ParsedTemplateList implements LongFunction<List<?>> {
         for (int i = 0; i < sublist.size(); i++) {
             Object item = sublist.get(i);
             Templatizer.Result result = Templatizer.make(bindings, item, null, cfgsources);
+            this.captures.addAll(result.getCaptures());
             switch (result.getType()) {
                 case literal:
                     protolist.add(result.getValue());
@@ -64,4 +68,8 @@ public class ParsedTemplateList implements LongFunction<List<?>> {
     public boolean isStatic() {
         return dynamic_idx.length==0;
     }
+
+    public List<CapturePoint> getCaptures() {
+        return this.captures;
+    }
 }

virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateMap.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ import io.nosqlbench.virtdata.core.bindings.DataMapper;
 import io.nosqlbench.virtdata.core.bindings.VirtData;
 import io.nosqlbench.virtdata.core.templates.BindPoint;
 import io.nosqlbench.virtdata.core.templates.CapturePoint;
-import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
 import io.nosqlbench.virtdata.core.templates.StringBindings;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -76,7 +76,7 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
      * representation of a result. If the values are defined, then each one represents the name
      * that the found value should be saved as instead of the original name.
      */
-    private final List<List<CapturePoint>> captures = new ArrayList<>();
+    private final List<CapturePoint> captures = new ArrayList<>();
     private final int mapsize;
 
     /**
@@ -103,13 +103,13 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
         this.specmap = map;
         this.bindings = bindings;
         map.forEach((k, v) -> {
-            if (v instanceof CharSequence) {
+            if (v instanceof CharSequence charvalue) {
-                ParsedStringTemplate pt = ParsedStringTemplate.of(((CharSequence) v).toString(), bindings);
+                ParsedTemplateString pt = ParsedTemplateString.of(charvalue.toString(), bindings);
-                this.captures.add(pt.getCaptures());
+                this.captures.addAll(pt.getCaptures());
                 switch (pt.getType()) {
                     case literal:
-                        statics.put(k, ((CharSequence) v).toString());
+                        statics.put(k, charvalue.toString());
-                        protomap.put(k, ((CharSequence) v).toString());
+                        protomap.put(k, charvalue.toString());
                         break;
                     case bindref:
                         String spec = pt.asBinding().orElseThrow().getBindspec();
@@ -126,14 +126,15 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
                         protomap.put(k, null);
                         break;
                 }
-            } else if (v instanceof Map) {
+            } else if (v instanceof Map mapvalue) {
-                ((Map) v).keySet().forEach(smk -> {
+                mapvalue.keySet().forEach(smk -> {
                     if (!CharSequence.class.isAssignableFrom(smk.getClass())) {
                         throw new OpConfigError("Only string keys are allowed in submaps.");
                     }
                 });
                 Map<String, Object> submap = (Map<String, Object>) v;
                 ParsedTemplateMap subtpl = new ParsedTemplateMap(getName(),submap, bindings, cfgsources);
+                this.captures.addAll(subtpl.getCaptures());
                 if (subtpl.isStatic()) {
                     statics.put(k, submap);
                     protomap.put(k, submap);
@@ -141,9 +142,10 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
                     dynamics.put(k, subtpl);
                     protomap.put(k, null);
                 }
-            } else if (v instanceof List) {
+            } else if (v instanceof List listvalue) {
-                List<Object> sublist = (List<Object>) v;
+                List<Object> sublist = listvalue;
                 ParsedTemplateList subtpl = new ParsedTemplateList(sublist, bindings, cfgsources);
+                this.captures.addAll(subtpl.getCaptures());
                 if (subtpl.isStatic()) {
                     statics.put(k, sublist);
                     protomap.put(k, sublist);
@@ -162,6 +164,10 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
 
     }
 
+    public List<CapturePoint> getCaptures() {
+        return this.captures;
+    }
+
     /**
      * @return true if any field of this template map is dynamic
      */
@@ -695,11 +701,11 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
         return false;
     }
 
-    public Optional<ParsedStringTemplate> getAsStringTemplate(String fieldname) {
+    public Optional<ParsedTemplateString> getAsStringTemplate(String fieldname) {
         if (specmap.containsKey(fieldname)) {
             Object fval = specmap.get(fieldname);
             if (fval instanceof CharSequence) {
-                return Optional.of(new ParsedStringTemplate(fval.toString(), this.bindings));
+                return Optional.of(new ParsedTemplateString(fval.toString(), this.bindings));
             } else {
                 throw new RuntimeException("Can not make a parsed text template from op template field '" + fieldname + "' of type '" + fval.getClass().getSimpleName() + "'");
             }
@@ -982,4 +988,5 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
         }
         return sb.toString();
     }
+
 }

virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/Templatizer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@ import io.nosqlbench.api.errors.OpConfigError;
 import io.nosqlbench.virtdata.core.bindings.DataMapper;
 import io.nosqlbench.virtdata.core.bindings.VirtData;
 import io.nosqlbench.virtdata.core.templates.CapturePoint;
-import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
 import io.nosqlbench.virtdata.core.templates.StringBindings;
 
 import java.util.ArrayList;
@@ -36,7 +36,7 @@ public class Templatizer {
         result.setName(name);
 
         if (v instanceof CharSequence) {
-            ParsedStringTemplate pt = ParsedStringTemplate.of(((CharSequence) v).toString(), bindings);
+            ParsedTemplateString pt = ParsedTemplateString.of(((CharSequence) v).toString(), bindings);
             result.addCaptures(pt.getCaptures());
             result.setType(pt.getType());
             switch (pt.getType()) {
@@ -87,7 +87,7 @@ public class Templatizer {
     }
 
     public static class Result {
-        private BindType type;
+        private ParsedSpanType type;
         private final List<CapturePoint> captures = new ArrayList<>();
         private String name;
         private Object value;
@@ -97,7 +97,7 @@ public class Templatizer {
             this.type = null;
         }
 
-        public BindType getType() {
+        public ParsedSpanType getType() {
             return this.type;
         }
 
@@ -109,7 +109,7 @@ public class Templatizer {
             this.captures.addAll(captures);
         }
 
-        public void setType(BindType type) {
+        public void setType(ParsedSpanType type) {
             this.type = type;
         }
 

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/BindPoint.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -53,7 +53,15 @@ public class BindPoint {
          * a definition bindpoint is expressed as anything between double curly braces like <pre>{@code {{Identity()}}</pre>
          * or <pre>{@code {(Identity())}}
          */
-        definition
+        definition,
+        /**
+         * a reference to a captured variable, local to the current thread
+         */
+        localref,
+        /**
+         * a reference to a global captured variable, across the local JVM instance
+         */
+        globalref
     }
 
     public BindPoint(String anchor, String bindspec, Type type) {
@@ -84,7 +92,7 @@ public class BindPoint {
     }
 
     /**
-     * The name of the anchor for a binding as it appears in a user-specified template, parsed by {@link ParsedStringTemplate}.
+     * The name of the anchor for a binding as it appears in a user-specified template, parsed by {@link ParsedTemplateString}.
      * @return A string name for the bind point anchor, or null for {@link BindPoint.Type#definition} types.
      */
     public String getAnchor() {

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplateString.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 package io.nosqlbench.virtdata.core.templates;
 
 
-import io.nosqlbench.engine.api.templating.BindType;
+import io.nosqlbench.engine.api.templating.ParsedSpanType;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -74,7 +74,7 @@ import java.util.stream.StreamSupport;
  * <LI>provide a text template for re-assembly with injected data</LI>
  * </UL>
  *
- * Once the parsed template is constructed, the method {@link ParsedStringTemplate#orError()}
+ * Once the parsed template is constructed, the method {@link ParsedTemplateString#orError()}
  * should <em>always</em> called before it is used.
  *
  * <H2>Validity Checks</H2>
@@ -106,16 +106,16 @@ import java.util.stream.StreamSupport;
  * This is a list of binding names which were provided by the user, but which were not used in the raw template by name.
  * </p>
  */
-public class ParsedStringTemplate {
+public class ParsedTemplateString {
 
-    private final static Logger logger = LogManager.getLogger(ParsedStringTemplate.class);
+    private final static Logger logger = LogManager.getLogger(ParsedTemplateString.class);
 
     private final String rawtemplate;
-    private final List<CapturePoint> captures;
+    private final List<CapturePoint> captures = new ArrayList<>();
     private final List<BindPoint> bindpoints;
 
-    public static ParsedStringTemplate of(String rawtemplate, Map<String, String> bindings) {
+    public static ParsedTemplateString of(String rawtemplate, Map<String, String> bindings) {
-        return new ParsedStringTemplate(rawtemplate, bindings);
+        return new ParsedTemplateString(rawtemplate, bindings);
     }
 
     /**
@@ -130,19 +130,19 @@ public class ParsedStringTemplate {
     private final Map<String, String> bindings = new LinkedHashMap<>();
 
     /**
-     * Parse the given raw template, check the bind points against the provide bindings, and
+     * Parse the given raw template, check the bind points against the provided bindings, and
      * provide detailed template checks for validity.
      *
      * @param rawtemplate       A string template which contains optionally embedded named anchors
      * @param availableBindings The bindings which are provided by the user to fulfill the named anchors in this raw template
      */
-    public ParsedStringTemplate(String rawtemplate, Map<String, String> availableBindings) {
+    public ParsedTemplateString(String rawtemplate, Map<String, String> availableBindings) {
         this.bindings.putAll(availableBindings);
         this.rawtemplate = rawtemplate;
 
         CapturePointParser capturePointParser = new CapturePointParser();
         CapturePointParser.Result captureData = capturePointParser.apply(rawtemplate);
-        this.captures = captureData.getCaptures();
+        this.captures.addAll(captureData.getCaptures());
 
         BindPointParser bindPointParser = new BindPointParser();
         BindPointParser.Result bindPointsResult = bindPointParser.apply(captureData.getRawTemplate(), availableBindings);
@@ -150,17 +150,17 @@ public class ParsedStringTemplate {
         this.bindpoints = bindPointsResult.getBindpoints();
     }
 
-    public BindType getType() {
+    public ParsedSpanType getType() {
         if (this.spans.length == 1) {
-            return BindType.literal;
+            return ParsedSpanType.literal;
         } else if (this.spans[0].isEmpty() && this.spans[2].isEmpty()) {
-            return BindType.bindref;
+            return ParsedSpanType.bindref;
         } else {
-            return BindType.concat;
+            return ParsedSpanType.concat;
         }
     }
 
-    public ParsedStringTemplate orError() {
+    public ParsedTemplateString orError() {
         if (hasError()) {
             throw new RuntimeException("Unable to parse statement: " + this);
         }

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindings.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -37,15 +37,15 @@ public class StringBindings implements Binder<String> {
     }
 
     public StringBindings(String template, Map<String,String> bindings, Map<String,Object> fconfig) {
-        ParsedStringTemplate parsed = new ParsedStringTemplate(template,bindings);
+        ParsedTemplateString parsed = new ParsedTemplateString(template,bindings);
         this.compositor = new StringCompositor(parsed, fconfig);
     }
 
-    public StringBindings(ParsedStringTemplate parsedStringTemplate) {
+    public StringBindings(ParsedTemplateString parsedStringTemplate) {
         this(parsedStringTemplate, Map.of());
     }
 
-    public StringBindings(ParsedStringTemplate pt, Map<String,Object> fconfig) {
+    public StringBindings(ParsedTemplateString pt, Map<String,Object> fconfig) {
         this.compositor = new StringCompositor(pt,fconfig);
     }
 

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringCompositor.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@ public class StringCompositor implements LongFunction<String> {
 
     private final Function<Object, String> stringfunc;
 
-    public StringCompositor(ParsedStringTemplate template, Map<String,Object> fconfig, Function<Object,String> stringfunc) {
+    public StringCompositor(ParsedTemplateString template, Map<String,Object> fconfig, Function<Object,String> stringfunc) {
         Map<String,Integer> specs = new HashMap<>();
         List<BindPoint> bindpoints = template.getBindPoints();
         for (BindPoint bindPoint : bindpoints) {
@@ -69,7 +69,7 @@ public class StringCompositor implements LongFunction<String> {
         bufsize = minsize*2;
     }
 
-    public StringCompositor(ParsedStringTemplate template, Map<String,Object> fconfig) {
+    public StringCompositor(ParsedTemplateString template, Map<String,Object> fconfig) {
         this(template,fconfig,Object::toString);
     }
 

virtdata-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedTemplateMapTest.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.engine.api.templating;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ParsedTemplateMapTest {
+
+    @Test
+    public void testParsedTemplateMap() {
+        ParsedTemplateMap ptm = new ParsedTemplateMap("name1", Map.of("string1", "string2"), Map.of(), List.of());
+        assertThat(ptm.getOpFieldNames()).isEqualTo(Set.of("string1"));
+    }
+
+}

virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointParserTest.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.core.templates;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class CapturePointParserTest {
+
+    @Test
+    public void testCapturePoint1() {
+        CapturePointParser cpp = new CapturePointParser();
+        CapturePointParser.Result result = cpp.apply("string with [capture1]");
+        assertThat(result).isEqualTo(
+            new CapturePointParser.Result(
+                "string with capture1",
+                List.of(CapturePoint.of("capture1"))
+            )
+        );
+    }
+
+}

virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/FastStringCompositorTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,7 +28,7 @@ public class FastStringCompositorTest {
     public void testFastStringCompositor() {
         String rawTpl = "template {b1}, {{TestValue(5)}}";
         Map<String, String> bindings = Map.of("b1", "TestIdentity()");
-        ParsedStringTemplate ptpl = new ParsedStringTemplate(rawTpl, bindings);
+        ParsedTemplateString ptpl = new ParsedTemplateString(rawTpl, bindings);
         StringCompositor fsc = new StringCompositor(ptpl,Map.of());
         System.out.println(fsc);
     }

virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/ParsedTemplateStringTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,14 +16,14 @@
 
 package io.nosqlbench.virtdata.core.templates;
 
-import io.nosqlbench.engine.api.templating.BindType;
+import io.nosqlbench.engine.api.templating.ParsedSpanType;
 import org.junit.jupiter.api.Test;
 
 import java.util.Map;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-public class ParsedStringTemplateTest {
+public class ParsedTemplateStringTest {
 
     private final Map<String, String> bindings = Map.of(
             "bindname1", "bindspec1",
@@ -32,7 +32,7 @@ public class ParsedStringTemplateTest {
     @Test
     public void testShouldMatchRawLiteral() {
         String rawNothing = "This has no anchors";
-        ParsedStringTemplate pt = new ParsedStringTemplate(rawNothing, bindings);
+        ParsedTemplateString pt = new ParsedTemplateString(rawNothing, bindings);
         assertThat(pt.getSpans()).containsExactly("This has no anchors");
         assertThat(pt.getBindPoints()).isEmpty();
         assertThat(pt.getMissing()).isEmpty();
@@ -41,7 +41,7 @@ public class ParsedStringTemplateTest {
     @Test
     public void testShouldIgnoreExtraneousAnchors() {
         String oneExtraneous = "An {this is an extraneous form} invalid anchor.";
-        ParsedStringTemplate pt = new ParsedStringTemplate(oneExtraneous, bindings);
+        ParsedTemplateString pt = new ParsedTemplateString(oneExtraneous, bindings);
         assertThat(pt.getSpans()).containsExactly("An {this is an extraneous form} invalid anchor.");
         assertThat(pt.getBindPoints()).isEmpty();
         assertThat(pt.getMissing()).isEmpty();
@@ -50,7 +50,7 @@ public class ParsedStringTemplateTest {
     @Test
     public void testShouldAllowArbitraryNonGreedyInExtendedBindPoint() {
         String oneExtendedBindPoint = "An {{this is an extended form}} {{and another}} invalid anchor.";
-        ParsedStringTemplate pt = new ParsedStringTemplate(oneExtendedBindPoint, bindings);
+        ParsedTemplateString pt = new ParsedTemplateString(oneExtendedBindPoint, bindings);
         assertThat(pt.getSpans()).containsExactly("An ","this is an extended form"," ","and another"," invalid anchor.");
         assertThat(pt.getAnchors()).containsExactly("this is an extended form","and another");
     }
@@ -58,7 +58,7 @@ public class ParsedStringTemplateTest {
     @Test
     public void testShouldMatchLiteralVariableOnly() {
         String literalVariableOnly = "literal {bindname1}";
-        ParsedStringTemplate pt = new ParsedStringTemplate(literalVariableOnly, bindings);
+        ParsedTemplateString pt = new ParsedTemplateString(literalVariableOnly, bindings);
         assertThat(pt.getSpans()).containsExactly("literal ", "bindname1", "");
         assertThat(pt.getAnchors()).containsOnly("bindname1");
         assertThat(pt.getMissing()).isEmpty();
@@ -67,7 +67,7 @@ public class ParsedStringTemplateTest {
     @Test
     public void testShouldMatchVariableLiteralOnly() {
         String variableLiteralOnly = "{bindname2} literal";
-        ParsedStringTemplate pt = new ParsedStringTemplate(variableLiteralOnly, bindings);
+        ParsedTemplateString pt = new ParsedTemplateString(variableLiteralOnly, bindings);
         assertThat(pt.getSpans()).containsExactly("", "bindname2", " literal");
         assertThat(pt.getAnchors()).containsOnly("bindname2");
         assertThat(pt.getMissing()).isEmpty();
@@ -76,7 +76,7 @@ public class ParsedStringTemplateTest {
     @Test
     public void testPositionalExpansionShouldBeValid() {
         String multi = "A {bindname1} of {bindname2} sort.";
-        ParsedStringTemplate pt = new ParsedStringTemplate(multi, bindings);
+        ParsedTemplateString pt = new ParsedTemplateString(multi, bindings);
         assertThat(pt.getSpans()).containsExactly("A ", "bindname1", " of ", "bindname2", " sort.");
         assertThat(pt.getAnchors()).containsOnly("bindname1", "bindname2");
         assertThat(pt.getMissing()).isEmpty();
@@ -91,11 +91,11 @@ public class ParsedStringTemplateTest {
 
     @Test
     public void shouldMatchBasicCapturePoint() {
-        ParsedStringTemplate pt = new ParsedStringTemplate(
+        ParsedTemplateString pt = new ParsedTemplateString(
             "select [u],[v as v1] from users where userid={userid}", Map.of("userid", "NumberNameToString()")
         );
         assertThat(pt.getAnchors()).containsExactly("userid");
-        assertThat(pt.getType()).isEqualTo(BindType.concat);
+        assertThat(pt.getType()).isEqualTo(ParsedSpanType.concat);
         assertThat(pt.getCaptures()).containsExactly(CapturePoint.of("u"),CapturePoint.of("v","v1"));
 
     }

virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/StringCompositorTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,13 +26,13 @@ public class StringCompositorTest {
 
     @Test
     public void testShouldMatchSpanOnly() {
-        ParsedStringTemplate pt = new ParsedStringTemplate("A\\{ {one}two", Map.of());
+        ParsedTemplateString pt = new ParsedTemplateString("A\\{ {one}two", Map.of());
         assertThat(pt.getSpans()).containsExactly("A\\{ ", "one", "two");
     }
 
     @Test
     public void testShouldNotMatchEscaped() {
-        ParsedStringTemplate pt = new ParsedStringTemplate("A\\{{B}C",Map.of());
+        ParsedTemplateString pt = new ParsedTemplateString("A\\{{B}C",Map.of());
         assertThat(pt.getSpans()).containsExactly("A\\{","B","C");
     }
 

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_charbuffer/CharBufferExtract.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -50,12 +50,12 @@ public class CharBufferExtract implements LongFunction<CharBuffer> {
 
     public CharBufferExtract(Object initFunc, Object sizeFunc) {
         CharBuffer image = null;
-        if (initFunc instanceof Number) {
+        if (initFunc instanceof Number number) {
-            int bufsize = ((Number) initFunc).intValue();
+            int bufsize = number.intValue();
             this.image = new CharBufImage(bufsize).apply(1L);
         } else {
-            LongFunction<String> bbfunc = VirtDataConversions.adaptFunction(initFunc, LongFunction.class, String.class);
+            LongFunction<String> stringFunc = VirtDataConversions.adaptFunction(initFunc, LongFunction.class, String.class);
-            this.image = CharBuffer.wrap(bbfunc.apply(0));
+            this.image = CharBuffer.wrap(stringFunc.apply(0));
         }
         this.imgsize = this.image.limit();