Merge branch 'listfuncs'

# Conflicts:
#	activitytype-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
#	docsys/src/main/resources/docs-for-docsys/docsys/design/topics.md
#	virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/VirtDataComposer.java
#	virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindingsTemplate.java
#	virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListHashed.java
#	virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSized.java
#	virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSizedHashed.java
#	virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSizedStepped.java
#	virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListStepped.java

activitytype-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java

@@ -138,13 +138,23 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
         String format = getParams().getOptionalString("format").orElse(null);
 
         if ((stmts.size()==0 && stmtsDocList.getDocBindings().size() > 0) || format!=null) {
-            logger.info("Creating stdout statement template from bindings...");
+            if (format!=null && format.startsWith("diag")) {
-            String generatedStmt = genStatementTemplate(stmtsDocList.getDocBindings().keySet());
+                logger.info("Creating diagnostic log for resolver construction...");
-            BindingsTemplate bt = new BindingsTemplate();
+                BindingsTemplate bt = new BindingsTemplate();
-            stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
+                stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
-            StringBindingsTemplate sbt = new StringBindingsTemplate(generatedStmt, bt);
+                String diagnostics = bt.getDiagnostics();
-            StringBindings sb = sbt.resolve();
+                System.out.println(diagnostics);
-            sequencer.addOp(sb,1L);
+                System.out.flush();
+                System.exit(2);
+            } else {
+                logger.info("Creating stdout statement template from bindings, since none is otherwise defined.");
+                String generatedStmt = genStatementTemplate(stmtsDocList.getDocBindings().keySet());
+                BindingsTemplate bt = new BindingsTemplate();
+                stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
+                StringBindingsTemplate sbt = new StringBindingsTemplate(generatedStmt, bt);
+                StringBindings sb = sbt.resolve();
+                sequencer.addOp(sb,1L);
+            }
         } else if (stmts.size() > 0) {
             for (StmtDef stmt : stmts) {
                 ParsedStmt parsed = stmt.getParsed().orError();

activitytype-stdout/src/main/resources/stdout.md

@@ -33,9 +33,11 @@ activity types.
 - **newline** - whether to automatically add a missing newline to the end
    of any statements.
    default: true
-- **format** - which format to use. If provided, the format will override
+- **format** - which format to use. If provided, the format will override any statement formats provided by the YAML.
-   any statement formats provided by the YAML.
+  valid values are (csv, readout, json, inlinejson, assignments, and diag)
-   valid values are (csv, readout, json, inlinejson, and assignments)
+  - When 'format=diag', then the internal construction logic for the binding is logged in detail and nosqlbench exits.
+    This is useful for detailed diagnostics when you run into trouble, but not generally otherwise. This provides
+    details that you may include in a bug report if you think there is a bindings bug.
 
 ## Configuration
 

devdocs/inspirations.md

@@ -0,0 +1,15 @@
+# Virtual DocSys
+
+## Inspiring Examples
+
+These are doc sites that have examples of good docs.
+
+- [Apache Groovy](http://groovy-lang.org/documentation.html)
+- [Prometheus](https://prometheus.io/docs/prometheus/latest/querying/basics/)
+- [NetData](https://docs.netdata.cloud/)
+- [Optimizely](https://developers.optimizely.com/x/solutions/javascript/reference/index.html)
+- [Elastic Step by Step](https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-search.html)
+- [Javascript.info](https://javascript.info/strict-mode)
+- [TiKV docs - explore the tabs](https://tikv.org/docs/3.0/concepts/overview/)
+- [rocket](https://rocket.rs/v0.4/overview/)
+

docsys/src/main/resources/docs-for-docsys/docsys/design/docs_intro.md

@@ -1,77 +0,0 @@
-# Virtual DocSys
-
-## Inspiring Examples
-
-These are doc sites that have examples of good docs.
-
-- [Apache Groovy](http://groovy-lang.org/documentation.html)
-- [Prometheus](https://prometheus.io/docs/prometheus/latest/querying/basics/)
-- [NetData](https://docs.netdata.cloud/)
-- [Optimizely](https://developers.optimizely.com/x/solutions/javascript/reference/index.html)
-- [Elastic Step by Step](https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-search.html)
-- [Javascript.info](https://javascript.info/strict-mode)
-- [TiKV docs - explore the tabs](https://tikv.org/docs/3.0/concepts/overview/)
-- [rocket](https://rocket.rs/v0.4/overview/)
-
-## Architecture
-
-The RealSimpleDoc system is meant to be embedded within
-other projects. It has been design to require a minimum of
-additional programming at the web layer, while allowing the tool
-builder to focus on direct content transforms of file types.
-
-It is also designed to allow multiple projects to contribute
-documentation from their constituent components, with
-the contents being layered and composited dynamically.
-
-Thus, the primary interface to the web server is provided as a filesystem
-instance that presents the directory structure and file content as
-the designer would want the user to see it, including content from
-multiple contributing components. This shifts the classic problem
-of doing server-side web programming in-depth for basic content
-authoring to one of simply having the right transformers in place.
-
-## MetaFS
-
-The content provided to the web server from the filesystem is provided
-by a set of filesystem modules that are collectively called MetaFS. This
-consists of three specific filesystems which each serve a simple purpose:
-
-### VirtualFS
-
-The VirtualFS filesystem type is simply a way to provide a view to a
-filesystem that is rooted at some directory path in a host filesystem.
-For example, a VirtFS that is created with a host filesystem path of
-`/usr/local/branson` will contain a Path entry for `/` which will look
-and behave like a root directory, but all contents accessed via this
-path will come directly from the host's filesystem `/usr/local/branson`.
-
-### LayerFS
-
-This filesystem type implements the ability to combine multiple filesystems
-in a layered fashion. Any attempt to access a file or directory in this
-filesystem will cause an internal request to the filesystems that have
-been added. LayerFS follows a couple basic rules when answering a request:
-
-1. For calls that access attributes, the first response that 
-   contains attributes for a file that does exist will be used.
-2. When opening a file for write, the first filesystem which 
-   is writable will be used to open the file.
-3. Any requests which ask whether a file is readable or writable 
-   will have their answers filtered to match the effective best case.
-
-### RenderFS
-
-The RenderFS filesystem takes a set of transformers that are associated with a
-source and target file extension. The rules observed by this filesystem are:
-
-1. For directory listings, if there is a file with one of the source extensions,
-but the target file for that basename does not exist, then the directory listing
-is modified to show both.
-2. For file attribute views, if a call fails internally to find file attributes,
-and a source version of the base name exist for that extension, then a virtual
-attribute view is created that has all the same attributes as the base name, but
-with a different file name.
-3. If there is attempt to read file contents which are of a known target extension,
-where the source file exists, the content will be rendered from the source file
-and returned. 

engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/BindingEscapingTest.java

@@ -30,10 +30,22 @@ public class BindingEscapingTest {
     private final static Logger logger = LoggerFactory.getLogger(BindingEscapingTest.class);
 
     @Test
-    public void testEscapedBindings() {
+    public void testThatEscapesAreNotUnescapedInSingleQuotes() {
-        DataMapper<String> mapper = VirtData.getMapper("Template('\"-{}-\"Func(234)\\\"\\)',NumberNameToString());'",String.class);
+        DataMapper<String> mapper = VirtData.getMapper(
+            "Template('\"-{}-\"Func(234)\\\"\\)',NumberNameToString());"
+        );
         String s = mapper.get(234);
         assertThat(s).isEqualTo("\"-two hundred and thirty four-\"Func(234)\\\"\\)");
     }
 
+    @Test
+    public void testThatEscapesAreUnescapedInSingleQuotes() {
+        DataMapper<String> mapper =
+            VirtData.getMapper(
+                "Template(\"\\\"-{}-\\\"Func(234)\\\")\",NumberNameToString());"
+            );
+        String s = mapper.get(234);
+        assertThat(s).isEqualTo("\"-two hundred and thirty four-\"Func(234)\")");
+    }
+
 }

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/BindingsTemplate.java

@@ -38,7 +38,7 @@ import java.util.Optional;
  * bindings will be used in.
  */
 public class BindingsTemplate {
-    private final static Logger logger  = LogManager.getLogger(BindingsTemplate.class);
+    private final static Logger logger = LogManager.getLogger(BindingsTemplate.class);
     private List<String> bindPointNames = new ArrayList<>();
     private List<String> specifiers = new ArrayList<>();
 
@@ -47,11 +47,11 @@ public class BindingsTemplate {
 //    }
 
     public BindingsTemplate(List<String> anchors, List<String> specs) {
-        if (anchors.size()!=specs.size()) {
+        if (anchors.size() != specs.size()) {
             throw new InvalidParameterException("Anchors and Specifiers must be matched pair-wise.");
         }
         for (int i = 0; i < anchors.size(); i++) {
-            addFieldBinding(anchors.get(i),specs.get(i));
+            addFieldBinding(anchors.get(i), specs.get(i));
         }
     }
 
@@ -64,14 +64,15 @@ public class BindingsTemplate {
 
     public void addFieldBindings(List<BindPoint> bindPoints) {
         for (BindPoint bindPoint : bindPoints) {
-            addFieldBinding(bindPoint.getAnchor(),bindPoint.getBindspec());
+            addFieldBinding(bindPoint.getAnchor(), bindPoint.getBindspec());
         }
     }
 
     /**
      * Add a named binding specifier to the template
+     *
      * @param bindPointName the name associated with the binding specifier
-     * @param genSpec the binding specifier
+     * @param genSpec       the binding specifier
      */
     public void addFieldBinding(String bindPointName, String genSpec) {
         this.bindPointNames.add(bindPointName);
@@ -80,19 +81,39 @@ public class BindingsTemplate {
 
     /**
      * Add multiple named bindings to the template
+     *
      * @param bindPairs A map of named binding specifiers
      */
-    public void addFieldBindings(Map<String,String> bindPairs) {
+    public void addFieldBindings(Map<String, String> bindPairs) {
         for (Map.Entry<String, String> e : bindPairs.entrySet()) {
             this.bindPointNames.add(e.getKey());
             this.specifiers.add(e.getValue());
         }
     }
 
+    public String getDiagnostics() {
+        StringBuilder diaglog = new StringBuilder();
+        for (String specifier : specifiers) {
+            diaglog.append("for ").append(specifier).append(":");
+
+            ResolverDiagnostics mapperDiagnostics = VirtData.getMapperDiagnostics(specifier);
+            String diagnostics = mapperDiagnostics.toString();
+            diaglog.append(diagnostics);
+            if (mapperDiagnostics.getResolvedFunction().isPresent()) {
+                diaglog.append("☑ RESOLVED:")
+                    .append(mapperDiagnostics.getResolvedFunction().get().toString()).append("\n");
+            } else {
+                diaglog.append("☐ UNRESOLVED\n");
+            }
+        }
+        return diaglog.toString();
+    }
+
     /**
      * Use the data mapping library and the specifier to create instances of data mapping functions.
      * If you need thread-aware mapping, be sure to call this in the proper thread. Each time this method
      * is called, it creates a new instance.
+     *
      * @return A set of bindings that can be used to yield mapped data values later.
      */
     public Bindings resolveBindings() {
@@ -104,9 +125,9 @@ public class BindingsTemplate {
             } else {
                 logAvailableDataMappers();
                 throw new RuntimeException(
-                        "data mapper binding was unsuccessful for "
+                    "data mapper binding was unsuccessful for "
-                                + ", spec:" + specifier
+                        + ", spec:" + specifier
-                                + ", see log for known data mapper names.");
+                        + ", see log for known data mapper names.");
             }
         }
         return new Bindings(this, dataMappers);
@@ -149,7 +170,7 @@ public class BindingsTemplate {
             sb.append("=>[");
             sb.append(values[i]);
             sb.append("](");
-            sb.append((null!=values[i]) ? values[i].getClass().getSimpleName() : "NULL");
+            sb.append((null != values[i]) ? values[i].getClass().getSimpleName() : "NULL");
             sb.append(")");
             delim = ", ";
         }

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/VirtDataComposer.java

@@ -50,12 +50,12 @@ import java.util.stream.Collectors;
 public class VirtDataComposer {
 
     private final static String PREAMBLE = "compose ";
-    private final static Logger logger  = LogManager.getLogger(DataMapperLibrary.class);
+    private final static Logger logger = LogManager.getLogger(DataMapperLibrary.class);
     private final static MethodHandles.Lookup lookup = MethodHandles.publicLookup();
 
     private final VirtDataFunctionLibrary functionLibrary;
 
-    private final Map<String,Object> customElements = new HashMap<>();
+    private final Map<String, Object> customElements = new HashMap<>();
 
     public VirtDataComposer(VirtDataFunctionLibrary functionLibrary) {
         this.functionLibrary = functionLibrary;
@@ -91,23 +91,22 @@ public class VirtDataComposer {
 
     public ResolverDiagnostics resolveDiagnosticFunctionFlow(VirtDataFlow flow) {
         ResolverDiagnostics diagnostics = new ResolverDiagnostics();
-        diagnostics.trace("processing flow " + flow.toString() + " from output to input");
-
         LinkedList<List<ResolvedFunction>> funcs = new LinkedList<>();
 
         LinkedList<Set<Class<?>>> nextFunctionInputTypes = new LinkedList<>();
         Optional<Class<?>> finalValueTypeOption =
-                Optional.ofNullable(flow.getLastExpression().getCall().getOutputType())
+            Optional.ofNullable(flow.getLastExpression().getCall().getOutputType())
-                        .map(ValueType::valueOfClassName).map(ValueType::getValueClass);
+                .map(ValueType::valueOfClassName).map(ValueType::getValueClass);
 
         nextFunctionInputTypes.add(new HashSet<>());
         finalValueTypeOption.ifPresent(t -> nextFunctionInputTypes.get(0).add(t));
 
-        diagnostics.trace("working backwards from " + (flow.getExpressions().size()-1));
+        diagnostics.trace("working backwards from index " + (flow.getExpressions().size() - 1) + " to index 0");
 
         for (int i = flow.getExpressions().size() - 1; i >= 0; i--) {
             FunctionCall call = flow.getExpressions().get(i).getCall();
-            diagnostics.trace("resolving args for " + call.toString());
+            diagnostics.trace("FUNCTION[" + i + "]: " + call.toString() + ", resolving args");
+//            diagnostics.trace("resolving args for " + call.toString());
 
             List<ResolvedFunction> nodeFunctions = new LinkedList<>();
 
@@ -115,23 +114,24 @@ public class VirtDataComposer {
             Class<?> inputType = ValueType.classOfType(call.getInputType());
             Class<?> outputType = ValueType.classOfType(call.getOutputType());
             Object[] args = call.getArguments();
+
             try {
                 args = populateFunctions(diagnostics, args, this.customElements);
             } catch (Exception e) {
                 return diagnostics.error(e);
             }
 
-            diagnostics.trace("resolved args: ");
+            diagnostics.trace(" resolved args:");
             for (Object arg : args) {
-                diagnostics.trace(" " + arg.getClass().getSimpleName() + ": " + arg.toString());
+                diagnostics.trace(" " + arg.getClass().getSimpleName() + ": " + arg.getClass().getCanonicalName());
             }
 
-            List<ResolvedFunction> resolved = functionLibrary.resolveFunctions(outputType, inputType, funcName, this.customElements,args);
+            List<ResolvedFunction> resolved = functionLibrary.resolveFunctions(outputType, inputType, funcName, this.customElements, args);
             if (resolved.size() == 0) {
                 return diagnostics.error(new RuntimeException("Unable to find even one function for " + call));
             }
-            diagnostics.trace(" resolved functions:");
+            diagnostics.trace(" resolved functions");
-            diagnostics.trace(summarize(resolved));
+            diagnostics.trace(summarize(resolved, "  - "));
 
             nodeFunctions.addAll(resolved);
             funcs.addFirst(nodeFunctions);
@@ -153,10 +153,10 @@ public class VirtDataComposer {
         }
 
         FunctionAssembly assembly = new FunctionAssembly();
-        diagnostics.trace("composed summary: " + summarize(flattenedFuncs));
+//        diagnostics.trace("composed summary: " + summarize(flattenedFuncs));
 
         boolean isThreadSafe = true;
-        diagnostics.trace("FUNCTION chain selected: (multi) '" + this.summarize(flattenedFuncs) + "'");
+        diagnostics.trace("FUNCTION chain selected: (multi) '" + this.summarize(flattenedFuncs, "  - ") + "'");
         for (ResolvedFunction resolvedFunction : flattenedFuncs) {
             try {
                 Object functionObject = resolvedFunction.getFunctionObject();
@@ -165,7 +165,7 @@ public class VirtDataComposer {
                     isThreadSafe = false;
                 }
             } catch (Exception e) {
-                String flowdata = flow!=null? flow.toString() : "undefined";
+                String flowdata = flow != null ? flow.toString() : "undefined";
                 return diagnostics.error(new RuntimeException("FUNCTION resolution failed: '" + flowdata + "': " + e.toString()));
             }
         }
@@ -179,7 +179,7 @@ public class VirtDataComposer {
         return resolverDiagnostics.getResolvedFunction();
     }
 
-    private Object[] populateFunctions(ResolverDiagnostics diagnostics, Object[] args, Map<String,?> cconfig) {
+    private Object[] populateFunctions(ResolverDiagnostics diagnostics, Object[] args, Map<String, ?> cconfig) {
         for (int i = 0; i < args.length; i++) {
             Object o = args[i];
             if (o instanceof FunctionCall) {
@@ -189,7 +189,8 @@ public class VirtDataComposer {
                 Class<?> inputType = ValueType.classOfType(call.getInputType());
                 Class<?> outputType = ValueType.classOfType(call.getOutputType());
                 Object[] fargs = call.getArguments();
-                diagnostics.trace("resolving argument as function '" + call.toString() + "'");
+                diagnostics.trace(" arg (function): " + call.toString());
+//                diagnostics.trace("resolving argument as function '" + call.toString() + "'");
                 fargs = populateFunctions(diagnostics, fargs, cconfig);
 
                 List<ResolvedFunction> resolved = functionLibrary.resolveFunctions(outputType, inputType, funcName, cconfig, fargs);
@@ -216,9 +217,11 @@ public class VirtDataComposer {
         funcs.removeAll(toRemove);
     }
 
-    private String summarize(List<ResolvedFunction> funcs) {
+    private String summarize(List<ResolvedFunction> funcs, String prefix) {
         return funcs.stream()
-                .map(String::valueOf).collect(Collectors.joining("|"));
+            .map(String::valueOf)
+            .map(s -> prefix + s)
+            .collect(Collectors.joining("\n"));
     }
 
     private String summarizeBulk(List<List<ResolvedFunction>> funcs) {
@@ -226,9 +229,9 @@ public class VirtDataComposer {
         List<List<String>> spans = new LinkedList<>();
         funcs.forEach(l -> spans.add(l.stream().map(String::valueOf).collect(Collectors.toList())));
         List<Optional<Integer>> widths = spans.stream().map(
-                l -> l.stream().map(String::length).max(Integer::compare)).collect(Collectors.toList());
+            l -> l.stream().map(String::length).max(Integer::compare)).collect(Collectors.toList());
         String funcsdata = spans.stream().map(
-                l -> l.stream().map(String::valueOf).collect(Collectors.joining("|\n"))
+            l -> l.stream().map(String::valueOf).collect(Collectors.joining("|\n"))
         ).collect(Collectors.joining("\n\n"));
 
         StringBuilder sb = new StringBuilder();
@@ -322,7 +325,7 @@ public class VirtDataComposer {
             funcs.sort(ResolvedFunction.PREFERRED_TYPE_COMPARATOR);
             while (funcs.size() > 1) {
                 logger.trace("BY-SINGLE-PREFERRED-TYPE removing func " + funcs.get(funcs.size() - 1)
-                        + " because " + funcs.get(0) + " has more preferred types.");
+                    + " because " + funcs.get(0) + " has more preferred types.");
                 funcs.remove(funcs.size() - 1);
             }
 
@@ -337,7 +340,7 @@ public class VirtDataComposer {
             prevFuncs.sort(ResolvedFunction.PREFERRED_TYPE_COMPARATOR);
             while (prevFuncs.size() > 1) {
                 String logmsg = "BY-PREV-PREFERRED-TYPE removing func " + prevFuncs.get(prevFuncs.size() - 1)
-                        + " because " + prevFuncs.get(0) + " has more preferred types.";
+                    + " because " + prevFuncs.get(0) + " has more preferred types.";
                 logger.trace(logmsg);
                 prevFuncs.remove(prevFuncs.size() - 1);
             }
@@ -346,7 +349,7 @@ public class VirtDataComposer {
             nextFuncs.sort(ResolvedFunction.PREFERRED_TYPE_COMPARATOR);
             while (nextFuncs.size() > 1) {
                 String logmsg = "BY-NEXT-PREFERRED-TYPE removing func " + nextFuncs.get(nextFuncs.size() - 1)
-                        + " because " + nextFuncs.get(0) + " has more preferred types.";
+                    + " because " + nextFuncs.get(0) + " has more preferred types.";
                 logger.trace(logmsg);
                 nextFuncs.remove(nextFuncs.size() - 1);
             }
@@ -370,7 +373,7 @@ public class VirtDataComposer {
         Set<Class<?>> outputs = getOutputs(prevFuncs);
         Set<Class<?>> inputs = getInputs(nextFuncs);
         Set<Class<?>> directMatches =
-                inputs.stream().filter(outputs::contains).collect(Collectors.toCollection(HashSet::new));
+            inputs.stream().filter(outputs::contains).collect(Collectors.toCollection(HashSet::new));
 
         if (directMatches.size() > 0) {
             List<ResolvedFunction> toremove = new ArrayList<>();
@@ -424,9 +427,9 @@ public class VirtDataComposer {
             return 0;
         } else {
             toremove.forEach(nextfunc -> {
-                        String logmsg = "BY-ASSIGNABLE-TYPE removing next func: " + nextfunc + " because its input types are not assignable from any of the previous funcs";
+                    String logmsg = "BY-ASSIGNABLE-TYPE removing next func: " + nextfunc + " because its input types are not assignable from any of the previous funcs";
-                        logger.trace(logmsg);
+                    logger.trace(logmsg);
-                    }
+                }
             );
 
             nextFuncs.removeAll(toremove);
@@ -451,9 +454,10 @@ public class VirtDataComposer {
         return inputs;
     }
 
-    public Map<String,?> getCustomElements() {
+    public Map<String, ?> getCustomElements() {
         return this.customElements;
     }
+
     public VirtDataComposer addCustomElement(String name, Object element) {
         this.customElements.put(name, element);
         return this;

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindingsTemplate.java

@@ -63,6 +63,16 @@ public class StringBindingsTemplate {
         return new StringBindings(compositor,bindings);
     }
 
+    public String getDiagnostics() {
+        StringCompositor compositor = new StringCompositor(stringTemplate);
+        HashSet<String> unqualifiedNames = new HashSet<>(compositor.getBindPointNames());
+        unqualifiedNames.removeAll(new HashSet<>(bindingsTemplate.getBindPointNames()));
+        if (unqualifiedNames.size()>0) {
+            throw new RuntimeException("Named anchors were specified in the template which were not provided in the bindings: " + unqualifiedNames.toString());
+        }
+        return bindingsTemplate.getDiagnostics();
+    }
+
     @Override
     public String toString() {
         return "TEMPLATE:"+this.stringTemplate+" BINDING:"+bindingsTemplate.toString();

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListHashed.java

@@ -30,15 +30,13 @@ public class ListHashed implements LongFunction<List<Object>> {
 
     private final List<LongFunction<? extends Object>> valueFuncs;
     private final int size;
-    private final LongToIntFunction sizeFunc;
     private final Hash hasher = new Hash();
 
     @Example({
         "ListFunctions(NumberNameToString(),NumberNameToString())",
         "Create a list of ['one','one']"
     })
-    public ListHashed(LongToIntFunction sizeFunc, LongFunction<? extends Object>... funcs) {
+    public ListHashed(LongFunction<? extends Object>... funcs) {
-        this.sizeFunc = sizeFunc;
         this.valueFuncs = Arrays.asList(funcs);
         this.size = valueFuncs.size();
     }
@@ -47,12 +45,11 @@ public class ListHashed implements LongFunction<List<Object>> {
         "ListFunctions(NumberNameToString(),NumberNameToString())",
         "Create a list of ['one','one']"
     })
-    public ListHashed(LongToIntFunction sizeFunc, LongUnaryOperator... funcs) {
+    public ListHashed(LongUnaryOperator... funcs) {
         List<LongFunction<?>> building = new ArrayList<>(funcs.length);
         for (LongUnaryOperator func : funcs) {
             building.add(func::applyAsLong);
         }
-        this.sizeFunc = sizeFunc;
         this.valueFuncs = building;
         this.size = building.size();
     }
@@ -61,12 +58,11 @@ public class ListHashed implements LongFunction<List<Object>> {
         "ListFunctions(NumberNameToString(),NumberNameToString())",
         "Create a list of ['one','one']"
     })
-    public ListHashed(LongToIntFunction sizeFunc, Function<Long,Object>... funcs) {
+    public ListHashed(Function<Long,Object>... funcs) {
         List<LongFunction<?>> building = new ArrayList<>(funcs.length);
         for (Function<Long,Object> func : funcs) {
             building.add(func::apply);
         }
-        this.sizeFunc = sizeFunc;
         this.valueFuncs = building;
         this.size = building.size();
     }
@@ -74,7 +70,6 @@ public class ListHashed implements LongFunction<List<Object>> {
     @Override
     public List<Object> apply(long value) {
         long hash = value;
-        int size = sizeFunc.applyAsInt(value);
         List<Object> list = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
             int selector = Math.min(i, valueFuncs.size() - 1);

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSized.java

@@ -28,7 +28,6 @@ import java.util.function.LongUnaryOperator;
 public class ListSized implements LongFunction<List<Object>> {
 
     private final List<LongFunction<? extends Object>> valueFuncs;
-    private final int size;
     private final LongToIntFunction sizeFunc;
 
     @Example({
@@ -38,7 +37,6 @@ public class ListSized implements LongFunction<List<Object>> {
     public ListSized(LongToIntFunction sizeFunc, LongFunction<? extends Object>... funcs) {
         this.sizeFunc = sizeFunc;
         this.valueFuncs = Arrays.asList(funcs);
-        this.size = valueFuncs.size();
     }
 
     @Example({
@@ -52,7 +50,6 @@ public class ListSized implements LongFunction<List<Object>> {
         }
         this.sizeFunc = sizeFunc;
         this.valueFuncs = building;
-        this.size = building.size();
     }
 
     @Example({
@@ -66,7 +63,6 @@ public class ListSized implements LongFunction<List<Object>> {
         }
         this.sizeFunc = sizeFunc;
         this.valueFuncs = building;
-        this.size = building.size();
     }
 
     @Override

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSizedHashed.java

@@ -29,7 +29,6 @@ import java.util.function.LongUnaryOperator;
 public class ListSizedHashed implements LongFunction<List<Object>> {
 
     private final List<LongFunction<? extends Object>> valueFuncs;
-    private final int size;
     private final Hash hasher = new Hash();
     private final LongToIntFunction sizeFunc;
 
@@ -40,7 +39,6 @@ public class ListSizedHashed implements LongFunction<List<Object>> {
     public ListSizedHashed(LongToIntFunction sizeFunc, LongFunction<? extends Object>... funcs) {
         this.sizeFunc = sizeFunc;
         this.valueFuncs = Arrays.asList(funcs);
-        this.size = valueFuncs.size();
     }
 
     @Example({
@@ -54,7 +52,6 @@ public class ListSizedHashed implements LongFunction<List<Object>> {
         }
         this.sizeFunc = sizeFunc;
         this.valueFuncs = building;
-        this.size = building.size();
     }
 
     @Example({
@@ -68,7 +65,6 @@ public class ListSizedHashed implements LongFunction<List<Object>> {
         }
         this.sizeFunc = sizeFunc;
         this.valueFuncs = building;
-        this.size = building.size();
     }
 
     @Override

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSizedStepped.java

@@ -29,7 +29,6 @@ import java.util.function.LongUnaryOperator;
 public class ListSizedStepped implements LongFunction<List<Object>> {
 
     private final List<LongFunction<? extends Object>> valueFuncs;
-    private final int size;
     private final LongToIntFunction sizeFunc;
 
     @Example({
@@ -39,7 +38,6 @@ public class ListSizedStepped implements LongFunction<List<Object>> {
     public ListSizedStepped(LongToIntFunction sizeFunc, LongFunction<? extends Object>... funcs) {
         this.sizeFunc = sizeFunc;
         this.valueFuncs = Arrays.asList(funcs);
-        this.size = valueFuncs.size();
     }
 
     @Example({
@@ -53,7 +51,6 @@ public class ListSizedStepped implements LongFunction<List<Object>> {
         }
         this.sizeFunc = sizeFunc;
         this.valueFuncs = building;
-        this.size = building.size();
     }
 
     @Example({
@@ -67,7 +64,6 @@ public class ListSizedStepped implements LongFunction<List<Object>> {
         }
         this.sizeFunc = sizeFunc;
         this.valueFuncs = building;
-        this.size = building.size();
     }
 
     @Override

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListStepped.java

@@ -29,14 +29,12 @@ public class ListStepped implements LongFunction<List<Object>> {
 
     private final List<LongFunction<? extends Object>> valueFuncs;
     private final int size;
-    private final LongToIntFunction sizeFunc;
 
     @Example({
         "ListFunctions(NumberNameToString(),NumberNameToString())",
         "Create a list of ['one','one']"
     })
-    public ListStepped(LongToIntFunction sizeFunc, LongFunction<? extends Object>... funcs) {
+    public ListStepped(LongFunction<? extends Object>... funcs) {
-        this.sizeFunc = sizeFunc;
         this.valueFuncs = Arrays.asList(funcs);
         this.size = valueFuncs.size();
     }
@@ -45,12 +43,11 @@ public class ListStepped implements LongFunction<List<Object>> {
         "ListFunctions(NumberNameToString(),NumberNameToString())",
         "Create a list of ['one','one']"
     })
-    public ListStepped(LongToIntFunction sizeFunc, LongUnaryOperator... funcs) {
+    public ListStepped(LongUnaryOperator... funcs) {
         List<LongFunction<?>> building = new ArrayList<>(funcs.length);
         for (LongUnaryOperator func : funcs) {
             building.add(func::applyAsLong);
         }
-        this.sizeFunc = sizeFunc;
         this.valueFuncs = building;
         this.size = building.size();
     }
@@ -59,19 +56,17 @@ public class ListStepped implements LongFunction<List<Object>> {
         "ListFunctions(NumberNameToString(),NumberNameToString())",
         "Create a list of ['one','one']"
     })
-    public ListStepped(LongToIntFunction sizeFunc, Function<Long,Object>... funcs) {
+    public ListStepped(Function<Long,Object>... funcs) {
         List<LongFunction<?>> building = new ArrayList<>(funcs.length);
         for (Function<Long,Object> func : funcs) {
             building.add(func::apply);
         }
-        this.sizeFunc = sizeFunc;
         this.valueFuncs = building;
         this.size = building.size();
     }
 
     @Override
     public List<Object> apply(long value) {
-        int size = sizeFunc.applyAsInt(value);
         List<Object> list = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
             int selector = Math.min(i, valueFuncs.size() - 1);

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/Template.java

@@ -19,6 +19,9 @@ import java.util.regex.Pattern;
  * result of the provided functions. The number of <code>{}</code> entries in the template
  * must strictly match the number of functions or an error will be thrown.
  *
+ * If you need to include single quotes or other special characters, you may use a
+ * backslash "\" in your template.
+ *
  * The objects passed must be functions of any of the following types:
  * <UL>
  *     <LI>LongUnaryOperator</LI>
@@ -66,7 +69,82 @@ public class Template implements LongFunction<String> {
         return adapted.toArray(new LongFunction<?>[0]);
     }
 
-//    public Template(String template, LongFunction<?>... funcs) {
+
+    /**
+     * If an operator is provided, it is used to change the function input value in an additional way before each function.
+     *
+     * @param iterOp   A pre-generation value mapping function
+     * @param template A string template containing <pre>{}</pre> anchors
+     * @param funcs    A varargs length of LongFunctions of any output type
+     */
+    public Template(LongUnaryOperator iterOp, String template, LongFunction<?>... funcs) {
+        this(template, funcs);
+        this.iterOp = iterOp;
+    }
+
+    @SuppressWarnings("unchecked")
+    private String[] parseTemplate(String template, int funcCount) {
+        try {
+            List<String> literals = new ArrayList<>();
+            Pattern p = Pattern.compile("\\{}");
+            Matcher m = p.matcher(template);
+            int pos = 0;
+            while (m.find()) {
+                literals.add(template.substring(pos, m.start()));
+                pos = m.end();
+            }
+            String partial = template.substring(pos);
+//            partial = unescape(partial);
+            literals.add(partial);
+            if (literals.size() != funcCount + 1) {
+                throw new RuntimeException("The number of {} place holders in '" + template + "' should equal the number of functions.");
+            }
+            return literals.toArray(new String[0]);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+//    // for testing
+//    public String unescape(String partial) {
+//        StringBuilder unescaped = new StringBuilder();
+//        try {
+//            Pattern escapes = Pattern.compile("\\\\.");
+//            Matcher m = escapes.matcher(partial);
+//            int pos = 0;
+//
+//            while (m.find()) {
+//                String prefix = partial.substring(pos,m.start());
+//                unescaped.append(prefix);
+//                String segment = partial.substring(m.start(),m.end());
+//                unescaped.append(segment.substring(1));
+//                pos = m.end();
+//            }
+//            unescaped.append(partial.substring(pos));
+//
+//        } catch (Exception e) {
+//            throw new RuntimeException(e);
+//        }
+//        return unescaped.toString();
+//    }
+
+    @Override
+    public String apply(long value) {
+        StringBuilder buffer = sb.get();
+        buffer.setLength(0);
+        buffer.append(literals[0]);
+        if (literals.length > 1) {
+            for (int i = 0; i < adaptedFuncs.length; i++) {
+                long input = iterOp != null ? iterOp.applyAsLong(value + i) : value + i;
+                String genString = String.valueOf(adaptedFuncs[i].apply(input));
+                buffer.append(genString);
+                buffer.append(literals[i + 1]);
+            }
+        }
+        return buffer.toString();
+    }
+
+    //    public Template(String template, LongFunction<?>... funcs) {
 //        this.adaptedFuncs = funcs;
 //        this.rawTemplate = template;
 //        this.literals = parseTemplate(template, funcs.length);
@@ -134,52 +212,4 @@ public class Template implements LongFunction<String> {
 //
 //    }
 
-    /**
-     * If an operator is provided, it is used to change the function input value in an additional way before each function.
-     *
-     * @param iterOp   A pre-generation value mapping function
-     * @param template A string template containing <pre>{}</pre> anchors
-     * @param funcs    A varargs length of LongFunctions of any output type
-     */
-    public Template(LongUnaryOperator iterOp, String template, LongFunction<?>... funcs) {
-        this(template, funcs);
-        this.iterOp = iterOp;
-    }
-
-    @SuppressWarnings("unchecked")
-    private String[] parseTemplate(String template, int funcCount) {
-        try {
-            List<String> literals = new ArrayList<>();
-            Pattern p = Pattern.compile("\\{}");
-            Matcher m = p.matcher(template);
-            int pos = 0;
-            while (m.find()) {
-                literals.add(template.substring(pos, m.start()));
-                pos = m.end();
-            }
-            literals.add(template.substring(pos));
-            if (literals.size() != funcCount + 1) {
-                throw new RuntimeException("The number of {} place holders in '" + template + "' should equal the number of functions.");
-            }
-            return literals.toArray(new String[0]);
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
-    public String apply(long value) {
-        StringBuilder buffer = sb.get();
-        buffer.setLength(0);
-        buffer.append(literals[0]);
-        if (literals.length > 1) {
-            for (int i = 0; i < adaptedFuncs.length; i++) {
-                long input = iterOp != null ? iterOp.applyAsLong(value + i) : value + i;
-                String genString = String.valueOf(adaptedFuncs[i].apply(input));
-                buffer.append(genString);
-                buffer.append(literals[i + 1]);
-            }
-        }
-        return buffer.toString();
-    }
 }

virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/unary_string/TemplateTest.java

@@ -52,4 +52,25 @@ public class TemplateTest {
         }
     }
 
-}
+//    @Test
+//    public void testUnescaping() {
+//        String escaped="front \\' back";
+//        LongFunction<String> func = String::valueOf;
+//        Template template = new Template("{} extra", func);
+//        String unescaped = template.unescape(escaped);
+//        assertThat(unescaped).isEqualTo("front ' back");
+//
+//        String unescaped2= template.unescape("\\' one \\' two \\'");
+//        assertThat(unescaped2).isEqualTo("' one ' two '");
+//    }
+//
+//    @Test
+//    public void testBackslashUnescaping() {
+//        String escaped="front \\\\\" back";
+//        LongFunction<String> func = String::valueOf;
+//        Template template = new Template("{} extra", func);
+//        String unescaped = template.unescape(escaped);
+//        assertThat(unescaped).isEqualTo("front \\\" back");
+//
+//    }
+}