From 35871718d51863bfb05948badb6789c9aca5fead Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 15 Feb 2022 21:23:46 -0600 Subject: [PATCH] improve string bindings API and functionality --- .../core/bindings/BindingsTemplate.java | 25 +++-- .../core/templates/BindPointParser.java | 4 + .../core/templates/ParsedTemplate.java | 6 +- .../core/templates/StringBindings.java | 35 ++++-- .../templates/StringBindingsTemplate.java | 49 +------- .../core/templates/StringCompositor.java | 105 +++++++++--------- .../templates/FastStringCompositorTest.java | 20 ++++ .../core/templates/StringCompositorTest.java | 12 +- .../virtdata/core/templates/TestIdentity.java | 14 +++ .../virtdata/core/templates/TestValue.java | 19 ++++ ...java => IntegratedStringBindingsTest.java} | 51 ++++----- 11 files changed, 179 insertions(+), 161 deletions(-) create mode 100644 virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/FastStringCompositorTest.java create mode 100644 virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/TestIdentity.java create mode 100644 virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/TestValue.java rename virtdata-userlibs/src/test/java/io/virtdata/{IntegratedStringCompositorTest.java => IntegratedStringBindingsTest.java} (50%) diff --git a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/BindingsTemplate.java b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/BindingsTemplate.java index 42a0cfa5f..d3c5f4843 100644 --- a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/BindingsTemplate.java +++ b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/BindingsTemplate.java @@ -25,10 +25,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import java.security.InvalidParameterException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; /** * Maps a set of parameters on an associated object of type T to specifiers for data mappers. @@ -38,14 +35,11 @@ import java.util.Optional; * bindings will be used in. */ public class BindingsTemplate { + private final static Logger logger = LogManager.getLogger(BindingsTemplate.class); private final Map fconfig; - private List bindPointNames = new ArrayList<>(); - private List specifiers = new ArrayList<>(); - -// public BindingsTemplate(Map specs) { -// specs.forEach(this::addFieldBinding); -// } + private final List bindPointNames = new ArrayList<>(); + private final List specifiers = new ArrayList<>(); public BindingsTemplate(Map config, List anchors, List specs) { this.fconfig = config; @@ -61,10 +55,10 @@ public class BindingsTemplate { this.fconfig = config; addFieldBindings(bindpoints); } + public BindingsTemplate(List bindPoints) { this.fconfig = Map.of(); addFieldBindings(bindPoints); - } public BindingsTemplate(Map config) { @@ -114,7 +108,7 @@ public class BindingsTemplate { diaglog.append(diagnostics); if (mapperDiagnostics.getResolvedFunction().isPresent()) { diaglog.append("☑ RESOLVED:") - .append(mapperDiagnostics.getResolvedFunction().get().toString()).append("\n"); + .append(mapperDiagnostics.getResolvedFunction().get()).append("\n"); } else { diaglog.append("☐ UNRESOLVED\n"); } @@ -190,4 +184,11 @@ public class BindingsTemplate { return sb.toString(); } + public Map getMap() { + LinkedHashMap bindmap = new LinkedHashMap<>(); + for (int i = 0; i < this.specifiers.size(); i++) { + bindmap.put(this.bindPointNames.get(i),this.specifiers.get(i)); + } + return bindmap; + } } diff --git a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/BindPointParser.java b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/BindPointParser.java index e3e4bf6c8..9e3706564 100644 --- a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/BindPointParser.java +++ b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/BindPointParser.java @@ -33,6 +33,7 @@ public class BindPointParser implements BiFunction, List spans = new ArrayList<>(); List bindpoints = new ArrayList<>(); + int genid=0; while (m.find()) { String pre = template.substring(lastMatch, m.start()); spans.add(pre); @@ -67,6 +68,9 @@ public class BindPointParser implements BiFunction, this.bindpoints = bindpoints; } + /** + * @return the spans of literal values which are between the bind points + */ public List getSpans() { return spans; } diff --git a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplate.java b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplate.java index 746430a48..ceede5529 100644 --- a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplate.java +++ b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplate.java @@ -130,9 +130,6 @@ public class ParsedTemplate { */ private final Map bindings = new LinkedHashMap<>(); - private final BindPointParser bindPointParser = new BindPointParser(); - private final CapturePointParser capturePointParser = new CapturePointParser(); - /** * Parse the given raw template, check the bind points against the provide bindings, and * provide detailed template checks for validity. @@ -144,8 +141,11 @@ public class ParsedTemplate { this.bindings.putAll(availableBindings); this.rawtemplate = rawtemplate; + CapturePointParser capturePointParser = new CapturePointParser(); CapturePointParser.Result captureData = capturePointParser.apply(rawtemplate); this.captures = captureData.getCaptures(); + + BindPointParser bindPointParser = new BindPointParser(); BindPointParser.Result bindPointsResult = bindPointParser.apply(captureData.getRawTemplate(), availableBindings); this.spans = bindPointsResult.getSpans().toArray(new String[0]); this.bindpoints = bindPointsResult.getBindpoints(); diff --git a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindings.java b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindings.java index 111b97e05..cf6c039a7 100644 --- a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindings.java +++ b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindings.java @@ -1,25 +1,40 @@ package io.nosqlbench.virtdata.core.templates; import io.nosqlbench.virtdata.core.bindings.Binder; -import io.nosqlbench.virtdata.core.bindings.Bindings; import io.nosqlbench.virtdata.core.bindings.BindingsTemplate; +import java.util.Map; + /** * Allows the generation of strings from a string template and bindings template. */ public class StringBindings implements Binder { private final StringCompositor compositor; - private final Bindings bindings; - public StringBindings(StringCompositor compositor, Bindings bindings) { - this.compositor = compositor; - this.bindings = bindings; + public StringBindings(String template) { + this(template,Map.of(),Map.of()); } - public StringBindings(ParsedTemplate pt) { - this.compositor = new StringCompositor(pt); - this.bindings = new BindingsTemplate(pt.getBindPoints()).resolveBindings(); + public StringBindings(String template, Map bindings) { + this(template,bindings,Map.of()); + } + + public StringBindings(String template, Map bindings, Map fconfig) { + ParsedTemplate parsed = new ParsedTemplate(template,bindings); + this.compositor = new StringCompositor(parsed, fconfig); + } + + public StringBindings(ParsedTemplate parsedTemplate) { + this(parsedTemplate, Map.of()); + } + + public StringBindings(ParsedTemplate pt, Map fconfig) { + this.compositor = new StringCompositor(pt,fconfig); + } + + public StringBindings(String stringTemplate, BindingsTemplate bindingsTemplate) { + this(stringTemplate,bindingsTemplate.getMap()); } /** @@ -30,15 +45,13 @@ public class StringBindings implements Binder { */ @Override public String bind(long value) { - String s = compositor.bindValues(compositor, bindings, value); - return s; + return compositor.apply(value); } @Override public String toString() { return "StringBindings{" + "compositor=" + compositor + - ", bindings=" + bindings + '}'; } } diff --git a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindingsTemplate.java b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindingsTemplate.java index 3009c3835..151b52950 100644 --- a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindingsTemplate.java +++ b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindingsTemplate.java @@ -2,10 +2,8 @@ package io.nosqlbench.virtdata.core.templates; //import io.nosqlbench.engine.api.activityconfig.ParsedStmt; //import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef; -import io.nosqlbench.virtdata.core.bindings.Bindings; -import io.nosqlbench.virtdata.core.bindings.BindingsTemplate; -import java.util.HashSet; +import io.nosqlbench.virtdata.core.bindings.BindingsTemplate; /** * Uses a string template and a bindings template to create instances of {@link StringBindings}. @@ -20,55 +18,12 @@ public class StringBindingsTemplate { this.bindingsTemplate = bindingsTemplate; } -// /** -// * Build a default string bindings template using the standard representation -// * for a string template in NoSQLBench, which is a literal string interspersed -// * with named anchors in {@code {{curlybraces}}} form. -// * @param stmtDef A stmtDef -// */ -// public StringBindingsTemplate(StmtDef stmtDef) { -// this(stmtDef, s->"{{"+s+"}}"); -// } -// -// /** -// * Build a string bindings template using a custom representation that maps -// * the named anchors to a different form than the default {@code {{curlybraces}}} form. -// * The mapping function provides the textual substitution which is used to composite -// * the normative representation of the statement. -// * @param stmtdef The {@link StmtDef} which provides the bindpoints -// * @param tokenMapper A custom named anchor formatting function -// */ -// public StringBindingsTemplate(StmtDef stmtdef, Function tokenMapper) { -// ParsedStmt parsedStmt = stmtdef.getParsed().orError(); -// this.stringTemplate = parsedStmt.getPositionalStatement(tokenMapper); -// this.bindingsTemplate = new BindingsTemplate(parsedStmt.getBindPoints()); -// } - /** * Create a new instance of {@link StringBindings}, preferably in the thread context that will use it. * @return a new StringBindings */ public StringBindings resolve() { - - StringCompositor compositor = new StringCompositor(stringTemplate); - HashSet unqualifiedNames = new HashSet<>(compositor.getBindPointNames()); - unqualifiedNames.removeAll(new HashSet<>(bindingsTemplate.getBindPointNames())); - if (unqualifiedNames.size()>0) { - throw new RuntimeException("Named anchors were specified in the template which were not provided in the bindings: " + unqualifiedNames); - } - - Bindings bindings = bindingsTemplate.resolveBindings(); - return new StringBindings(compositor,bindings); - } - - public String getDiagnostics() { - StringCompositor compositor = new StringCompositor(stringTemplate); - HashSet unqualifiedNames = new HashSet<>(compositor.getBindPointNames()); - unqualifiedNames.removeAll(new HashSet<>(bindingsTemplate.getBindPointNames())); - if (unqualifiedNames.size()>0) { - throw new RuntimeException("Named anchors were specified in the template which were not provided in the bindings: " + unqualifiedNames); - } - return bindingsTemplate.getDiagnostics(); + return new StringBindings(stringTemplate,bindingsTemplate); } @Override diff --git a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringCompositor.java b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringCompositor.java index ff43e7eec..b3092e04f 100644 --- a/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringCompositor.java +++ b/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringCompositor.java @@ -1,74 +1,73 @@ package io.nosqlbench.virtdata.core.templates; -import io.nosqlbench.virtdata.core.bindings.ValuesBinder; -import io.nosqlbench.virtdata.core.bindings.Bindings; +import io.nosqlbench.virtdata.core.bindings.DataMapper; +import io.nosqlbench.virtdata.core.bindings.VirtData; -import java.util.ArrayList; -import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.function.Function; +import java.util.function.LongFunction; /** - * StringCompositor provides a way to build strings from a string template and provided values. - * - *

- * The template is simply an array of string values, where odd indices represent token positions, and even indices represent - * literals. This version of the StringCompositor fetches data from the bindings only for the named fields in the template. - *

+ * This implementation of a string compositor takes a logically coherent + * string template and bindings set. It employs a few simplistic optimizations + * to avoid re-generating duplicate values, as well as lower allocation + * rate of buffer data. */ -public class StringCompositor implements ValuesBinder { +public class StringCompositor implements LongFunction { - private final String[] templateSegments; - private Function stringfunc = String::valueOf; + private final String[] spans; + private final DataMapper[] mappers; + private final int[] LUT; + private final int bufsize; - /** - * Create a string template which has positional tokens, in "{}" form. - * - * @param template The string template - */ - public StringCompositor(String template) { - templateSegments = parseTemplate(template); - } + private final Function stringfunc; - public StringCompositor(String template, Function stringfunc) { - this(template); + public StringCompositor(ParsedTemplate template, Map fconfig, Function stringfunc) { + Map specs = new HashMap<>(); + List bindpoints = template.getBindPoints(); + for (BindPoint bindPoint : bindpoints) { + String spec = bindPoint.getBindspec(); + specs.compute(spec,(s,i) -> i==null ? specs.size() : i); + } + mappers = new DataMapper[specs.size()]; + specs.forEach((k,v) -> { + mappers[v]= VirtData.getOptionalMapper(k,fconfig).orElseThrow(); + }); + String[] even_odd_spans = template.getSpans(); + this.spans = new String[bindpoints.size()+1]; + LUT = new int[bindpoints.size()]; + for (int i = 0; i < bindpoints.size(); i++) { + spans[i]=even_odd_spans[i<<1]; + LUT[i]=specs.get(template.getBindPoints().get(i).getBindspec()); + } + spans[spans.length-1]=even_odd_spans[even_odd_spans.length-1]; this.stringfunc = stringfunc; + + int minsize = 0; + for (int i = 0; i < 100; i++) { + String result = apply(i); + minsize = Math.max(minsize,result.length()); + } + bufsize = minsize*2; } - public StringCompositor(ParsedTemplate pt) { - templateSegments = pt.getSpans(); + public StringCompositor(ParsedTemplate template, Map fconfig) { + this(template,fconfig,Object::toString); } - // for testing - protected String[] parseTemplate(String template) { - ParsedTemplate parsed = new ParsedTemplate(template, Collections.emptyMap()); - return parsed.getSpans(); - } - - @Override - public String bindValues(StringCompositor context, Bindings bindings, long cycle) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < templateSegments.length; i++) { - if (i % 2 == 0) { - sb.append(templateSegments[i]); - } else { - String key = templateSegments[i]; - Object value = bindings.get(key, cycle); - String valueString = stringfunc.apply(value); - sb.append(valueString); - } + public String apply(long value) { + StringBuilder sb = new StringBuilder(bufsize); + String[] ary = new String[mappers.length]; + for (int i = 0; i < ary.length; i++) { + ary[i] = stringfunc.apply(mappers[i].apply(value)); } + for (int i = 0; i < LUT.length; i++) { + sb.append(spans[i]).append(ary[LUT[i]]); + } + sb.append(spans[spans.length-1]); return sb.toString(); } - - public List getBindPointNames() { - List tokens = new ArrayList<>(); - for (int i = 0; i < templateSegments.length; i++) { - if (i % 2 == 1) { - tokens.add(templateSegments[i]); - } - } - return tokens; - } } diff --git a/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/FastStringCompositorTest.java b/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/FastStringCompositorTest.java new file mode 100644 index 000000000..3857d111d --- /dev/null +++ b/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/FastStringCompositorTest.java @@ -0,0 +1,20 @@ +package io.nosqlbench.virtdata.core.templates; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +public class FastStringCompositorTest { + + @Test + @Disabled // Needs to have annotation processor run in test scope first + public void testFastStringCompositor() { + String rawTpl = "template {b1}, {{TestValue(5)}}"; + Map bindings = Map.of("b1", "TestIdentity()"); + ParsedTemplate ptpl = new ParsedTemplate(rawTpl, bindings); + StringCompositor fsc = new StringCompositor(ptpl,Map.of()); + System.out.println(fsc); + } + +} diff --git a/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/StringCompositorTest.java b/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/StringCompositorTest.java index c0a177d38..85ed84efb 100644 --- a/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/StringCompositorTest.java +++ b/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/StringCompositorTest.java @@ -2,22 +2,22 @@ package io.nosqlbench.virtdata.core.templates; import org.junit.jupiter.api.Test; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; public class StringCompositorTest { @Test public void testShouldMatchSpanOnly() { - StringCompositor c = new StringCompositor("A"); - String[] spans = c.parseTemplate("A\\{ {one}two"); - assertThat(spans).containsExactly("A\\{ ", "one", "two"); + ParsedTemplate pt = new ParsedTemplate("A\\{ {one}two", Map.of()); + assertThat(pt.getSpans()).containsExactly("A\\{ ", "one", "two"); } @Test public void testShouldNotMatchEscaped() { - StringCompositor c = new StringCompositor("A"); - String[] spans = c.parseTemplate("A\\{{B}C"); - assertThat(spans).containsExactly("A\\{","B","C"); + ParsedTemplate pt = new ParsedTemplate("A\\{{B}C",Map.of()); + assertThat(pt.getSpans()).containsExactly("A\\{","B","C"); } // @Test diff --git a/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/TestIdentity.java b/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/TestIdentity.java new file mode 100644 index 000000000..9e924c364 --- /dev/null +++ b/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/TestIdentity.java @@ -0,0 +1,14 @@ +package io.nosqlbench.virtdata.core.templates; + +import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; + +import java.util.function.LongFunction; + +@ThreadSafeMapper +public class TestIdentity implements LongFunction { + + @Override + public Object apply(long value) { + return value; + } +} diff --git a/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/TestValue.java b/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/TestValue.java new file mode 100644 index 000000000..8ffc98f59 --- /dev/null +++ b/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/TestValue.java @@ -0,0 +1,19 @@ +package io.nosqlbench.virtdata.core.templates; + +import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; + +import java.util.function.LongFunction; + +@ThreadSafeMapper +public class TestValue implements LongFunction { + + private final Object value; + + public TestValue(Object value) { + this.value = value; + } + @Override + public Object apply(long value) { + return this.value; + } +} diff --git a/virtdata-userlibs/src/test/java/io/virtdata/IntegratedStringCompositorTest.java b/virtdata-userlibs/src/test/java/io/virtdata/IntegratedStringBindingsTest.java similarity index 50% rename from virtdata-userlibs/src/test/java/io/virtdata/IntegratedStringCompositorTest.java rename to virtdata-userlibs/src/test/java/io/virtdata/IntegratedStringBindingsTest.java index f89d3805d..e70eaabf1 100644 --- a/virtdata-userlibs/src/test/java/io/virtdata/IntegratedStringCompositorTest.java +++ b/virtdata-userlibs/src/test/java/io/virtdata/IntegratedStringBindingsTest.java @@ -2,15 +2,13 @@ package io.virtdata; import io.nosqlbench.virtdata.core.bindings.Bindings; import io.nosqlbench.virtdata.core.bindings.BindingsTemplate; -import io.nosqlbench.virtdata.core.templates.StringCompositor; +import io.nosqlbench.virtdata.core.templates.StringBindings; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.util.function.Function; - import static org.assertj.core.api.Assertions.assertThat; -public class IntegratedStringCompositorTest { +public class IntegratedStringBindingsTest { private static BindingsTemplate template; private static Bindings bindings; @@ -27,41 +25,36 @@ public class IntegratedStringCompositorTest { bindings = bindingsTemplate.resolveBindings(); } + @Test + public void testEven() { + StringBindings c = new StringBindings("A{ident}B{ident}C{mod5}D{mod-5}",template); + String bind3 = c.bind(3); + assertThat(bind3).isEqualTo("A3B3C3D3"); + } + + @Test + public void testOdd() { + StringBindings c = new StringBindings("A{ident}B{ident}C{mod5}D{mod-5}E",template); + String bind3 = c.bind(7); + assertThat(bind3).isEqualTo("A7B7C2D2E"); + } + @Test public void testBindValues() { - StringCompositor c = new StringCompositor("A{ident}C"); - String s = c.bindValues(c, bindings, 0L); + StringBindings c = new StringBindings("A{ident}C", template); + String s = c.apply(0); assertThat(s).isEqualTo("A0C"); } @Test public void testBindValuesSpecialChars() { - StringCompositor c = new StringCompositor("A{mod-5}C"); - String s = c.bindValues(c, bindings, 6L); + StringBindings c = new StringBindings("A{mod-5}C", template); + String s = c.apply(6L); assertThat(s).isEqualTo("A1C"); - c = new StringCompositor("A{5_mod_5}C"); - s = c.bindValues(c, bindings, 7L); + c = new StringBindings("A{5_mod_5}C", template); + s = c.apply(7L); assertThat(s).isEqualTo("A2C"); - -// c = new StringCompositor("A{.mod5}C"); -// s = c.bindValues(c, bindings, 8L); -// assertThat(s).isEqualTo("A3C"); - } - -// @Test -// public void testBindEscapedAnchor() { -// StringCompositor c = new StringCompositor("A\\{{mod-5}C"); -// String s = c.bindValues(c, bindings, 6L); -// assertThat(s).isEqualTo("A{1C"); -// } - - @Test - public void testBindCustomTransform() { - Function f = (o) -> "'" + o.toString() + "'"; - StringCompositor c = new StringCompositor("A{mod5}C", f); - String s = c.bindValues(c, bindings, 13L); - assertThat(s).isEqualTo("A'3'C"); } }