IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-12-24 08:00:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

improve string bindings API and functionality

Browse Source
This commit is contained in:
Jonathan Shook 2022-02-15 21:23:46 -06:00
parent dde2eab49b
commit 35871718d5
11 changed files with 179 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/BindingsTemplate.java
View File

@ -25,10 +25,7 @@ import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.security.InvalidParameterException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.*;
/**
* Maps a set of parameters on an associated object of type T to specifiers for data mappers.
@ -38,14 +35,11 @@ import java.util.Optional;
* bindings will be used in.
*/
public class BindingsTemplate {
private final static Logger logger = LogManager.getLogger(BindingsTemplate.class);
private final Map<String, Object> fconfig;
private List<String> bindPointNames = new ArrayList<>();
private List<String> specifiers = new ArrayList<>();
// public BindingsTemplate(Map<String,String> specs) {
// specs.forEach(this::addFieldBinding);
// }
private final List<String> bindPointNames = new ArrayList<>();
private final List<String> specifiers = new ArrayList<>();
public BindingsTemplate(Map<String,Object> config, List<String> anchors, List<String> specs) {
this.fconfig = config;
@ -61,10 +55,10 @@ public class BindingsTemplate {
this.fconfig = config;
addFieldBindings(bindpoints);
}
public BindingsTemplate(List<BindPoint> bindPoints) {
this.fconfig = Map.of();
addFieldBindings(bindPoints);
}
public BindingsTemplate(Map<String,Object> config) {
@ -114,7 +108,7 @@ public class BindingsTemplate {
diaglog.append(diagnostics);
if (mapperDiagnostics.getResolvedFunction().isPresent()) {
diaglog.append("☑ RESOLVED:")
.append(mapperDiagnostics.getResolvedFunction().get().toString()).append("\n");
.append(mapperDiagnostics.getResolvedFunction().get()).append("\n");
} else {
diaglog.append("☐ UNRESOLVED\n");
}
@ -190,4 +184,11 @@ public class BindingsTemplate {
return sb.toString();
}
public Map<String, String> getMap() {
LinkedHashMap<String, String> bindmap = new LinkedHashMap<>();
for (int i = 0; i < this.specifiers.size(); i++) {
bindmap.put(this.bindPointNames.get(i),this.specifiers.get(i));
}
return bindmap;
}
}

4
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/BindPointParser.java
View File

@ -33,6 +33,7 @@ public class BindPointParser implements BiFunction<String, Map<String, String>,
List<String> spans = new ArrayList<>();
List<BindPoint> bindpoints = new ArrayList<>();
int genid=0;
while (m.find()) {
String pre = template.substring(lastMatch, m.start());
spans.add(pre);
@ -67,6 +68,9 @@ public class BindPointParser implements BiFunction<String, Map<String, String>,
this.bindpoints = bindpoints;
}
/**
* @return the spans of literal values which are between the bind points
*/
public List<String> getSpans() {
return spans;
}

6
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplate.java
View File

@ -130,9 +130,6 @@ public class ParsedTemplate {
*/
private final Map<String, String> bindings = new LinkedHashMap<>();
private final BindPointParser bindPointParser = new BindPointParser();
private final CapturePointParser capturePointParser = new CapturePointParser();
/**
* Parse the given raw template, check the bind points against the provide bindings, and
* provide detailed template checks for validity.
@ -144,8 +141,11 @@ public class ParsedTemplate {
this.bindings.putAll(availableBindings);
this.rawtemplate = rawtemplate;
CapturePointParser capturePointParser = new CapturePointParser();
CapturePointParser.Result captureData = capturePointParser.apply(rawtemplate);
this.captures = captureData.getCaptures();
BindPointParser bindPointParser = new BindPointParser();
BindPointParser.Result bindPointsResult = bindPointParser.apply(captureData.getRawTemplate(), availableBindings);
this.spans = bindPointsResult.getSpans().toArray(new String[0]);
this.bindpoints = bindPointsResult.getBindpoints();

35
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindings.java
View File

@ -1,25 +1,40 @@
package io.nosqlbench.virtdata.core.templates;
import io.nosqlbench.virtdata.core.bindings.Binder;
import io.nosqlbench.virtdata.core.bindings.Bindings;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import java.util.Map;
/**
* Allows the generation of strings from a string template and bindings template.
*/
public class StringBindings implements Binder<String> {
private final StringCompositor compositor;
private final Bindings bindings;
public StringBindings(StringCompositor compositor, Bindings bindings) {
this.compositor = compositor;
this.bindings = bindings;
public StringBindings(String template) {
this(template,Map.of(),Map.of());
}
public StringBindings(ParsedTemplate pt) {
this.compositor = new StringCompositor(pt);
this.bindings = new BindingsTemplate(pt.getBindPoints()).resolveBindings();
public StringBindings(String template, Map<String, String> bindings) {
this(template,bindings,Map.of());
}
public StringBindings(String template, Map<String,String> bindings, Map<String,Object> fconfig) {
ParsedTemplate parsed = new ParsedTemplate(template,bindings);
this.compositor = new StringCompositor(parsed, fconfig);
}
public StringBindings(ParsedTemplate parsedTemplate) {
this(parsedTemplate, Map.of());
}
public StringBindings(ParsedTemplate pt, Map<String,Object> fconfig) {
this.compositor = new StringCompositor(pt,fconfig);
}
public StringBindings(String stringTemplate, BindingsTemplate bindingsTemplate) {
this(stringTemplate,bindingsTemplate.getMap());
}
/**
@ -30,15 +45,13 @@ public class StringBindings implements Binder<String> {
*/
@Override
public String bind(long value) {
String s = compositor.bindValues(compositor, bindings, value);
return s;
return compositor.apply(value);
}
@Override
public String toString() {
return "StringBindings{" +
"compositor=" + compositor +
", bindings=" + bindings +
'}';
}
}

49
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindingsTemplate.java
View File

@ -2,10 +2,8 @@ package io.nosqlbench.virtdata.core.templates;
//import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
//import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.virtdata.core.bindings.Bindings;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import java.util.HashSet;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
/**
* Uses a string template and a bindings template to create instances of {@link StringBindings}.
@ -20,55 +18,12 @@ public class StringBindingsTemplate {
this.bindingsTemplate = bindingsTemplate;
}
// /**
// * Build a default string bindings template using the standard representation
// * for a string template in NoSQLBench, which is a literal string interspersed
// * with named anchors in {@code {{curlybraces}}} form.
// * @param stmtDef A stmtDef
// */
// public StringBindingsTemplate(StmtDef stmtDef) {
// this(stmtDef, s->"{{"+s+"}}");
// }
//
// /**
// * Build a string bindings template using a custom representation that maps
// * the named anchors to a different form than the default {@code {{curlybraces}}} form.
// * The mapping function provides the textual substitution which is used to composite
// * the normative representation of the statement.
// * @param stmtdef The {@link StmtDef} which provides the bindpoints
// * @param tokenMapper A custom named anchor formatting function
// */
// public StringBindingsTemplate(StmtDef stmtdef, Function<String,String> tokenMapper) {
// ParsedStmt parsedStmt = stmtdef.getParsed().orError();
// this.stringTemplate = parsedStmt.getPositionalStatement(tokenMapper);
// this.bindingsTemplate = new BindingsTemplate(parsedStmt.getBindPoints());
// }
/**
* Create a new instance of {@link StringBindings}, preferably in the thread context that will use it.
* @return a new StringBindings
*/
public StringBindings resolve() {
StringCompositor compositor = new StringCompositor(stringTemplate);
HashSet<String> unqualifiedNames = new HashSet<>(compositor.getBindPointNames());
unqualifiedNames.removeAll(new HashSet<>(bindingsTemplate.getBindPointNames()));
if (unqualifiedNames.size()>0) {
throw new RuntimeException("Named anchors were specified in the template which were not provided in the bindings: " + unqualifiedNames);
}
Bindings bindings = bindingsTemplate.resolveBindings();
return new StringBindings(compositor,bindings);
}
public String getDiagnostics() {
StringCompositor compositor = new StringCompositor(stringTemplate);
HashSet<String> unqualifiedNames = new HashSet<>(compositor.getBindPointNames());
unqualifiedNames.removeAll(new HashSet<>(bindingsTemplate.getBindPointNames()));
if (unqualifiedNames.size()>0) {
throw new RuntimeException("Named anchors were specified in the template which were not provided in the bindings: " + unqualifiedNames);
}
return bindingsTemplate.getDiagnostics();
return new StringBindings(stringTemplate,bindingsTemplate);
}
@Override

105
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringCompositor.java
View File

@ -1,74 +1,73 @@
package io.nosqlbench.virtdata.core.templates;
import io.nosqlbench.virtdata.core.bindings.ValuesBinder;
import io.nosqlbench.virtdata.core.bindings.Bindings;
import io.nosqlbench.virtdata.core.bindings.DataMapper;
import io.nosqlbench.virtdata.core.bindings.VirtData;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.function.LongFunction;
/**
* StringCompositor provides a way to build strings from a string template and provided values.
*
* <p>
* The template is simply an array of string values, where odd indices represent token positions, and even indices represent
* literals. This version of the StringCompositor fetches data from the bindings only for the named fields in the template.
* </p>
* This implementation of a string compositor takes a logically coherent
* string template and bindings set. It employs a few simplistic optimizations
* to avoid re-generating duplicate values, as well as lower allocation
* rate of buffer data.
*/
public class StringCompositor implements ValuesBinder<StringCompositor, String> {
public class StringCompositor implements LongFunction<String> {
private final String[] templateSegments;
private Function<Object, String> stringfunc = String::valueOf;
private final String[] spans;
private final DataMapper<?>[] mappers;
private final int[] LUT;
private final int bufsize;
/**
* Create a string template which has positional tokens, in "{}" form.
*
* @param template The string template
*/
public StringCompositor(String template) {
templateSegments = parseTemplate(template);
}
private final Function<Object, String> stringfunc;
public StringCompositor(String template, Function<Object, String> stringfunc) {
this(template);
public StringCompositor(ParsedTemplate template, Map<String,Object> fconfig, Function<Object,String> stringfunc) {
Map<String,Integer> specs = new HashMap<>();
List<BindPoint> bindpoints = template.getBindPoints();
for (BindPoint bindPoint : bindpoints) {
String spec = bindPoint.getBindspec();
specs.compute(spec,(s,i) -> i==null ? specs.size() : i);
}
mappers = new DataMapper<?>[specs.size()];
specs.forEach((k,v) -> {
mappers[v]= VirtData.getOptionalMapper(k,fconfig).orElseThrow();
});
String[] even_odd_spans = template.getSpans();
this.spans = new String[bindpoints.size()+1];
LUT = new int[bindpoints.size()];
for (int i = 0; i < bindpoints.size(); i++) {
spans[i]=even_odd_spans[i<<1];
LUT[i]=specs.get(template.getBindPoints().get(i).getBindspec());
}
spans[spans.length-1]=even_odd_spans[even_odd_spans.length-1];
this.stringfunc = stringfunc;
int minsize = 0;
for (int i = 0; i < 100; i++) {
String result = apply(i);
minsize = Math.max(minsize,result.length());
}
bufsize = minsize*2;
}
public StringCompositor(ParsedTemplate pt) {
templateSegments = pt.getSpans();
public StringCompositor(ParsedTemplate template, Map<String,Object> fconfig) {
this(template,fconfig,Object::toString);
}
// for testing
protected String[] parseTemplate(String template) {
ParsedTemplate parsed = new ParsedTemplate(template, Collections.emptyMap());
return parsed.getSpans();
}
@Override
public String bindValues(StringCompositor context, Bindings bindings, long cycle) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < templateSegments.length; i++) {
if (i % 2 == 0) {
sb.append(templateSegments[i]);
} else {
String key = templateSegments[i];
Object value = bindings.get(key, cycle);
String valueString = stringfunc.apply(value);
sb.append(valueString);
}
public String apply(long value) {
StringBuilder sb = new StringBuilder(bufsize);
String[] ary = new String[mappers.length];
for (int i = 0; i < ary.length; i++) {
ary[i] = stringfunc.apply(mappers[i].apply(value));
}
for (int i = 0; i < LUT.length; i++) {
sb.append(spans[i]).append(ary[LUT[i]]);
}
sb.append(spans[spans.length-1]);
return sb.toString();
}
public List<String> getBindPointNames() {
List<String> tokens = new ArrayList<>();
for (int i = 0; i < templateSegments.length; i++) {
if (i % 2 == 1) {
tokens.add(templateSegments[i]);
}
}
return tokens;
}
}

20
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/FastStringCompositorTest.java Normal file
View File

@ -0,0 +1,20 @@
package io.nosqlbench.virtdata.core.templates;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import java.util.Map;
public class FastStringCompositorTest {
@Test
@Disabled // Needs to have annotation processor run in test scope first
public void testFastStringCompositor() {
String rawTpl = "template {b1}, {{TestValue(5)}}";
Map<String, String> bindings = Map.of("b1", "TestIdentity()");
ParsedTemplate ptpl = new ParsedTemplate(rawTpl, bindings);
StringCompositor fsc = new StringCompositor(ptpl,Map.of());
System.out.println(fsc);
}
}

12
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/StringCompositorTest.java
View File

@ -2,22 +2,22 @@ package io.nosqlbench.virtdata.core.templates;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class StringCompositorTest {
@Test
public void testShouldMatchSpanOnly() {
StringCompositor c = new StringCompositor("A");
String[] spans = c.parseTemplate("A\\{ {one}two");
assertThat(spans).containsExactly("A\\{ ", "one", "two");
ParsedTemplate pt = new ParsedTemplate("A\\{ {one}two", Map.of());
assertThat(pt.getSpans()).containsExactly("A\\{ ", "one", "two");
}
@Test
public void testShouldNotMatchEscaped() {
StringCompositor c = new StringCompositor("A");
String[] spans = c.parseTemplate("A\\{{B}C");
assertThat(spans).containsExactly("A\\{","B","C");
ParsedTemplate pt = new ParsedTemplate("A\\{{B}C",Map.of());
assertThat(pt.getSpans()).containsExactly("A\\{","B","C");
}
// @Test

14
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/TestIdentity.java Normal file
View File

@ -0,0 +1,14 @@
package io.nosqlbench.virtdata.core.templates;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.function.LongFunction;
@ThreadSafeMapper
public class TestIdentity implements LongFunction<Object> {
@Override
public Object apply(long value) {
return value;
}
}

19
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/TestValue.java Normal file
View File

@ -0,0 +1,19 @@
package io.nosqlbench.virtdata.core.templates;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.function.LongFunction;
@ThreadSafeMapper
public class TestValue implements LongFunction<Object> {
private final Object value;
public TestValue(Object value) {
this.value = value;
}
@Override
public Object apply(long value) {
return this.value;
}
}

51
virtdata-userlibs/src/test/java/io/virtdata/IntegratedStringCompositorTest.java → virtdata-userlibs/src/test/java/io/virtdata/IntegratedStringBindingsTest.java
View File

@ -2,15 +2,13 @@ package io.virtdata;
import io.nosqlbench.virtdata.core.bindings.Bindings;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import io.nosqlbench.virtdata.core.templates.StringCompositor;
import io.nosqlbench.virtdata.core.templates.StringBindings;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.util.function.Function;
import static org.assertj.core.api.Assertions.assertThat;
public class IntegratedStringCompositorTest {
public class IntegratedStringBindingsTest {
private static BindingsTemplate template;
private static Bindings bindings;
@ -27,41 +25,36 @@ public class IntegratedStringCompositorTest {
bindings = bindingsTemplate.resolveBindings();
}
@Test
public void testEven() {
StringBindings c = new StringBindings("A{ident}B{ident}C{mod5}D{mod-5}",template);
String bind3 = c.bind(3);
assertThat(bind3).isEqualTo("A3B3C3D3");
}
@Test
public void testOdd() {
StringBindings c = new StringBindings("A{ident}B{ident}C{mod5}D{mod-5}E",template);
String bind3 = c.bind(7);
assertThat(bind3).isEqualTo("A7B7C2D2E");
}
@Test
public void testBindValues() {
StringCompositor c = new StringCompositor("A{ident}C");
String s = c.bindValues(c, bindings, 0L);
StringBindings c = new StringBindings("A{ident}C", template);
String s = c.apply(0);
assertThat(s).isEqualTo("A0C");
}
@Test
public void testBindValuesSpecialChars() {
StringCompositor c = new StringCompositor("A{mod-5}C");
String s = c.bindValues(c, bindings, 6L);
StringBindings c = new StringBindings("A{mod-5}C", template);
String s = c.apply(6L);
assertThat(s).isEqualTo("A1C");
c = new StringCompositor("A{5_mod_5}C");
s = c.bindValues(c, bindings, 7L);
c = new StringBindings("A{5_mod_5}C", template);
s = c.apply(7L);
assertThat(s).isEqualTo("A2C");
// c = new StringCompositor("A{.mod5}C");
// s = c.bindValues(c, bindings, 8L);
// assertThat(s).isEqualTo("A3C");
}
// @Test
// public void testBindEscapedAnchor() {
// StringCompositor c = new StringCompositor("A\\{{mod-5}C");
// String s = c.bindValues(c, bindings, 6L);
// assertThat(s).isEqualTo("A{1C");
// }
@Test
public void testBindCustomTransform() {
Function<Object,String> f = (o) -> "'" + o.toString() + "'";
StringCompositor c = new StringCompositor("A{mod5}C", f);
String s = c.bindValues(c, bindings, 13L);
assertThat(s).isEqualTo("A'3'C");
}
}