IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

virtdata changes for vector branch

Browse Source
This commit is contained in:
Jonathan Shook
2023-07-25 11:49:56 -05:00
parent 4fdd39fff9
commit d10c78150b
50 changed files with 2491 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
virtdata-api/pom.xml
View File

@@ -82,12 +82,6 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-statistics-distribution</artifactId>
</dependency>
<dependency>
<groupId>com.datastax.oss</groupId>
<artifactId>java-driver-core</artifactId>
<version>4.16.0</version>
<scope>compile</scope>
</dependency>
</dependencies>
</project>

104
virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateMap.java
View File

@@ -84,8 +84,26 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
* when rendering the full map with dynamic values.
*/
private final LinkedHashMap<String, Object> protomap = new LinkedHashMap<>();
/**
* Any auxiliary source of values to be applied beyond what is specified directly in the op fields.
* This includes, for example, the activity parameters which are allowed by the config model on
* an adapter. This means that you can specify defaults for an op field outside of the workload/op
* templates simply by providing them on the command line or activity parameters otherwise.
* This is exactly how the required op field `driver` works.
*/
private final List<Map<String, Object>> cfgsources;
private Map<String, Object> specmap;
/**
* This remembers the original template object so that diagnostic and debugging views
* may see the original specifiers, whether they are literals of any type, or a string
* value which is recognized as being or containing some dynamic span, i.e. bind points.
*/
private Map<String, Object> originalTemplateObject;
/**
* The bindings definitions from the raw op template data structure.
*/
private Map<String, String> bindings;
private final String name;
@@ -100,7 +118,7 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
// fields. This seems like the saner and less confusing approach, so implementing
// op field references should be left until it is requested if at all
private void applyTemplateFields(Map<String, Object> map, Map<String, String> bindings) {
this.specmap = map;
this.originalTemplateObject = map;
this.bindings = bindings;
map.forEach((k, v) -> {
if (v instanceof CharSequence charvalue) {
@@ -701,9 +719,84 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
return false;
}
public Optional<ParsedTemplateString> takeAsOptionalStringTemplate(String field) {
Optional<ParsedTemplateString> asStringTemplate = this.getAsStringTemplate(field);
if (asStringTemplate.isPresent()) {
originalTemplateObject.remove(field);
return asStringTemplate;
}
return Optional.empty();
}
public <V> Optional<V> takeAsOptionalRawSpecifier(String field) {
if (dynamics.containsKey(name)) {
Object value = statics.remove(name);
protomap.remove(name);
return (Optional<V>) Optional.of(value);
}
if (statics.containsKey(name)) {
Object value = statics.remove(name);
protomap.remove(name);
return (Optional<V>) Optional.of(value);
}
return Optional.empty();
}
/**
* Take the value of the specified field from the dynamic or static layers, or reference it
* from the config layer without removal. Then, flatten any string, list, or map structures
* into a map of strings with names injected as needed. Then, convert the values to string
* templates and return that.
* @param fieldname the field to take the templates from
* @return A map of templates, or an empty map if the field is not defined or is empty.
*/
public Map<String,ParsedTemplateString> takeAsNamedTemplates(String fieldname) {
Object entry = originalTemplateObject.get(fieldname);
if (entry !=null) {
dynamics.remove(fieldname);
statics.remove(fieldname);
protomap.remove(fieldname);
}
if (entry==null) {
for (Map<String, Object> cfgsource : cfgsources) {
if (cfgsource.containsKey(fieldname)) {
entry = cfgsource.get(fieldname);
break;
}
}
}
if (entry==null) {
return Map.of();
}
Map<String,Object> elements = new LinkedHashMap<>();
if (entry instanceof CharSequence chars) {
elements.put(this.getName()+"-verifier-0",chars.toString());
} else if (entry instanceof List list) {
for (int i = 0; i < list.size(); i++) {
elements.put(this.getName()+"-verifier-"+i,list.get(0));
}
} else if (entry instanceof Map map) {
map.forEach((k,v) -> {
elements.put(this.getName()+"-verifier-"+k,v);
});
}
Map<String,ParsedTemplateString> parsedStringTemplates
= new LinkedHashMap<>();
elements.forEach((k,v) -> {
if (v instanceof CharSequence chars) {
parsedStringTemplates.put(k,new ParsedTemplateString(chars.toString(), this.bindings));
}
});
return parsedStringTemplates;
}
public Optional<ParsedTemplateString> getAsStringTemplate(String fieldname) {
if (specmap.containsKey(fieldname)) {
Object fval = specmap.get(fieldname);
if (originalTemplateObject.containsKey(fieldname)) {
Object fval = originalTemplateObject.get(fieldname);
if (fval instanceof CharSequence) {
return Optional.of(new ParsedTemplateString(fval.toString(), this.bindings));
} else {
@@ -993,11 +1086,12 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
.append(k)
.append("->")
.append(
v ==null? specmap.get(k) : v.toString()
v ==null? originalTemplateObject.get(k) : v.toString()
).append("\n");
}
return sb.toString();
}
}

2
virtdata-api/src/main/java/io/nosqlbench/virtdata/api/annotations/Category.java
View File

@@ -31,5 +31,7 @@ public enum Category {
objects,
periodic,
experimental,
combinitoric,
vectors,
HOF
}

5
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/BindingsTemplate.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -89,11 +89,12 @@ public class BindingsTemplate {
*
* @param bindPairs A map of named binding specifiers
*/
public void addFieldBindings(Map<String, String> bindPairs) {
public BindingsTemplate addFieldBindings(Map<String, String> bindPairs) {
for (Map.Entry<String, String> e : bindPairs.entrySet()) {
this.bindPointNames.add(e.getKey());
this.specifiers.add(e.getValue());
}
return this;
}
public String getDiagnostics() {

2
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/VirtDataComposer.java
View File

@@ -200,7 +200,7 @@ public class VirtDataComposer {
FunctionAssembly assembly = new FunctionAssembly();
boolean isThreadSafe = true;
diagnostics.trace("FUNCTION chain selected: (multi) '" + this.summarize(flattenedFuncs, " - ") + "'");
diagnostics.trace("FUNCTION chain selected: (multi):\n" + this.summarize(flattenedFuncs, " - "));
for (ResolvedFunction resolvedFunction : flattenedFuncs) {
try {
Object functionObject = resolvedFunction.getFunctionObject();

26
virtdata-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedTemplateMapTest.java
View File

@@ -16,8 +16,10 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import org.junit.jupiter.api.Test;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -32,4 +34,28 @@ public class ParsedTemplateMapTest {
assertThat(ptm.getOpFieldNames()).isEqualTo(Set.of("string1"));
}
@Test
public void testTakeAsNamedTemplates() {
ParsedTemplateMap ptm = new ParsedTemplateMap(
"test2",
new LinkedHashMap<String,Object>(Map.of(
"astring","astring",
"alist",List.of("listentry1","listentry2"),
"amap", Map.of("entry1","val1", "entry2", "val2")
)),
new LinkedHashMap<>(Map.of()),
List.of(Map.of())
);
Map<String, ParsedTemplateString> ofString = ptm.takeAsNamedTemplates("astring");
assertThat(ofString).containsKey("test2-verifier-0");
Map<String, ParsedTemplateString> ofList = ptm.takeAsNamedTemplates("alist");
assertThat(ofList).containsKey("test2-verifier-0");
assertThat(ofList).containsKey("test2-verifier-1");
Map<String, ParsedTemplateString> ofMap = ptm.takeAsNamedTemplates("amap");
assertThat(ofMap).containsKey("test2-verifier-entry1");
assertThat(ofMap).containsKey("test2-verifier-entry2");
// TODO: Get actual testing bindings into this example
}
}

36
virtdata-api/src/test/java/io/nosqlbench/virtdata/testmappers/TestingStringFunc.java Normal file
View File

@@ -0,0 +1,36 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.testmappers;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.function.LongFunction;
@ThreadSafeMapper
public class TestingStringFunc implements LongFunction<String> {
private final String stringValue;
public TestingStringFunc(String stringValue) {
this.stringValue = stringValue;
}
@Override
public String apply(long value) {
return stringValue;
}
}