IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

improvements to generic configuration APIs

Browse Source
This commit is contained in:
Jonathan Shook
2021-01-13 02:28:13 -06:00
parent 0b886931b1
commit ead06c0a25
16 changed files with 631 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmt.java
View File

@@ -18,6 +18,8 @@
package io.nosqlbench.engine.api.activityconfig;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
import io.nosqlbench.nb.api.config.params.NBParams;
import io.nosqlbench.nb.api.config.params.Element;
import io.nosqlbench.virtdata.core.templates.BindPoint;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import org.apache.logging.log4j.Logger;
@@ -152,11 +154,19 @@ public class ParsedStmt {
/**
* @return the params from the enclosed {@link StmtDef}
* @deprecated You should use {@link #getParamReader()} instead.
*/
public Map<String, Object> getParams() {
return stmtDef.getParams();
}
/**
* @return a params reader from the enclosed {@link StmtDef} params map
*/
public Element getParamReader() {
return NBParams.one(stmtDef.getParams());
}
public List<BindPoint> getBindPoints() {
return parsed.getBindPoints();
}

26
nb-api/src/main/java/io/nosqlbench/nb/api/config/ParamsParser.java
View File

@@ -17,10 +17,13 @@
package io.nosqlbench.nb.api.config;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.*;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
@@ -73,11 +76,24 @@ import java.util.*;
*
*/
public class ParamsParser {
public final static String ASSIGN_CHARS = "=:";
private final static Logger logger = LogManager.getLogger(ParamsParser.class);
public static Map<String, String> parse(String input, boolean canonicalize) {
return parse(input, "=:", canonicalize);
return parse(input, ASSIGN_CHARS, canonicalize);
}
public static boolean hasValues(String input) {
return hasValues(input, ASSIGN_CHARS);
}
public static boolean hasValues(String input, String assignChars) {
for (int i = 0; i < assignChars.length(); i++) {
if (input.contains(assignChars.substring(i, i + 1))) {
return true;
}
}
return false;
}
/**
@@ -123,7 +139,7 @@ public class ParamsParser {
case expectingName:
if (c =='\'' || c=='"') {
throw new RuntimeException("Unable to parse a name starting with character '" + c + "'. Names" +
" must be literaal values.");
" must be literal values.");
} else if (c != ' ' && c != ';') {
s = ParseState.readingName;
varname.append(c);

29
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/ConfigSource.java Normal file
View File

@@ -0,0 +1,29 @@
package io.nosqlbench.nb.api.config.params;
import java.util.List;
/**
* A Config Source knows how to read a block of data and convert it
* into a stream of zero or more configuration elements.
*/
public interface ConfigSource {
/**
* Test the input data format to see if it appears valid for reading
* with this config source.
*
* @param source An object of any kind
* @return true if the text is parsable by this config source
*/
boolean canRead(Object source);
/**
* Read the source of data into a collection of config elements
*
* @param source An object of any kind
* @return a collection of {@link Element}s
*/
List<ElementData> getAll(Object source);
// ElementData getOneElementData(Object src);
}

57
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/DataSources.java Normal file
View File

@@ -0,0 +1,57 @@
package io.nosqlbench.nb.api.config.params;
import io.nosqlbench.nb.api.content.NBIO;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.nio.file.Path;
import java.util.List;
public class DataSources {
private final static Logger logger = LogManager.getLogger(DataSources.class);
private static final List<ConfigSource> sources = List.of(
new MapBackedConfigSource(),
new JsonConfigSource(),
new ParamsParserSource(),
new ListBackedConfigSource()
);
public static List<ElementData> elements(Object src) {
if (src instanceof CharSequence && src.toString().startsWith("IMPORT{") && src.toString().endsWith("}")) {
String data = src.toString();
String filename = data.substring("IMPORT{".length(), data.length() - 1);
Path filepath = Path.of(filename);
src = NBIO.all().name(filename).first()
.map(c -> {
logger.debug("found 'data' at " + c.getURI());
return c.asString();
}).orElseThrow();
}
if (src instanceof ElementData) {
return List.of((ElementData) src);
}
for (ConfigSource source : sources) {
if (source.canRead(src)) {
List<ElementData> elements = source.getAll(src);
return elements;
}
}
throw new RuntimeException("Unable to find a config reader for source type " + src.getClass().getCanonicalName());
}
public static ElementData element(Object object) {
List<ElementData> elements = elements(object);
if (elements.size() != 1) {
throw new RuntimeException("Expected exactly one object, but found " + elements.size());
}
return elements.get(0);
}
}

20
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/Element.java Normal file
View File

@@ -0,0 +1,20 @@
package io.nosqlbench.nb.api.config.params;
import java.util.Optional;
/**
* A generic type-safe reader interface for parameters.
* TODO: This should be consolidated with the design of ConfigLoader once the features of these two APIs are stabilized.
*
* The source data for a param reader is intended to be a collection of something, not a single value.
* As such, if a single value is provided, an attempt will be made to convert it from JSON if it starts with
* object or array notation. If not, the value is assumed to be in the simple ParamsParser form.
*/
public interface Element {
String getElementName();
<T> Optional<T> get(String name, Class<? extends T> classOfT);
<T> T getOr(String name, T defaultValue);
}

51
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/ElementData.java Normal file
View File

@@ -0,0 +1,51 @@
package io.nosqlbench.nb.api.config.params;
/**
* A generic type-safe reader interface for parameters.
* TODO: This should be consolidated with the design of ConfigLoader once the features of these two APIs are stabilized.
*
* The source data for a param reader is intended to be a collection of something, not a single value.
* As such, if a single value is provided, an attempt will be made to convert it from JSON if it starts with
* object or array notation. If not, the value is assumed to be in the simple ParamsParser form.
*/
public interface ElementData {
String NAME = "name";
Object get(String name);
boolean containsKey(String name);
// default ElementData getChildElementData(String name) {
// Object o = get(name);
// return DataSources.element(o);
// List<ElementData> datas = DataSources.elements(o);
// if (datas.size() == 0) {
// return null;
// } else if (datas.size() > 1) {
// throw new RuntimeException("expected one element for '" + name + "'");
// } else {
// return datas.get(0);
// }
// }
default String getElementName() {
if (containsKey(NAME)) {
Object o = get(NAME);
if (o != null) {
String converted = convert(o, String.class);
return converted;
}
}
return null;
}
default <T> T convert(Object input, Class<T> type) {
if (type.isAssignableFrom(input.getClass())) {
return type.cast(input);
} else {
throw new RuntimeException("Conversion from " + input.getClass().getSimpleName() + " to " + type.getSimpleName() +
" is not supported natively. You need to add a type converter to your ElementData implementation for " + getClass().getSimpleName());
}
}
}

54
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/ElementImpl.java Normal file
View File

@@ -0,0 +1,54 @@
package io.nosqlbench.nb.api.config.params;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
public class ElementImpl implements Element {
private final ElementData data;
public ElementImpl(ElementData data) {
this.data = data;
}
public String getElementName() {
return get(ElementData.NAME, String.class).orElse(null);
}
public <T> Optional<T> get(String name, Class<? extends T> classOfT) {
List<String> path = Arrays.asList(name.split("\\."));
ElementData top = data;
int idx = 0;
String lookup = path.get(idx);
while (idx + 1 < path.size()) {
if (!top.containsKey(lookup)) {
throw new RuntimeException("unable to find '" + lookup + "' in '" + String.join(".", path));
}
Object o = top.get(lookup);
top = DataSources.element(o);
// top = top.getChildElementData(lookup);
idx++;
lookup = path.get(idx);
}
if (top.containsKey(lookup)) {
Object elem = top.get(lookup);
T convertedValue = top.convert(elem, classOfT);
// T typeCastedValue = classOfT.cast(elem);
return Optional.of(convertedValue);
} else {
return Optional.empty();
}
}
public <T> T getOr(String name, T defaultValue) {
Class<T> cls = (Class<T>) defaultValue.getClass();
return get(name, cls).orElse(defaultValue);
}
}

4
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/IterableNamedParams.java Normal file
View File

@@ -0,0 +1,4 @@
package io.nosqlbench.nb.api.config.params;
public interface IterableNamedParams extends Iterable<Element> {
}

38
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/JsonBackedConfigElement.java Normal file
View File

@@ -0,0 +1,38 @@
package io.nosqlbench.nb.api.config.params;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
public class JsonBackedConfigElement implements ElementData {
private final Gson gson = new GsonBuilder().setPrettyPrinting().create();
private final JsonObject jsonObject;
public JsonBackedConfigElement(JsonObject jsonObject) {
this.jsonObject = jsonObject;
}
@Override
public Object get(String name) {
return jsonObject.get(name);
}
@Override
public boolean containsKey(String name) {
return jsonObject.keySet().contains(name);
}
@Override
public <T> T convert(Object input, Class<T> type) {
if (input instanceof JsonElement) {
T result = gson.fromJson((JsonElement) input, type);
return result;
} else {
throw new RuntimeException("Unable to convert json element from '" + input.getClass().getSimpleName() + "' to '" + type.getSimpleName() + "'");
}
}
}

71
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/JsonConfigSource.java Normal file
View File

@@ -0,0 +1,71 @@
package io.nosqlbench.nb.api.config.params;
import com.google.gson.*;
import java.util.ArrayList;
import java.util.List;
public class JsonConfigSource implements ConfigSource {
private final static Gson gson = new GsonBuilder().setPrettyPrinting().create();
@Override
public boolean canRead(Object data) {
if (data instanceof JsonElement) {
return true;
}
if (data instanceof CharSequence) {
return (data.toString().startsWith("[") || data.toString().startsWith("{"));
}
return false;
}
@Override
public List<ElementData> getAll(Object data) {
JsonElement element = null;
// Pull JSON element from data
if (data instanceof CharSequence) {
JsonParser p = new JsonParser();
element = p.parse(data.toString());
} else if (data instanceof JsonElement) {
element = (JsonElement) data;
}
// Handle element modally by type
List<ElementData> readers = new ArrayList<>();
if (element.isJsonArray()) {
JsonArray ary = element.getAsJsonArray();
for (JsonElement jsonElem : ary) {
if (jsonElem.isJsonObject()) {
readers.add(new JsonBackedConfigElement(jsonElem.getAsJsonObject()));
} else {
throw new RuntimeException("invalid object type for element in sequence: "
+ jsonElem.getClass().getSimpleName());
}
}
} else if (element.isJsonObject()) {
readers.add(new JsonBackedConfigElement(element.getAsJsonObject()));
} else if (element.isJsonPrimitive() && element.getAsJsonPrimitive().isString()) {
String asString = element.getAsJsonPrimitive().getAsString();
ElementData e = DataSources.element(asString);
readers.add(e);
} else {
throw new RuntimeException("Invalid object type for element:" +
element.getClass().getSimpleName());
}
return readers;
}
//
// @Override
// public ElementData getOneElementData(Object src) {
// JsonElement element = (JsonElement) src;
//
//
// }
}

21
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/ListBackedConfigSource.java Normal file
View File

@@ -0,0 +1,21 @@
package io.nosqlbench.nb.api.config.params;
import java.util.ArrayList;
import java.util.List;
public class ListBackedConfigSource implements ConfigSource {
@Override
public boolean canRead(Object source) {
return (source instanceof List);
}
@Override
public List<ElementData> getAll(Object source) {
List<ElementData> data = new ArrayList<>();
for (Object o : (List) source) {
data.add(DataSources.element(o));
}
return data;
}
}

17
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/MapBackedConfigSource.java Normal file
View File

@@ -0,0 +1,17 @@
package io.nosqlbench.nb.api.config.params;
import java.util.List;
import java.util.Map;
public class MapBackedConfigSource implements ConfigSource {
@Override
public boolean canRead(Object source) {
return (source instanceof Map);
}
@Override
public List<ElementData> getAll(Object source) {
return List.of(new MapBackedElement((Map) source));
}
}

22
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/MapBackedElement.java Normal file
View File

@@ -0,0 +1,22 @@
package io.nosqlbench.nb.api.config.params;
import java.util.Map;
public class MapBackedElement implements ElementData {
private final Map map;
public MapBackedElement(Map map) {
this.map = map;
}
@Override
public Object get(String name) {
return map.get(name);
}
@Override
public boolean containsKey(String name) {
return map.containsKey(name);
}
}

112
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/NBParams.java Normal file
View File

@@ -0,0 +1,112 @@
package io.nosqlbench.nb.api.config.params;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* NBParams is the main entry point into accessing parameters in a type-safe way.
* It provides a reader interface which normalizes all access patterns for reading
* configuration parameters from a variety of sources.
*
* NBParams not a general purpose data interface. It is a parameter reading interface.
* As such, all data which is presented for reading must be named at every level.
* This means that index access such as '[3]' that you might see in other access
* vernaculars is <em>NOT</em> supported.
*
* However, multiplicity is allowed at the API level in order to support reading
* zero or more of something when the number of provided elements is intended to
* be user-specified. In this usage, direct indexing is not intended nor allowed,
* but order is preserved. This means that if there are any dependency relationships
* within multiple elements at the same level, the developer can rely on them
* being provided in the order specific by the user or underlying configuration source.
* To be crystal clear, direct indexing within configuration parameters is highly
* discouraged and should not be supported directly by this API.
*
* When configuration elements are named within the element definition, regardless
* of the source, these names can be taken as naming structure. To enable this, simply
* provide a name property on the element.
*
* <H2>element naming</H2>
*
* If an element contains a property named <i>name</i>, then the value of this property
* is taken as the name of the element. This is useful in certain contexts when you
* need to define a name at the source of a configuration element but expose it
* to readers. This means that an element can be positioned with a hierarchic structure
* simply by naming it appropriately.
*
* <H2>Element Views</H2>
*
* The parameter API allows a developer to choose the structural model imposed on
* configuration data. Specifically, you must choose whether or not to consume the
* parameter data as a set of properties of one element instance, or as as set
* of elements, each with their own properties.
*
* <table border="1">
* <tr><td></td><th>view<br>as single</th><th>view<br>as multiple</th></tr>
* <tr><th>source is<br>single element</th><td><i>param name</i></td><td>ERROR</td></tr>
* <tr><th>source is<br>multiple elements</th><td>using <i>element name</i>.<i>param name</td><td>iterable<br>elements</td></tr>
* </table>
*
* <H2>single element view</H2>
*
* The <i>one element access</i> interface is mean to provide basic support for
* parameterizing a single entity. The names of the parameters surfaced at the
* top level should map directly to the names of properties as provided by the
* underlying data source. This is irrespective of whatever other structure may
* be contained within such properties. The key distinction is that the top level
* names of the configuration object are available under the same top-level names
* within the one element interface.
*
* As data sources can provide either one or many style results, it is important
* that each data source provide a clear explanation about how it distinguishes
* reading a single element vs reading (possibly) multiple elements.
*
* When explicitly reading a single element, the underlying data source must provide
* exactly one element <EM>OR</EM> provide a series of elements of which some contain
* <i>name</i> properties. Non-distinct names are allowed, although the last element
* for a given name will be the only one visible to readers. It is an error for the
* underlying data source in this mode to be null, empty, or otherwise provide zero
* elements. When multiple elements are provided, It is also an error if
* none of them has a name property. Otherwise, those with no name property are
* silently ignored and the ones with a name property are exposed.
*
* <H2>element list view</H2>
*
* When accessing <i>some elements</i>, any number of elements may be provided, even zero.
*
* <H2>Naming</H2>
*
* A parameter can be read from a reader by simple name or by a hierarchic name.
* hierarchic names are simply simple names concatenated by a dot '.'.
*/
public class NBParams {
public static List<Element> some(Object source) {
return DataSources.elements(source).stream().map(ElementImpl::new).collect(Collectors.toList());
}
public static Element one(Object source) {
List<ElementData> some = DataSources.elements(source);
if (some.size() == 0) {
throw new RuntimeException("One param object expected, but none found in '" + source + "'");
}
if (some.size() > 1) {
Map<String, ElementData> data = new LinkedHashMap<>();
for (ElementData elementData : some) {
String name = elementData.getElementName();
if (name != null && !name.isBlank()) {
data.put(name, elementData);
}
}
if (data.isEmpty()) {
throw new RuntimeException("multiple elements found, but none contained a name for flattening to a map.");
}
return new ElementImpl(new MapBackedElement(data));
}
return new ElementImpl(some.get(0));
}
}

20
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/ParamsParserSource.java Normal file
View File

@@ -0,0 +1,20 @@
package io.nosqlbench.nb.api.config.params;
import io.nosqlbench.nb.api.config.ParamsParser;
import java.util.List;
import java.util.Map;
public class ParamsParserSource implements ConfigSource {
@Override
public boolean canRead(Object source) {
return (source instanceof CharSequence && ParamsParser.hasValues(source.toString()));
}
@Override
public List<ElementData> getAll(Object source) {
Map<String, String> paramsMap = ParamsParser.parse(source.toString(), false);
return List.of(new MapBackedElement(paramsMap));
}
}

84
nb-api/src/test/java/io/nosqlbench/nb/api/config/params/NBParamsTest.java Normal file
View File

@@ -0,0 +1,84 @@
package io.nosqlbench.nb.api.config.params;
import org.junit.Ignore;
import org.junit.Test;
import java.util.Date;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
public class NBParamsTest {
@Test
public void testMapObject() {
Element one = NBParams.one(Map.of("key1", "value1", "key2", new Date()));
assertThat(one.get("key1", String.class)).isPresent();
assertThat(one.get("key1", String.class).get()).isOfAnyClassIn(String.class);
assertThat(one.get("key1", String.class).get()).isEqualTo("value1");
}
@Test
public void testNestedMapObject() {
Element one = NBParams.one(Map.of("key1", Map.of("key2", "value2")));
assertThat(one.get("key1.key2", String.class).get()).isOfAnyClassIn(String.class);
assertThat(one.get("key1.key2", String.class).get()).isEqualTo("value2");
}
@Test
public void testNestedMixedJsonMapParams() {
Element one = NBParams.one("{\"key1\":{\"key2\":\"key3=value3 key4=value4\"}}");
assertThat(one.get("key1.key2.key3", String.class)).isPresent();
assertThat(one.get("key1.key2.key3", String.class).get()).isEqualTo("value3");
assertThat(one.get("key1.key2.key4", String.class).get()).isEqualTo("value4");
}
@Test
@Ignore("This case is unwieldy and generally not useful")
public void testNestedMixedJsonParamsMap() {
Element one = NBParams.one("{\"key1\":\"key2={\"key3\":\"value3\",\"key4\":\"value4\"}\"}");
assertThat(one.get("key1.key2.key3", String.class)).isPresent();
assertThat(one.get("key1.key2.key3", String.class).get()).isEqualTo("value3");
assertThat(one.get("key1.key2.key4", String.class).get()).isEqualTo("value4");
}
@Test
public void testNestedMixedMapJsonParams() {
Element one = NBParams.one(Map.of("key1", "{ \"key2\": \"key3=value3 key4=value4\"}"));
assertThat(one.get("key1.key2.key3", String.class)).isPresent();
assertThat(one.get("key1.key2.key3", String.class).get()).isEqualTo("value3");
assertThat(one.get("key1.key2.key4", String.class).get()).isEqualTo("value4");
}
@Test
public void testNestedMixedMapParamsJson() {
Element one = NBParams.one(Map.of("key1", "key2: {\"key3\":\"value3\",\"key4\":\"value4\"}"));
assertThat(one.get("key1.key2.key3", String.class)).isPresent();
assertThat(one.get("key1.key2.key3", String.class).get()).isEqualTo("value3");
assertThat(one.get("key1.key2.key4", String.class).get()).isEqualTo("value4");
}
@Test
public void testJsonText() {
Element one = NBParams.one("{\"key1\":\"value1\"}");
assertThat(one.get("key1", String.class)).isPresent();
assertThat(one.get("key1", String.class).get()).isEqualTo("value1");
}
@Test
public void testNamedFromJsonSeq() {
Element one = NBParams.one("[{\"name\":\"first\",\"val\":\"v1\"},{\"name\":\"second\",\"val\":\"v2\"}]");
assertThat(one.get("first.val", String.class)).isPresent();
assertThat(one.get("first.val", String.class).get()).isEqualTo("v1");
}
@Test
public void testNamedFromMapSeq() {
Element one = NBParams.one(List.of(Map.of("name", "first", "val", "v1"), Map.of("name", "second", "val", "v2")));
assertThat(one.get("first.val", String.class)).isPresent();
assertThat(one.get("first.val", String.class).get()).isEqualTo("v1");
}
}