IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Config API updates

Browse Source
This commit is contained in:
Jonathan Shook 2021-07-20 18:26:38 -05:00
parent 34d22c9c01
commit ef8ecdffc2
33 changed files with 764 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmtOp.java → drivers-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmtOp.java
View File

@ -120,7 +120,7 @@ public class ParsedStmtOp {
* @return a params reader from the enclosed {@link OpTemplate} params map
*/
public Element getParamReader() {
return NBParams.one(optpl.getParams());
return NBParams.one(getName(),optpl.getParams());
}
public List<BindPoint> getBindPoints() {

12
engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/NBErrorHandler.java
View File

@ -2,7 +2,7 @@ package io.nosqlbench.engine.api.activityapi.errorhandling.modular;
import io.nosqlbench.engine.api.activityapi.errorhandling.ErrorMetrics;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.config.ConfigAware;
import io.nosqlbench.nb.api.config.standard.NBMapConfigurable;
import io.nosqlbench.nb.api.config.params.Element;
import io.nosqlbench.nb.api.config.params.NBParams;
@ -81,8 +81,8 @@ public class NBErrorHandler {
throw new RuntimeException("ErrorHandler named '" + name + "' could not be found in " + providers.keySet());
}
ErrorHandler handler = provider.get();
if (handler instanceof ConfigAware) {
((ConfigAware) handler).applyConfig(cfg.getMap());
if (handler instanceof NBMapConfigurable) {
((NBMapConfigurable) handler).applyConfig(cfg.getMap());
}
if (handler instanceof ErrorMetrics.Aware) {
((ErrorMetrics.Aware) handler).setErrorMetricsSupplier(errorMetricsSupplier);
@ -141,11 +141,11 @@ public class NBErrorHandler {
ghandlers = leadmatch.group("rest") == null ? "" : leadmatch.group("rest");
Element next = null;
if (word.matches("\\d+")) {
next = NBParams.one("handler=code code=" + word);
next = NBParams.one(null,"handler=code code=" + word);
} else if (word.matches("[a-zA-Z]+")) {
next = NBParams.one("handler=" + word);
next = NBParams.one(null,"handler=" + word);
} else {
next = NBParams.one(word);
next = NBParams.one(null,word);
}
params.add(next);
} else {

12
engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/ResultCode.java
View File

@ -1,14 +1,14 @@
package io.nosqlbench.engine.api.activityapi.errorhandling.modular;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.config.ConfigAware;
import io.nosqlbench.nb.api.config.ConfigModel;
import io.nosqlbench.nb.api.config.MutableConfigModel;
import io.nosqlbench.nb.api.config.standard.NBMapConfigurable;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import java.util.Map;
@Service(value = ErrorHandler.class, selector = "code")
public class ResultCode implements ErrorHandler, ConfigAware {
public class ResultCode implements ErrorHandler, NBMapConfigurable {
private byte code;
@ -23,8 +23,8 @@ public class ResultCode implements ErrorHandler, ConfigAware {
}
@Override
public ConfigModel getConfigModel() {
return new MutableConfigModel(this)
public NBConfigModel getConfigModel() {
return ConfigModel.of(this.getClass())
.required("code", Byte.class)
.asReadOnly();
}

20
engine-core/src/main/java/io/nosqlbench/engine/core/metrics/LoggingAnnotator.java
View File

@ -3,10 +3,10 @@ package io.nosqlbench.engine.core.metrics;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.annotations.Annotation;
import io.nosqlbench.nb.api.annotations.Annotator;
import io.nosqlbench.nb.api.config.ConfigAware;
import io.nosqlbench.nb.api.config.ConfigModel;
import io.nosqlbench.nb.api.config.ConfigReader;
import io.nosqlbench.nb.api.config.MutableConfigModel;
import io.nosqlbench.nb.api.config.standard.NBMapConfigurable;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -15,7 +15,7 @@ import java.util.LinkedHashMap;
import java.util.Map;
@Service(value = Annotator.class, selector = "log")
public class LoggingAnnotator implements Annotator, ConfigAware {
public class LoggingAnnotator implements Annotator, NBMapConfigurable {
private final static Logger annotatorLog = LogManager.getLogger("ANNOTATION");
private Level level;
@ -33,16 +33,16 @@ public class LoggingAnnotator implements Annotator, ConfigAware {
@Override
public void applyConfig(Map<String, ?> providedConfig) {
ConfigModel configModel = getConfigModel();
ConfigReader cfg = configModel.apply(providedConfig);
NBConfigModel configModel = getConfigModel();
NBConfiguration cfg = configModel.apply(providedConfig);
String levelName = cfg.param("level", String.class);
this.level = Level.valueOf(levelName);
}
@Override
public ConfigModel getConfigModel() {
return new MutableConfigModel(this)
.defaultto("level", "INFO",
public NBConfigModel getConfigModel() {
return ConfigModel.of(this.getClass())
.defaults("level", "INFO",
"The logging level to use for this annotator")
.asReadOnly();
}

59
nb-api/src/main/java/io/nosqlbench/nb/api/config/ConfigElement.java
View File

@ -1,59 +0,0 @@
package io.nosqlbench.nb.api.config;
public class ConfigElement<T> {
public final String name;
public final Class<? extends T> type;
public final String description;
private final T defaultValue;
public boolean required;
public ConfigElement(
String name,
Class<? extends T> type,
String description,
boolean required,
T defaultValue
) {
this.name = name;
this.type = type;
this.description = description;
this.required = required;
this.defaultValue = defaultValue;
}
@Override
public String toString() {
return "Element{" +
"name='" + name + '\'' +
", type=" + type +
", description='" + description + '\'' +
", required=" + required +
", defaultValue = " + defaultValue +
'}';
}
public String getName() {
return name;
}
public Class<?> getType() {
return type;
}
public String getDescription() {
return description;
}
public boolean isRequired() {
return required;
}
public void setRequired(boolean required) {
this.required = required;
}
public T getDefaultValue() {
return defaultValue;
}
}

13
nb-api/src/main/java/io/nosqlbench/nb/api/config/ConfigModel.java
View File

@ -1,13 +0,0 @@
package io.nosqlbench.nb.api.config;
import java.util.Map;
public interface ConfigModel {
Map<String, ConfigElement> getElements();
Class<?> getOf();
void assertValidConfig(Map<String, ?> config);
ConfigReader apply(Map<String, ?> config);
}

48
nb-api/src/main/java/io/nosqlbench/nb/api/config/ConfigReader.java
View File

@ -1,48 +0,0 @@
package io.nosqlbench.nb.api.config;
import io.nosqlbench.nb.api.NBEnvironment;
import java.util.LinkedHashMap;
import java.util.Optional;
public class ConfigReader extends LinkedHashMap<String, Object> {
private final ConfigModel configModel;
public ConfigReader(ConfigModel model, LinkedHashMap<String, Object> validConfig) {
super(validConfig);
this.configModel = model;
}
public <T> T paramEnv(String name, Class<? extends T> vclass) {
T param = param(name, vclass);
if (param instanceof String) {
Optional<String> interpolated = NBEnvironment.INSTANCE.interpolate(param.toString());
if (interpolated.isEmpty()) {
throw new RuntimeException("Unable to interpolate env and sys props in '" + param + "'");
}
return (T) interpolated.get();
} else {
return param;
}
}
public <T> T param(String name, Class<? extends T> vclass) {
Object o = get(name);
ConfigElement<?> elem = configModel.getElements().get(name);
if (elem == null) {
throw new RuntimeException("Invalid config element named '" + name + "'" );
}
Class<T> type = (Class<T>) elem.getType();
T typeCastedValue = type.cast(o);
return typeCastedValue;
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(this.configModel.getOf().getSimpleName()).append(":" );
sb.append(this.configModel.toString());
return sb.toString();
}
}

9
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/ConfigSource.java
View File

@ -23,7 +23,12 @@ public interface ConfigSource {
* @param source An object of any kind
* @return a collection of {@link Element}s
*/
List<ElementData> getAll(Object source);
List<ElementData> getAll(String injectedName, Object source);
// ElementData getOneElementData(Object src);
/**
* If an element was created with a name, this name must be returned as the
* canonical name. If it was not, then the name field can provide the name.
* @return A name, or null if it is not given nor present in the name field
*/
String getName();
}

11
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/DataSources.java
View File

@ -19,6 +19,10 @@ public class DataSources {
);
public static List<ElementData> elements(Object src) {
return elements(null, src);
}
public static List<ElementData> elements(String name, Object src) {
if (src instanceof CharSequence && src.toString().startsWith("IMPORT{") && src.toString().endsWith("}")) {
String data = src.toString();
@ -38,7 +42,7 @@ public class DataSources {
for (ConfigSource source : sources) {
if (source.canRead(src)) {
List<ElementData> elements = source.getAll(src);
List<ElementData> elements = source.getAll(name, src);
return elements;
}
}
@ -48,7 +52,10 @@ public class DataSources {
}
public static ElementData element(Object object) {
List<ElementData> elements = elements(object);
return element(null, object);
}
public static ElementData element(String name, Object object) {
List<ElementData> elements = elements(name, object);
if (elements.size() != 1) {
throw new RuntimeException("Expected exactly one object, but found " + elements.size());
}

91
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/Element.java
View File

@ -5,18 +5,99 @@ import java.util.Optional;
/**
* A generic type-safe reader interface for parameters.
* TODO: This should be consolidated with the design of ConfigLoader once the features of these two APIs are stabilized.
*
* The source data for a param reader is intended to be a collection of something, not a single value.
* As such, if a single value is provided, an attempt will be made to convert it from JSON if it starts with
* object or array notation. If not, the value is assumed to be in the simple ParamsParser form.
*/
public interface Element {
String getElementName();
/**
* <p>Hierarchic get of a named variable. If the name is a simple word with
* no dots, such as param2, then this is a simple lookup. If it is a
* hierarchic name such as car.cabin.dashlight1, then multiple
* representations of the structure could suffice to serve the request.
* This is enabled with name flattening.</p>
*
* <p>Name flattening - It is possible that multiple representations
* of backing data can suffice to hold the same logically named element.
* For example the three JSON objects below are all semantically equivalent.
*
* <pre>{@code
* [
* {
* "car": {
* "cabin": {
* "dashlight1": "enabled"
* }
* }
* },
* {
* "car": {
* "cabin.dashlight1": "enabled"
* }
* },
* {
* "car.cabin": {
* "dashlight1": "enabled"
* }
* },
* {
* "car.cabin.dashlight": "enabled"
* }
* ]
* }</pre>
*
* <p>It is necessary to honor all of these representations due to the various ways that
* users may provide the constructions of their configuration data. Some of them will
* be long-form property maps from files, others will be programmatic data structures
* passed into an API. This means that we must also establish a clear order of precedence
* between them.</p>
*
* <p><em>The non-collapsed form takes precedence, starting from the root level.</em> That is,
* if there are multiple backing data structures for the same name, the one with a flattened name
* will <em>NOT</em> be seen if there is another at the same level which is not flattened --
* even if the leaf node is fully defined under the flattened name.</p>
*
* <p>Thus the examples above
* are actually in precedence order. The first JSON object has the most complete name
* defined at the root level, so it is what will be found first. All implementations
* should ensure that this order is preserved.</p>
*
* @param name The simple or hierarchic variable name to resolve
* @param classOfT The type of value which the resolved value is required to be assignable to
* @param <T> The value type parameter
* @return An optional value of type T
*/
<T> Optional<T> get(String name, Class<? extends T> classOfT);
/**
* Perform the same lookup as {@link #get(String, Class)}, except allow for full type
* inferencing when possible. The value asked for will be cast to the type T at runtime,
* as with type erasure there is no simple way to capture the requested type without
* reifying it into a runtime instance in the caller. Thus, this method is provided
* as a syntactic convenience at best. It will almost always be better to use
* {@link #get(String, Class)}
*
* @param name The simple or hierarchic variable name to resolve
* @param <T> The value type parameter
* @return An optional value of type T
*/
<T> Optional<T> get(String name);
/**
* Perform the same lookup as {@link #get(String, Class)}, but return the default value
* when a value isn't found.
*
* @param name The simple or hierarchic variable name to resolve
* @param defaultValue The default value to return if the named variable is not found
* @param <T> The value type parameter
* @return Either the found value or the default value provided
*/
<T> T getOr(String name, T defaultValue);
/**
* Return the backing data for this element in map form.
*
* @return A Map of data.
*/
Map<String, Object> getMap();
}

82
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/ElementData.java
View File

@ -19,37 +19,79 @@ public interface ElementData {
boolean containsKey(String name);
// default ElementData getChildElementData(String name) {
// Object o = get(name);
// return DataSources.element(o);
// List<ElementData> datas = DataSources.elements(o);
// if (datas.size() == 0) {
// return null;
// } else if (datas.size() > 1) {
// throw new RuntimeException("expected one element for '" + name + "'");
// } else {
// return datas.get(0);
// }
// }
default String getName() {
String name = getGivenName();
if (name!=null) {
return name;
}
return extractElementName();
}
default String getElementName() {
String getGivenName();
default String extractElementName() {
if (containsKey(NAME)) {
Object o = get(NAME);
if (o != null) {
String converted = convert(o, String.class);
return converted;
if (o instanceof CharSequence) {
return ((CharSequence)o).toString();
}
}
return null;
}
default <T> T convert(Object input, Class<T> type) {
if (type.isAssignableFrom(input.getClass())) {
return type.cast(input);
if (type!=null) {
if (type.isAssignableFrom(input.getClass())) {
return type.cast(input);
} else {
throw new RuntimeException("Conversion from " + input.getClass().getSimpleName() + " to " + type.getSimpleName() +
" is not supported natively. You need to add a type converter to your ElementData implementation for " + getClass().getSimpleName());
}
} else {
throw new RuntimeException("Conversion from " + input.getClass().getSimpleName() + " to " + type.getSimpleName() +
" is not supported natively. You need to add a type converter to your ElementData implementation for " + getClass().getSimpleName());
return (T) input;
}
}
default <T> T get(String name, Class<T> type) {
Object o = get(name);
if (o!=null) {
return convert(o,type);
} else {
return null;
}
}
default <T> T lookup(String name, Class<T> type) {
int idx=name.indexOf('.');
while (idx>0) { // TODO: What about when idx==0 ?
// Needs to iterate through all terms
String parentName = name.substring(0,idx);
if (containsKey(parentName)) {
Object o = get(parentName);
ElementData parentElement = DataSources.element(parentName, o);
String childName = name.substring(idx+1);
int childidx = childName.indexOf('.');
while (childidx>0) {
String branchName = childName.substring(0,childidx);
Object branchObject = parentElement.lookup(branchName,type);
if (branchObject!=null) {
ElementData branch = DataSources.element(branchName, branchObject);
String leaf=childName.substring(childidx+1);
T found = branch.lookup(leaf, type);
if (found!=null) {
return found;
}
}
childidx=childName.indexOf('.',childidx+1);
}
T found = parentElement.lookup(childName,type);
if (found!=null) {
return found;
}
}
idx=name.indexOf('.',idx+1);
}
return get(name,type);
}
}

67
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/ElementImpl.java
View File

@ -2,6 +2,11 @@ package io.nosqlbench.nb.api.config.params;
import java.util.*;
/**
* The source data for a param reader is intended to be a collection of something, not a single value.
* As such, if a single value is provided, an attempt will be made to convert it from JSON if it starts with
* object or array notation. If not, the value is assumed to be in the simple ParamsParser form.
*/
public class ElementImpl implements Element {
private final ElementData data;
@ -11,38 +16,43 @@ public class ElementImpl implements Element {
}
public String getElementName() {
String name = data.getGivenName();
if (name!=null) {
return name;
}
return get(ElementData.NAME, String.class).orElse(null);
}
public <T> Optional<T> get(String name, Class<? extends T> classOfT) {
List<String> path = Arrays.asList(name.split("\\."));
ElementData top = data;
int idx = 0;
String lookup = path.get(idx);
while (idx + 1 < path.size()) {
if (!top.containsKey(lookup)) {
throw new RuntimeException("unable to find '" + lookup + "' in '" + String.join(".", path));
}
Object o = top.get(lookup);
top = DataSources.element(o);
// top = top.getChildElementData(lookup);
idx++;
lookup = path.get(idx);
}
if (top.containsKey(lookup)) {
Object elem = top.get(lookup);
T convertedValue = top.convert(elem, classOfT);
// T typeCastedValue = classOfT.cast(elem);
return Optional.of(convertedValue);
} else {
return Optional.empty();
}
T found = lookup(data,name, classOfT);
return Optional.ofNullable(found);
}
@Override
public <T> Optional<T> get(String name) {
return Optional.ofNullable(data.lookup(name,null));
}
private <T> T lookup(ElementData data, String name, Class<T> type) {
return data.lookup(name,type);
// int idx=name.indexOf('.');
// while (idx>0) { // TODO: What about when idx==0 ?
// String parentName = name.substring(0,idx);
// if (data.containsKey(parentName)) {
// Object o = data.get(parentName);
// ElementData parentElement = DataSources.element(o);
// String childName = name.substring(idx+1);
// T found = parentElement.lookup(name,type);
// if (found!=null) {
// return found;
// }
// }
// idx=name.indexOf('.',idx+1);
// }
// return data.get(name,type);
}
public <T> T getOr(String name, T defaultValue) {
Class<T> cls = (Class<T>) defaultValue.getClass();
return get(name, cls).orElse(defaultValue);
@ -61,5 +71,8 @@ public class ElementImpl implements Element {
return map;
}
@Override
public String toString() {
return data.toString();
}
}

28
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/JsonBackedConfigElement.java
View File

@ -1,19 +1,18 @@
package io.nosqlbench.nb.api.config.params;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.*;
import java.util.Set;
public class JsonBackedConfigElement implements ElementData {
private final Gson gson = new GsonBuilder().setPrettyPrinting().create();
private final static Gson gson = new GsonBuilder().setPrettyPrinting().create();
private final JsonObject jsonObject;
private final String name;
public JsonBackedConfigElement(JsonObject jsonObject) {
public JsonBackedConfigElement(String injectedName, JsonObject jsonObject) {
this.name = injectedName;
this.jsonObject = jsonObject;
}
@ -32,6 +31,11 @@ public class JsonBackedConfigElement implements ElementData {
return jsonObject.keySet().contains(name);
}
@Override
public String getGivenName() {
return this.name;
}
@Override
public <T> T convert(Object input, Class<T> type) {
if (input instanceof JsonElement) {
@ -42,4 +46,16 @@ public class JsonBackedConfigElement implements ElementData {
}
}
@Override
public String toString() {
return getGivenName() + "(" + (extractElementName()!=null ? extractElementName() : "null" ) +"):" + jsonObject.toString();
}
@Override
public String extractElementName() {
if (jsonObject.has("name")) {
return jsonObject.get("name").getAsString();
}
return null;
}
}

20
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/JsonConfigSource.java
View File

@ -7,6 +7,7 @@ import java.util.List;
public class JsonConfigSource implements ConfigSource {
private final static Gson gson = new GsonBuilder().setPrettyPrinting().create();
private String name;
@Override
public boolean canRead(Object data) {
@ -20,7 +21,8 @@ public class JsonConfigSource implements ConfigSource {
}
@Override
public List<ElementData> getAll(Object data) {
public List<ElementData> getAll(String name, Object data) {
this.name = name;
JsonElement element = null;
@ -33,32 +35,36 @@ public class JsonConfigSource implements ConfigSource {
}
// Handle element modally by type
List<ElementData> readers = new ArrayList<>();
List<ElementData> elements = new ArrayList<>();
if (element.isJsonArray()) {
JsonArray ary = element.getAsJsonArray();
for (JsonElement jsonElem : ary) {
if (jsonElem.isJsonObject()) {
readers.add(new JsonBackedConfigElement(jsonElem.getAsJsonObject()));
elements.add(new JsonBackedConfigElement(null, jsonElem.getAsJsonObject()));
} else {
throw new RuntimeException("invalid object type for element in sequence: "
+ jsonElem.getClass().getSimpleName());
}
}
} else if (element.isJsonObject()) {
readers.add(new JsonBackedConfigElement(element.getAsJsonObject()));
elements.add(new JsonBackedConfigElement(null,element.getAsJsonObject()));
} else if (element.isJsonPrimitive() && element.getAsJsonPrimitive().isString()) {
String asString = element.getAsJsonPrimitive().getAsString();
ElementData e = DataSources.element(asString);
readers.add(e);
ElementData e = DataSources.element(name,asString);
elements.add(e);
} else {
throw new RuntimeException("Invalid object type for element:" +
element.getClass().getSimpleName());
}
return elements;
}
return readers;
@Override
public String getName() {
return this.name;
}
//
// @Override

10
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/ListBackedConfigSource.java
View File

@ -5,17 +5,25 @@ import java.util.List;
public class ListBackedConfigSource implements ConfigSource {
private String name;
@Override
public boolean canRead(Object source) {
return (source instanceof List);
}
@Override
public List<ElementData> getAll(Object source) {
public List<ElementData> getAll(String name, Object source) {
this.name = name;
List<ElementData> data = new ArrayList<>();
for (Object o : (List) source) {
data.add(DataSources.element(o));
}
return data;
}
@Override
public String getName() {
return name;
}
}

12
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/MapBackedConfigSource.java
View File

@ -5,13 +5,21 @@ import java.util.Map;
public class MapBackedConfigSource implements ConfigSource {
private String name;
@Override
public boolean canRead(Object source) {
return (source instanceof Map);
}
@Override
public List<ElementData> getAll(Object source) {
return List.of(new MapBackedElement((Map) source));
public List<ElementData> getAll(String name, Object source) {
this.name = name;
return List.of(new MapBackedElement(name, (Map) source));
}
@Override
public String getName() {
return name;
}
}

16
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/MapBackedElement.java
View File

@ -5,9 +5,11 @@ import java.util.Set;
public class MapBackedElement implements ElementData {
private final Map map;
private final Map<String, ?> map;
private final String elementName;
public MapBackedElement(Map map) {
public MapBackedElement(String elementName, Map<String, ?> map) {
this.elementName = elementName;
this.map = map;
}
@ -25,4 +27,14 @@ public class MapBackedElement implements ElementData {
public boolean containsKey(String name) {
return map.containsKey(name);
}
@Override
public String getGivenName() {
return this.elementName;
}
@Override
public String toString() {
return this.getGivenName() + "(" + (this.extractElementName() != null ? this.extractElementName() : "null") + "):" + map.toString();
}
}

9
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/NBParams.java
View File

@ -96,14 +96,17 @@ public class NBParams {
}
public static Element one(Object source) {
List<ElementData> some = DataSources.elements(source);
return one(null, source);
}
public static Element one(String givenName, Object source) {
List<ElementData> some = DataSources.elements(givenName,source);
if (some.size() == 0) {
throw new RuntimeException("One param object expected, but none found in '" + source + "'");
}
if (some.size() > 1) {
Map<String, ElementData> data = new LinkedHashMap<>();
for (ElementData elementData : some) {
String name = elementData.getElementName();
String name = elementData.getName();
if (name != null && !name.isBlank()) {
data.put(name, elementData);
}
@ -111,7 +114,7 @@ public class NBParams {
if (data.isEmpty()) {
throw new RuntimeException("multiple elements found, but none contained a name for flattening to a map.");
}
return new ElementImpl(new MapBackedElement(data));
return new ElementImpl(new MapBackedElement(givenName,data));
}
return new ElementImpl(some.get(0));
}

14
nb-api/src/main/java/io/nosqlbench/nb/api/config/params/ParamsParserSource.java
View File

@ -1,20 +1,26 @@
package io.nosqlbench.nb.api.config.params;
import io.nosqlbench.nb.api.config.ParamsParser;
import java.util.List;
import java.util.Map;
public class ParamsParserSource implements ConfigSource {
private String name;
@Override
public boolean canRead(Object source) {
return (source instanceof CharSequence && ParamsParser.hasValues(source.toString()));
}
@Override
public List<ElementData> getAll(Object source) {
public List<ElementData> getAll(String name,Object source) {
this.name = name;
Map<String, String> paramsMap = ParamsParser.parse(source.toString(), false);
return List.of(new MapBackedElement(paramsMap));
return List.of(new MapBackedElement(name, paramsMap));
}
@Override
public String getName() {
return name;
}
}

33
nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigSuggestions.java Normal file
View File

@ -0,0 +1,33 @@
package io.nosqlbench.nb.api.config.standard;
import org.apache.commons.text.similarity.LevenshteinDistance;
import java.util.*;
import java.util.stream.Collectors;
public class ConfigSuggestions {
public static Optional<String> getForParam(ConfigModel model, String param) {
Map<Integer, Set<String>> suggestions = new HashMap<>();
for (String candidate : model.getElements().keySet()) {
try {
Integer distance = LevenshteinDistance.getDefaultInstance().apply(param, candidate);
Set<String> strings = suggestions.computeIfAbsent(distance, d -> new HashSet<>());
strings.add(candidate);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
ArrayList<Integer> params = new ArrayList<>(suggestions.keySet());
Collections.sort(params);
List<Set<String>> orderedSets = params.stream().map(suggestions::get).collect(Collectors.toList());
if (orderedSets.size()==0) {
return Optional.empty();
} else if (orderedSets.get(0).size()==1) {
return Optional.of("Did you mean '" + orderedSets.get(0).stream().findFirst().get() +"'?");
} else {
return Optional.of("Did you mean one of " + orderedSets.get(0).toString() + "?\n");
}
}
}

5
nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBCanConfigure.java Normal file
View File

@ -0,0 +1,5 @@
package io.nosqlbench.nb.api.config.standard;
public interface NBCanConfigure {
void applyConfig(NBConfiguration cfg);
}

5
nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBCanValidateConfig.java Normal file
View File

@ -0,0 +1,5 @@
package io.nosqlbench.nb.api.config.standard;
public interface NBCanValidateConfig {
NBConfigModel getConfigModel();
}

18
nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBConfigModel.java Normal file
View File

@ -0,0 +1,18 @@
package io.nosqlbench.nb.api.config.standard;
import java.util.Map;
/**
* This configuration model describes what is valid to submit
* for configuration for a given configurable object.
*/
public interface NBConfigModel {
Map<String, Param<?>> getElements();
Class<?> getOf();
void assertValidConfig(Map<String, ?> config);
NBConfiguration apply(Map<String, ?> config);
}

10
nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBConfigReadable.java Normal file
View File

@ -0,0 +1,10 @@
package io.nosqlbench.nb.api.config.standard;
import java.util.Optional;
public interface NBConfigReadable {
<T> T getOrDefault(String name, T defaultValue);
<T> Optional<T> getOptional(String name, Class<? extends T> type);
<T> Optional<T> getOptional(String name);
}

14
nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBConfigurable.java Normal file
View File

@ -0,0 +1,14 @@
package io.nosqlbench.nb.api.config.standard;
/**
* All implementation types which wish to have a type-marshalled configuration
* should implement this interface.
*/
public interface NBConfigurable extends NBCanConfigure, NBCanValidateConfig {
@Override
void applyConfig(NBConfiguration cfg);
@Override
NBConfigModel getConfigModel();
}

142
nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBConfiguration.java Normal file
View File

@ -0,0 +1,142 @@
package io.nosqlbench.nb.api.config.standard;
import io.nosqlbench.nb.api.NBEnvironment;
import io.nosqlbench.nb.api.errors.BasicError;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
public class NBConfiguration {
private final LinkedHashMap<String, Object> data;
private final NBConfigModel configModel;
/**
* Create a NBConfigReader from a known valid configuration and a config model.
* This method is restricted to encourage construction of readers only by passing
* through the friendly {@link NBConfigModel#apply(Map)} method.
*
* @param model A configuration model, describing what is allowed to be configured by name and type.
* @param validConfig A valid config reader.
*/
protected NBConfiguration(NBConfigModel model, LinkedHashMap<String, Object> validConfig) {
this.data = validConfig;
this.configModel = model;
}
/**
* Returns the value of the named parameter as {@link #getOptional(String)}, so long
* as no env vars were reference OR all env var references were found.
* @param name The name of the variable to look up
* @return An optional value, if present and (optionally) interpolated correctly from the environment
*/
public Optional<String> getEnvOptional(String name) {
Optional<String> optionalValue = getOptional(name);
if (optionalValue.isEmpty()) {
return Optional.empty();
}
String span = optionalValue.get();
Optional<String> maybeInterpolated = NBEnvironment.INSTANCE.interpolate(span);
if (maybeInterpolated.isEmpty()) {
throw new BasicError("Unable to interpolate '" + span +"' with env vars.");
}
return maybeInterpolated;
}
public String paramEnv(String name) {
return paramEnv(name, String.class);
}
public <T> T paramEnv(String name, Class<? extends T> vclass) {
T param = param(name, vclass);
if (param instanceof String) {
Optional<String> interpolated = NBEnvironment.INSTANCE.interpolate(param.toString());
if (interpolated.isEmpty()) {
throw new RuntimeException("Unable to interpolate env and sys props in '" + param + "'");
}
return (T) interpolated.get();
} else {
return param;
}
}
public <T> T get(String name, Class<? extends T> type) {
if (!configModel.getElements().containsKey(name)) {
throw new BasicError("Parameter named '" + name + "' is not valid for " + configModel.getOf().getSimpleName() + ".");
}
Object o = data.get(name);
if (o == null) {
throw new BasicError("config param '" + name + "' was not defined.");
}
if (type.isAssignableFrom(o.getClass())) {
return (T) o;
}
throw new BasicError("config param '" + name + "' was not assignable to class '" + type.getCanonicalName() + "'");
}
public Optional<String> getOptional(String name) {
return getOptional(new String[]{name});
}
public Optional<String> getOptional(String... names) {
return getOptional(String.class, names);
}
public <T> Optional<T> getOptional(Class<T> type, String... names) {
Object o = null;
for (String name : names) {
o = data.get(name);
if (o!=null) {
break;
}
}
if (o==null) {
return Optional.empty();
}
if (type.isAssignableFrom(o.getClass())) {
return Optional.of((T) o);
}
throw new BasicError("config param " + Arrays.toString(names) +" was not assignable to class '" + type.getCanonicalName() + "'");
}
public <T> T getOrDefault(String name, T defaultValue) {
Object o = data.get(name);
if (o == null) {
return defaultValue;
}
if (defaultValue.getClass().isAssignableFrom(o.getClass())) {
return (T) o;
}
throw new BasicError("config parameter '" + name + "' is not assignable to required type '" + defaultValue.getClass() + "'");
}
public <T> T param(String name, Class<? extends T> vclass) {
Object o = data.get(name);
Param<?> elem = configModel.getElements().get(name);
if (elem == null) {
throw new RuntimeException("Invalid config element named '" + name + "'");
}
Class<T> type = (Class<T>) elem.getType();
T typeCastedValue = type.cast(o);
return typeCastedValue;
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(this.configModel.getOf().getSimpleName()).append(":");
sb.append(this.configModel);
return sb.toString();
}
public boolean isEmpty() {
return data == null || data.isEmpty();
}
public Map<String, Object> getMap() {
return data;
}
}

158
nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/Param.java Normal file
View File

@ -0,0 +1,158 @@
package io.nosqlbench.nb.api.config.standard;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* A configuration element describes a single configurable parameter.
*
* @param <T> The type of value which can be stored in this named configuration
* parameter in in actual configuration data.
*/
public class Param<T> {
public final String name;
public final Class<? extends T> type;
public String description;
private final T defaultValue;
public boolean required;
private Pattern regex;
public Param(
String name,
Class<? extends T> type,
String description,
boolean required,
T defaultValue
) {
this.name = name;
this.type = type;
this.description = description;
this.required = required;
this.defaultValue = defaultValue;
}
public static <V> Param<V> optional(String name) {
return (Param<V>) optional(name,String.class);
}
public static <V> Param<V> optional(String name, Class<V> type) {
return new Param<V>(name,type,null,false,null);
}
public static <V> Param<V> defaultTo(String name, V defaultValue) {
return new Param<V>(name,(Class<V>) defaultValue.getClass(),null,false,null);
}
@Override
public String toString() {
return "Element{" +
"name='" + name + '\'' +
", type=" + type +
", description='" + description + '\'' +
", required=" + required +
", defaultValue = " + defaultValue +
'}';
}
public String getName() {
return name;
}
public Class<?> getType() {
return type;
}
public String getDescription() {
return description;
}
public boolean isRequired() {
return required;
}
public void setRequired(boolean required) {
this.required = required;
}
public T getDefaultValue() {
return defaultValue;
}
public Param<T> setDescription(String description) {
this.description = description;
return this;
}
public Param<T> setRegex(Pattern regex) {
this.regex = regex;
return this;
}
public Param<T> setRegex(String pattern) {
this.regex = Pattern.compile(pattern);
return this;
}
public Pattern getRegex() {
return regex;
}
public CheckResult<T> validate(Object value) {
if (value == null) {
if (isRequired()) {
return CheckResult.INVALID(this, null, "Value is null but " + this.getName() + " is required");
} else {
return CheckResult.VALID(this, null, "Value is null, but " + this.getName() + " is not required");
}
}
if (!this.getType().isAssignableFrom(value.getClass())) {
return CheckResult.INVALID(this, value, "Can't assign " + value.getClass().getSimpleName() + " to " + "" +
this.getType().getSimpleName());
}
if (getRegex() != null) {
if (value instanceof CharSequence) {
Matcher matcher = getRegex().matcher(value.toString());
if (!matcher.matches()) {
return CheckResult.INVALID(this, value,
"Could not match required pattern (" + getRegex().toString() +
") with value '" + value + "' for field '" + getName() + "'");
}
}
}
return CheckResult.VALID(this,value,"All validators passed for field '" + getName() + "'");
}
public final static class CheckResult<T> {
public final Param<T> element;
public final Object value;
public final String message;
private final boolean isValid;
private CheckResult(Param<T> e, Object value, String message, boolean isValid) {
this.element = e;
this.value = value;
this.message = message;
this.isValid = isValid;
}
public static <T> CheckResult<T> VALID(Param<T> element, Object value, String message) {
return new CheckResult<>(element, value, message, true);
}
public static <T> CheckResult<T> VALID(Param<T> element, Object value) {
return new CheckResult<>(element, value, "", true);
}
public static <T> CheckResult<T> INVALID(Param<T> element, Object value, String message) {
return new CheckResult<>(element, value, message, false);
}
public boolean isValid() {
return isValid;
}
}
}

2
nb-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/ParamsParserTest.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.activityimpl.motor;
import io.nosqlbench.nb.api.config.ParamsParser;
import io.nosqlbench.nb.api.config.params.ParamsParser;
import org.junit.jupiter.api.Test;
import java.util.Map;

3
nb-api/src/test/java/io/nosqlbench/nb/api/config/ConfigLoaderTest.java
View File

@ -1,5 +1,6 @@
package io.nosqlbench.nb.api.config;
import io.nosqlbench.nb.api.config.standard.ConfigLoader;
import org.junit.jupiter.api.Test;
import java.util.List;
@ -46,4 +47,4 @@ public class ConfigLoaderTest {
assertThat(cfg1).isNull();
}
}
}

56
nb-api/src/test/java/io/nosqlbench/nb/api/config/params/NBParamsTest.java
View File

@ -15,23 +15,23 @@ public class NBParamsTest {
public void testMapObject() {
Element one = NBParams.one(Map.of("key1", "value1", "key2", new Date()));
assertThat(one.get("key1", String.class)).isPresent();
assertThat(one.get("key1", String.class).get()).isOfAnyClassIn(String.class);
assertThat(one.get("key1", String.class).get()).isEqualTo("value1");
assertThat(one.get("key1", String.class)).containsInstanceOf(String.class);
assertThat(one.get("key1", String.class)).contains("value1");
}
@Test
public void testNestedMapObject() {
Element one = NBParams.one(Map.of("key1", Map.of("key2", "value2")));
assertThat(one.get("key1.key2", String.class).get()).isOfAnyClassIn(String.class);
assertThat(one.get("key1.key2", String.class).get()).isEqualTo("value2");
assertThat(one.get("key1.key2", String.class)).containsInstanceOf(String.class);
assertThat(one.get("key1.key2", String.class)).contains("value2");
}
@Test
public void testNestedMixedJsonMapParams() {
Element one = NBParams.one("{\"key1\":{\"key2\":\"key3=value3 key4=value4\"}}");
assertThat(one.get("key1.key2.key3", String.class)).isPresent();
assertThat(one.get("key1.key2.key3", String.class).get()).isEqualTo("value3");
assertThat(one.get("key1.key2.key4", String.class).get()).isEqualTo("value4");
assertThat(one.get("key1.key2.key3", String.class)).contains("value3");
assertThat(one.get("key1.key2.key4", String.class)).contains("value4");
}
@Test
@ -40,45 +40,71 @@ public class NBParamsTest {
String source = "{\"key1\":\"key2={\\\"key3\\\":\\\"value3\\\",\\\"key4\\\":\\\"value4\\\"}\"}";
Element one = NBParams.one(source);
assertThat(one.get("key1.key2.key3", String.class)).isPresent();
assertThat(one.get("key1.key2.key3", String.class).get()).isEqualTo("value3");
assertThat(one.get("key1.key2.key4", String.class).get()).isEqualTo("value4");
assertThat(one.get("key1.key2.key3", String.class)).contains("value3");
assertThat(one.get("key1.key2.key4", String.class)).contains("value4");
}
@Test
public void testNestedMixedMapJsonParams() {
Element one = NBParams.one(Map.of("key1", "{ \"key2\": \"key3=value3 key4=value4\"}"));
assertThat(one.get("key1.key2.key3", String.class)).isPresent();
assertThat(one.get("key1.key2.key3", String.class).get()).isEqualTo("value3");
assertThat(one.get("key1.key2.key4", String.class).get()).isEqualTo("value4");
assertThat(one.get("key1.key2.key3", String.class)).contains("value3");
assertThat(one.get("key1.key2.key4", String.class)).contains("value4");
}
@Test
public void testNestedMixedMapParamsJson() {
Element one = NBParams.one(Map.of("key1", "key2: {\"key3\":\"value3\",\"key4\":\"value4\"}"));
assertThat(one.get("key1.key2.key3", String.class)).isPresent();
assertThat(one.get("key1.key2.key3", String.class).get()).isEqualTo("value3");
assertThat(one.get("key1.key2.key4", String.class).get()).isEqualTo("value4");
assertThat(one.get("key1.key2.key3", String.class)).contains("value3");
assertThat(one.get("key1.key2.key4", String.class)).contains("value4");
}
@Test
public void testJsonText() {
Element one = NBParams.one("{\"key1\":\"value1\"}");
assertThat(one.get("key1", String.class)).isPresent();
assertThat(one.get("key1", String.class).get()).isEqualTo("value1");
assertThat(one.get("key1", String.class)).contains("value1");
}
@Test
public void testNamedFromJsonSeq() {
Element one = NBParams.one("[{\"name\":\"first\",\"val\":\"v1\"},{\"name\":\"second\",\"val\":\"v2\"}]");
assertThat(one.get("first.val", String.class)).isPresent();
assertThat(one.get("first.val", String.class).get()).isEqualTo("v1");
assertThat(one.get("first.val", String.class)).contains("v1");
}
@Test
public void testNamedFromMapSeq() {
Element one = NBParams.one(List.of(Map.of("name", "first", "val", "v1"), Map.of("name", "second", "val", "v2")));
assertThat(one.get("first.val", String.class)).isPresent();
assertThat(one.get("first.val", String.class).get()).isEqualTo("v1");
assertThat(one.get("first.val", String.class)).contains("v1");
}
@Test
public void testDepthPrecedence() {
Map<String, Object> a1 = Map.of(
"a1", Map.of("b1", "v_a1_b1"),
"a2.b2", Map.of("c2", "v_a2.b2_c2"),
"a3",Map.of("b3.c3","v_a3_b3.c3-1"),
"a3.b3",Map.of("c3","v_a3.b3_c3-2"),
"a4.b4",Map.of("c4","v_a4.b4_c4-3"),
"a5",Map.of("b5.c5","v_a5_b5.c5-4")
);
Element e = NBParams.one("testdata",a1);
assertThat(e.get("a1",Map.class)).contains(Map.of("b1","v_a1_b1"));
assertThat(e.get("a2.b2",Map.class)).contains(Map.of("c2","v_a2.b2_c2"));
assertThat(e.get("a3.b3",Map.class)).contains(Map.of("c3","v_a3.b3_c3-2"));
assertThat(e.get("a3.b3.c3",String.class)).contains("v_a3_b3.c3-1");
assertThat(e.get("a4.b4.c4")).contains("v_a4.b4_c4-3");
assertThat(e.get("a5.b5.c5")).contains("v_a5_b5.c5-4");
// So far, this code does not decompose logical structure for things passed in composite name elements
// This is not needed, maybe ever.
assertThat(e.get("a5.b5")).isEmpty();
}
}

17
nb-api/src/test/java/io/nosqlbench/nb/api/config/standard/ConfigElementTest.java Normal file
View File

@ -0,0 +1,17 @@
package io.nosqlbench.nb.api.config.standard;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class ConfigElementTest {
@Test
public void testRegex() {
Param<String> cfgmodel =
new Param<>("testvar",String.class,"testing a var",false,null).setRegex("WOO");
assertThat(cfgmodel.validate("WOO").isValid()).isTrue();
}
}

2
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/config/ConfigDataTest.java
View File

@ -1,6 +1,6 @@
package io.nosqlbench.virtdata.core.config;
import io.nosqlbench.nb.api.config.ConfigData;
import io.nosqlbench.nb.api.config.standard.ConfigData;
import org.junit.jupiter.api.Test;
import java.util.List;

12
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/stateful/LoadElement.java
View File

@ -1,10 +1,10 @@
package io.nosqlbench.virtdata.library.basics.shared.stateful;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.nb.api.config.ConfigAware;
import io.nosqlbench.nb.api.config.ConfigModel;
import io.nosqlbench.nb.api.config.MutableConfigModel;
import io.nosqlbench.nb.api.config.standard.NBMapConfigurable;
import java.util.Map;
import java.util.function.Function;
@ -17,7 +17,7 @@ import java.util.function.Function;
* by the provided variable name.
*/
@ThreadSafeMapper
public class LoadElement implements Function<Object,Object>, ConfigAware {
public class LoadElement implements Function<Object,Object>, NBMapConfigurable {
private final String varname;
private final Object defaultValue;
@ -50,7 +50,7 @@ public class LoadElement implements Function<Object,Object>, ConfigAware {
}
@Override
public ConfigModel getConfigModel() {
return new MutableConfigModel(this).optional("<mapname>", Map.class);
public NBConfigModel getConfigModel() {
return ConfigModel.of(this.getClass()).optional("<mapname>", Map.class);
}
}