IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-14 08:33:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

config API improvements

Browse Source
This commit is contained in:
Jonathan Shook 2020-11-16 17:28:39 -06:00
parent e0498ff29b
commit 4996d206c7
5 changed files with 273 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
nb-api/src/main/java/io/nosqlbench/nb/api/config/ConfigLoader.java Normal file
View File

@ -0,0 +1,115 @@
package io.nosqlbench.nb.api.config;
import com.google.gson.*;
import io.nosqlbench.nb.api.content.NBIO;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
/**
* <P>The config loader is meant to be the way that configurations
* for objects or subsystems are loaded generically.</P>
*
* <p>It supports value which are defined as JSON objects, lists
* of JSON objects, or as a fall-back simple parameter maps according to
* {@link ParamsParser} rules.</p>
*
* If a block of config data begins with a '[' (open square bracket),
* it is taken as a JSON list of configs. If it starts with a '{' (open curly
* brace), it is taken as a single config. Otherwise it is taken as a simple
* set of named parameters using '=' as an assignment operator.
*
* An empty string represents the null value.
*
* Users of this interface should be prepared to receive a null, or a list
* of zero or more config elements of the requested type.
*
* <H1>Importing configs</H1>
* <p>
* Configs can be imported from local files, classpath resources, or URLs.
* This is supported with the form of <pre>{@code IMPORT{URL}}</pre>
* where URL can be any form mentioned above. This syntax is obtuse and
* strange, but the point of this is to use something
* that should never occur in the wild, to avoid collisions with actual
* configuration content, but which is also clearly doing what it says.</p>
*/
public class ConfigLoader {
private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
private final static Logger logger = LogManager.getLogger("CONFIG");
/**
* Load a string into an ordered map of objects, with the key being defined
* by an extractor function over the objects. Any duplicate keys are treated
* as an error. This is a useful method for loading configuration blocks
* which must be distinctly named.
*
* @param source The config data
* @param type The type of configuration object to be stored in the map values
* @param keyer The function that extracts the key
* @param <V> The generic parameter for the type field
* @return A map of named configuration objects
*/
public <V> LinkedHashMap<String, V> loadMap(
CharSequence source,
Class<? extends V> type,
Function<V, String> keyer) {
LinkedHashMap<String, V> mapOf = new LinkedHashMap<>();
List<V> elems = load(source, type);
for (V elem : elems) {
String key = keyer.apply(elem);
if (mapOf.containsKey(key)) {
throw new RuntimeException("Duplicitous key mappings are disallowed here.");
}
mapOf.put(key, elem);
}
return mapOf;
}
public <T> List<T> load(CharSequence source, Class<? extends T> type) {
List<T> cfgs = new ArrayList<>();
String data = source.toString();
data = data.trim();
if (data.isEmpty()) {
return null;
}
if (data.startsWith("IMPORT{") && data.endsWith("}")) {
String filename = data.substring("IMPORT{".length(), data.length() - 1);
Path filepath = Path.of(filename);
data = NBIO.all().name(filename).first()
.map(c -> {
logger.debug("found 'data' at " + c.getURI());
return c.asString();
}).orElseThrow();
}
if (data.startsWith("{") || data.startsWith("[")) {
JsonParser parser = new JsonParser();
JsonElement jsonElement = parser.parse(data);
if (jsonElement.isJsonArray()) {
JsonArray asJsonArray = jsonElement.getAsJsonArray();
for (JsonElement element : asJsonArray) {
T object = gson.fromJson(element, type);
cfgs.add(object);
}
} else if (jsonElement.isJsonObject()) {
cfgs.add(gson.fromJson(jsonElement, type));
}
} else if (Map.class.isAssignableFrom(type)) {
Map<String, String> parsedMap = ParamsParser.parse(data, false);
cfgs.add(type.cast(parsedMap));
}
return cfgs;
}
}

16
nb-api/src/main/java/io/nosqlbench/nb/api/config/ConfigModel.java
View File

@ -1,17 +1,13 @@
package io.nosqlbench.nb.api.config;
import java.util.List;
import java.util.Map;
public interface ConfigModel {
List<Element> getElements();
Map<String, ConfigElement> getElements();
class Element {
public final String name;
public final Class<?> type;
Class<?> getOf();
public Element(String name, Class<?> type) {
this.name = name;
this.type = type;
}
}
void assertValidConfig(Map<String, ?> config);
ConfigReader apply(Map<String, ?> config);
}

110
nb-api/src/main/java/io/nosqlbench/nb/api/config/MutableConfigModel.java
View File

@ -1,22 +1,52 @@
package io.nosqlbench.nb.api.config;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.*;
public class MutableConfigModel implements ConfigModel {
private final List<ConfigModel.Element> elements = new ArrayList<>();
private final LinkedHashMap<String, ConfigElement> elements = new LinkedHashMap<>();
private final Class<?> ofType;
public MutableConfigModel() {}
public MutableConfigModel(Class<?> ofType) {
this.ofType = ofType;
}
public MutableConfigModel add(String name, Class<?> clazz) {
add(new ConfigModel.Element(name, clazz));
public MutableConfigModel(Object ofObject) {
this.ofType = ofObject.getClass();
}
public MutableConfigModel optional(String name, Class<?> clazz) {
add(new ConfigElement(name, clazz, "", false, null));
return this;
}
private void add(ConfigModel.Element element) {
this.elements.add(element);
public MutableConfigModel optional(String name, Class<?> clazz, String description) {
add(new ConfigElement(name, clazz, description, false, null));
return this;
}
public MutableConfigModel required(String name, Class<?> clazz, String description) {
add(new ConfigElement(name, clazz, description, true, null));
return this;
}
public MutableConfigModel required(String name, Class<?> clazz) {
add(new ConfigElement(name, clazz, "", true, null));
return this;
}
public MutableConfigModel defaultto(String name, Object defaultValue) {
add(new ConfigElement(name, defaultValue.getClass(), "", true, defaultValue));
return this;
}
public MutableConfigModel defaultto(String name, Object defaultValue, String description) {
add(new ConfigElement(name, defaultValue.getClass(), description, true, defaultValue));
return this;
}
private void add(ConfigElement element) {
this.elements.put(element.name, element);
}
public ConfigModel asReadOnly() {
@ -24,7 +54,65 @@ public class MutableConfigModel implements ConfigModel {
}
@Override
public List<Element> getElements() {
return Collections.unmodifiableList(elements);
public Map<String, ConfigElement> getElements() {
return Collections.unmodifiableMap(elements);
}
@Override
public Class<?> getOf() {
return ofType;
}
@Override
public void assertValidConfig(Map<String, ?> config) {
for (String configkey : config.keySet()) {
ConfigElement element = this.elements.get(configkey);
if (element == null) {
throw new RuntimeException(
"Unknown config parameter in config model '" + configkey + "'\n" +
"while configuring a " + getOf().getSimpleName());
}
Object value = config.get(configkey);
if (!element.getType().isAssignableFrom(value.getClass())) {
throw new RuntimeException("Unable to assign provided configuration\n" +
"of type '" + value.getClass().getSimpleName() + " to config\n" +
"parameter of type '" + element.getType().getSimpleName() + "'\n" +
"while configuring a " + getOf().getSimpleName());
}
}
for (ConfigElement element : elements.values()) {
if (element.isRequired() && element.getDefaultValue() == null) {
if (!config.containsKey(element.getName())) {
throw new RuntimeException("A required config element named '" + element.getName() +
"' and type '" + element.getType().getSimpleName() + "' was not found\n" +
"for configuring a " + getOf().getSimpleName());
}
}
}
}
@Override
public ConfigReader apply(Map<String, ?> config) {
assertValidConfig(config);
LinkedHashMap<String, Object> validConfig = new LinkedHashMap<>();
elements.forEach((k, v) -> {
String name = v.getName();
Class<?> type = v.getType();
Object cval = config.get(name);
if (cval == null && v.isRequired()) {
cval = v.getDefaultValue();
}
if (cval != null) {
if (type.isAssignableFrom(cval.getClass())) {
validConfig.put(name, cval);
} else {
throw new RuntimeException("Unable to assign a " + cval.getClass().getSimpleName() +
" to a " + type.getSimpleName());
}
}
});
return new ConfigReader(this.asReadOnly(), validConfig);
}
}

49
nb-api/src/test/java/io/nosqlbench/nb/api/config/ConfigLoaderTest.java Normal file
View File

@ -0,0 +1,49 @@
package io.nosqlbench.nb.api.config;
import org.junit.Test;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class ConfigLoaderTest {
@Test
public void testSingleParams() {
ConfigLoader cl = new ConfigLoader();
List<Map> cfg1 = cl.load("a=b c=234", Map.class);
assertThat(cfg1).contains(Map.of("a", "b", "c", "234"));
}
@Test
public void testSingleJsonObject() {
ConfigLoader cl = new ConfigLoader();
List<Map> cfg1 = cl.load("{a:'b', c:'234'}", Map.class);
assertThat(cfg1).contains(Map.of("a", "b", "c", "234"));
}
@Test
public void testJsonArray() {
ConfigLoader cl = new ConfigLoader();
List<Map> cfg1 = cl.load("[{a:'b', c:'234'}]", Map.class);
assertThat(cfg1).contains(Map.of("a", "b", "c", "234"));
}
@Test
public void testImportSingle() {
ConfigLoader cl = new ConfigLoader();
List<Map> imported = cl.load("IMPORT{importable-config.json}", Map.class);
assertThat(imported).contains(Map.of("a", "B", "b", "C", "c", 123.0, "d", 45.6));
}
@Test
public void testEmpty() {
ConfigLoader cl = new ConfigLoader();
List<Map> cfg1 = cl.load("", Map.class);
assertThat(cfg1).isNull();
}
}

8
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/stateful/LoadElement.java
View File

@ -42,15 +42,15 @@ public class LoadElement implements Function<Object,Object>, ConfigAware {
}
@Override
public void applyConfig(Map<String, ?> elements) {
Map<String,?> vars = (Map<String, ?>) elements.get(mapname);
if (vars!=null) {
public void applyConfig(Map<String, ?> providedConfig) {
Map<String, ?> vars = (Map<String, ?>) providedConfig.get(mapname);
if (vars != null) {
this.vars = vars;
}
}
@Override
public ConfigModel getConfigModel() {
return new MutableConfigModel().add("<mapname>",Map.class);
return new MutableConfigModel(this).optional("<mapname>", Map.class);
}
}