config API improvements

nb-api/src/main/java/io/nosqlbench/nb/api/config/ConfigLoader.java

@@ -0,0 +1,115 @@
+package io.nosqlbench.nb.api.config;
+
+import com.google.gson.*;
+import io.nosqlbench.nb.api.content.NBIO;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * <P>The config loader is meant to be the way that configurations
+ * for objects or subsystems are loaded generically.</P>
+ *
+ * <p>It supports value which are defined as JSON objects, lists
+ * of JSON objects, or as a fall-back simple parameter maps according to
+ * {@link ParamsParser} rules.</p>
+ *
+ * If a block of config data begins with a '[' (open square bracket),
+ * it is taken as a JSON list of configs. If it starts with a '{' (open curly
+ * brace), it is taken as a single config. Otherwise it is taken as a simple
+ * set of named parameters using '=' as an assignment operator.
+ *
+ * An empty string represents the null value.
+ *
+ * Users of this interface should be prepared to receive a null, or a list
+ * of zero or more config elements of the requested type.
+ *
+ * <H1>Importing configs</H1>
+ * <p>
+ * Configs can be imported from local files, classpath resources, or URLs.
+ * This is supported with the form of <pre>{@code IMPORT{URL}}</pre>
+ * where URL can be any form mentioned above. This syntax is obtuse and
+ * strange, but the point of this is to use something
+ * that should never occur in the wild, to avoid collisions with actual
+ * configuration content, but which is also clearly doing what it says.</p>
+ */
+public class ConfigLoader {
+    private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
+    private final static Logger logger = LogManager.getLogger("CONFIG");
+
+    /**
+     * Load a string into an ordered map of objects, with the key being defined
+     * by an extractor function over the objects. Any duplicate keys are treated
+     * as an error. This is a useful method for loading configuration blocks
+     * which must be distinctly named.
+     *
+     * @param source The config data
+     * @param type   The type of configuration object to be stored in the map values
+     * @param keyer  The function that extracts the key
+     * @param <V>    The generic parameter for the type field
+     * @return A map of named configuration objects
+     */
+    public <V> LinkedHashMap<String, V> loadMap(
+            CharSequence source,
+            Class<? extends V> type,
+            Function<V, String> keyer) {
+
+        LinkedHashMap<String, V> mapOf = new LinkedHashMap<>();
+        List<V> elems = load(source, type);
+
+        for (V elem : elems) {
+            String key = keyer.apply(elem);
+            if (mapOf.containsKey(key)) {
+                throw new RuntimeException("Duplicitous key mappings are disallowed here.");
+            }
+            mapOf.put(key, elem);
+        }
+        return mapOf;
+    }
+
+    public <T> List<T> load(CharSequence source, Class<? extends T> type) {
+        List<T> cfgs = new ArrayList<>();
+
+        String data = source.toString();
+        data = data.trim();
+        if (data.isEmpty()) {
+            return null;
+        }
+
+        if (data.startsWith("IMPORT{") && data.endsWith("}")) {
+            String filename = data.substring("IMPORT{".length(), data.length() - 1);
+            Path filepath = Path.of(filename);
+
+            data = NBIO.all().name(filename).first()
+                    .map(c -> {
+                        logger.debug("found 'data' at " + c.getURI());
+                        return c.asString();
+                    }).orElseThrow();
+        }
+
+        if (data.startsWith("{") || data.startsWith("[")) {
+            JsonParser parser = new JsonParser();
+
+            JsonElement jsonElement = parser.parse(data);
+            if (jsonElement.isJsonArray()) {
+                JsonArray asJsonArray = jsonElement.getAsJsonArray();
+                for (JsonElement element : asJsonArray) {
+                    T object = gson.fromJson(element, type);
+                    cfgs.add(object);
+                }
+            } else if (jsonElement.isJsonObject()) {
+                cfgs.add(gson.fromJson(jsonElement, type));
+            }
+        } else if (Map.class.isAssignableFrom(type)) {
+            Map<String, String> parsedMap = ParamsParser.parse(data, false);
+            cfgs.add(type.cast(parsedMap));
+        }
+        return cfgs;
+    }
+}

nb-api/src/main/java/io/nosqlbench/nb/api/config/ConfigModel.java

@@ -1,17 +1,13 @@
 package io.nosqlbench.nb.api.config;
 
-import java.util.List;
+import java.util.Map;
 
 public interface ConfigModel {
-    List<Element> getElements();
+    Map<String, ConfigElement> getElements();
 
-    class Element {
+    Class<?> getOf();
-        public final String name;
-        public final Class<?> type;
 
-        public Element(String name, Class<?> type) {
+    void assertValidConfig(Map<String, ?> config);
-            this.name = name;
+
-            this.type = type;
+    ConfigReader apply(Map<String, ?> config);
-        }
-    }
 }

nb-api/src/main/java/io/nosqlbench/nb/api/config/MutableConfigModel.java

@@ -1,22 +1,52 @@
 package io.nosqlbench.nb.api.config;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collections;
-import java.util.List;
 
 public class MutableConfigModel implements ConfigModel {
 
-    private final List<ConfigModel.Element> elements = new ArrayList<>();
+    private final LinkedHashMap<String, ConfigElement> elements = new LinkedHashMap<>();
+    private final Class<?> ofType;
 
-    public MutableConfigModel() {}
+    public MutableConfigModel(Class<?> ofType) {
+        this.ofType = ofType;
+    }
 
-    public MutableConfigModel add(String name, Class<?> clazz) {
+    public MutableConfigModel(Object ofObject) {
-        add(new ConfigModel.Element(name, clazz));
+        this.ofType = ofObject.getClass();
+    }
+
+    public MutableConfigModel optional(String name, Class<?> clazz) {
+        add(new ConfigElement(name, clazz, "", false, null));
         return this;
     }
 
-    private void add(ConfigModel.Element element) {
+    public MutableConfigModel optional(String name, Class<?> clazz, String description) {
-        this.elements.add(element);
+        add(new ConfigElement(name, clazz, description, false, null));
+        return this;
+    }
+
+    public MutableConfigModel required(String name, Class<?> clazz, String description) {
+        add(new ConfigElement(name, clazz, description, true, null));
+        return this;
+    }
+
+    public MutableConfigModel required(String name, Class<?> clazz) {
+        add(new ConfigElement(name, clazz, "", true, null));
+        return this;
+    }
+
+    public MutableConfigModel defaultto(String name, Object defaultValue) {
+        add(new ConfigElement(name, defaultValue.getClass(), "", true, defaultValue));
+        return this;
+    }
+
+    public MutableConfigModel defaultto(String name, Object defaultValue, String description) {
+        add(new ConfigElement(name, defaultValue.getClass(), description, true, defaultValue));
+        return this;
+    }
+
+    private void add(ConfigElement element) {
+        this.elements.put(element.name, element);
     }
 
     public ConfigModel asReadOnly() {
@@ -24,7 +54,65 @@ public class MutableConfigModel implements ConfigModel {
     }
 
     @Override
-    public List<Element> getElements() {
+    public Map<String, ConfigElement> getElements() {
-        return Collections.unmodifiableList(elements);
+        return Collections.unmodifiableMap(elements);
+    }
+
+    @Override
+    public Class<?> getOf() {
+        return ofType;
+    }
+
+    @Override
+    public void assertValidConfig(Map<String, ?> config) {
+        for (String configkey : config.keySet()) {
+            ConfigElement element = this.elements.get(configkey);
+            if (element == null) {
+                throw new RuntimeException(
+                        "Unknown config parameter in config model '" + configkey + "'\n" +
+                                "while configuring a " + getOf().getSimpleName());
+            }
+            Object value = config.get(configkey);
+            if (!element.getType().isAssignableFrom(value.getClass())) {
+                throw new RuntimeException("Unable to assign provided configuration\n" +
+                        "of type '" + value.getClass().getSimpleName() + " to config\n" +
+                        "parameter of type '" + element.getType().getSimpleName() + "'\n" +
+                        "while configuring a " + getOf().getSimpleName());
+            }
+        }
+        for (ConfigElement element : elements.values()) {
+            if (element.isRequired() && element.getDefaultValue() == null) {
+                if (!config.containsKey(element.getName())) {
+                    throw new RuntimeException("A required config element named '" + element.getName() +
+                            "' and type '" + element.getType().getSimpleName() + "' was not found\n" +
+                            "for configuring a " + getOf().getSimpleName());
+                }
+            }
+        }
+    }
+
+    @Override
+    public ConfigReader apply(Map<String, ?> config) {
+        assertValidConfig(config);
+        LinkedHashMap<String, Object> validConfig = new LinkedHashMap<>();
+
+        elements.forEach((k, v) -> {
+            String name = v.getName();
+            Class<?> type = v.getType();
+            Object cval = config.get(name);
+            if (cval == null && v.isRequired()) {
+                cval = v.getDefaultValue();
+            }
+            if (cval != null) {
+                if (type.isAssignableFrom(cval.getClass())) {
+                    validConfig.put(name, cval);
+                } else {
+                    throw new RuntimeException("Unable to assign a " + cval.getClass().getSimpleName() +
+                            " to a " + type.getSimpleName());
+                }
+            }
+        });
+
+        return new ConfigReader(this.asReadOnly(), validConfig);
     }
 }

nb-api/src/test/java/io/nosqlbench/nb/api/config/ConfigLoaderTest.java

@@ -0,0 +1,49 @@
+package io.nosqlbench.nb.api.config;
+
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+public class ConfigLoaderTest {
+
+    @Test
+    public void testSingleParams() {
+        ConfigLoader cl = new ConfigLoader();
+        List<Map> cfg1 = cl.load("a=b c=234", Map.class);
+        assertThat(cfg1).contains(Map.of("a", "b", "c", "234"));
+    }
+
+    @Test
+    public void testSingleJsonObject() {
+        ConfigLoader cl = new ConfigLoader();
+        List<Map> cfg1 = cl.load("{a:'b', c:'234'}", Map.class);
+        assertThat(cfg1).contains(Map.of("a", "b", "c", "234"));
+    }
+
+    @Test
+    public void testJsonArray() {
+        ConfigLoader cl = new ConfigLoader();
+        List<Map> cfg1 = cl.load("[{a:'b', c:'234'}]", Map.class);
+        assertThat(cfg1).contains(Map.of("a", "b", "c", "234"));
+
+    }
+
+    @Test
+    public void testImportSingle() {
+        ConfigLoader cl = new ConfigLoader();
+        List<Map> imported = cl.load("IMPORT{importable-config.json}", Map.class);
+        assertThat(imported).contains(Map.of("a", "B", "b", "C", "c", 123.0, "d", 45.6));
+    }
+
+    @Test
+    public void testEmpty() {
+        ConfigLoader cl = new ConfigLoader();
+        List<Map> cfg1 = cl.load("", Map.class);
+        assertThat(cfg1).isNull();
+
+    }
+}

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/stateful/LoadElement.java

@@ -42,15 +42,15 @@ public class LoadElement implements Function<Object,Object>, ConfigAware {
     }
 
     @Override
-    public void applyConfig(Map<String, ?> elements) {
+    public void applyConfig(Map<String, ?> providedConfig) {
-        Map<String,?> vars = (Map<String, ?>) elements.get(mapname);
+        Map<String, ?> vars = (Map<String, ?>) providedConfig.get(mapname);
-        if (vars!=null) {
+        if (vars != null) {
             this.vars = vars;
         }
     }
 
     @Override
     public ConfigModel getConfigModel() {
-        return new MutableConfigModel().add("<mapname>",Map.class);
+        return new MutableConfigModel(this).optional("<mapname>", Map.class);
     }
 }