From 41ce92cf9c3fc7ceb5cb0dbeabdeef59157dacf2 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 23 Jun 2021 11:42:36 -0500 Subject: [PATCH] OpTemplate type variation for testing --- .../api/activityconfig/yaml/OpData.java | 131 ++++++++++++ .../engine/api/activityconfig/yaml/OpDef.java | 172 ++-------------- .../api/activityconfig/yaml/OpTemplate.java | 186 +++++++++++++++--- .../api/activityconfig/rawyaml/OpDefTest.java | 12 +- 4 files changed, 321 insertions(+), 180 deletions(-) create mode 100644 engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpData.java diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpData.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpData.java new file mode 100644 index 000000000..a3e449eb7 --- /dev/null +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpData.java @@ -0,0 +1,131 @@ +package io.nosqlbench.engine.api.activityconfig.yaml; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +public class OpData extends OpTemplate { + + private String desc = ""; + private String name = ""; + private Map op; + private Map params = Map.of(); + private Map bindings = Map.of(); + private Map tags = new LinkedHashMap<>(); + + public OpData(String desc, String name, Map tags, Map bindings, Map params, Map op) { + this.desc = desc; + this.name = name; + this.tags = tags; + this.bindings = bindings; + this.params = params; + this.op = op; + } + + public OpData() {} + + + public OpData(Map opdata) { + applyFields(opdata); + if (opdata.size()>0) { + throw new RuntimeException("Unconsumed fields in construction of op data from map: " + opdata); + } + } + + public OpData applyFields(Map opdata) { + Optional.ofNullable(opdata.remove(OpTemplate.FIELD_DESC)).ifPresent(v -> this.setDesc(v.toString())); + Optional.ofNullable(opdata.remove(OpTemplate.FIELD_NAME)).ifPresent(v -> this.setName(v.toString())); + + Optional.ofNullable(opdata.remove(OpTemplate.FIELD_BINDINGS)).ifPresent(this::setBindings); + Optional.ofNullable(opdata.remove(OpTemplate.FIELD_OP)).ifPresent(this::setOp); + Optional.ofNullable(opdata.remove(OpTemplate.FIELD_PARAMS)).ifPresent(this::setParams); + Optional.ofNullable(opdata.remove(OpTemplate.FIELD_TAGS)).ifPresent(this::setTags); + return this; + } + + private void setTags(Object o) { + if (o instanceof Map) { + ((Map) o).forEach((k,v) -> { + this.tags.put(k.toString(),v.toString()); + }); + } else { + throw new RuntimeException("Invalid type for tags: " + o.getClass().getSimpleName()); + } + } + + private void setParams(Object o) { + if (o instanceof Map) { + this.params = new LinkedHashMap<>(); + ((Map) o).forEach((k,v) -> { + this.params.put(k.toString(),v); + }); + } else { + throw new RuntimeException("Invalid type for params: " + op.getClass().getSimpleName()); + } + } + + private void setOp(Object o) { + if (o instanceof CharSequence) { + this.op = new LinkedHashMap<>(Map.of("stmt",o.toString())); + } else if (o instanceof Map) { + this.op = new LinkedHashMap<>((Map)o); + } else { + throw new RuntimeException("Invalid type for op:" + op.getClass().getSimpleName()); + } + } + + private void setBindings(Object bindings) { + if (bindings instanceof Map) { + this.bindings = new LinkedHashMap<>(); + ((Map) bindings).forEach((k,v) -> { + this.bindings.put(k.toString(),v.toString()); + }); + } else if (bindings!=null) { + throw new RuntimeException("Invalid type for bindings: " + bindings.getClass().getSimpleName()); + } + } + + private void setName(String name) { + this.name = name; + this.tags.put("name",name); + } + + private void setDesc(String desc) { + this.desc = desc; + } + + @Override + public String getDesc() { + return this.desc; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public Map getTags() { + return this.tags; + } + + @Override + public Map getBindings() { + return this.bindings; + } + + @Override + public Map getParams() { + return this.params; + } + + @Override + public Optional> getOp() { + return Optional.of(this.op); + } + + @Override + public Optional getStmt() { + return Optional.empty(); + } +} diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java index b85a0f562..11215cd28 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java @@ -17,23 +17,19 @@ package io.nosqlbench.engine.api.activityconfig.yaml; -import com.google.gson.annotations.SerializedName; import io.nosqlbench.engine.api.activityconfig.MultiMapLookup; -import io.nosqlbench.engine.api.activityconfig.ParsedStmt; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef; import io.nosqlbench.nb.api.errors.BasicError; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; -import java.util.*; -import java.util.function.Function; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; -public class OpDef implements OpTemplate { - - private static final String FIELD_DESC = "description"; - private static final String FIELD_NAME = "name"; - private static final String FIELD_OP = "op"; - private static final String FIELD_BINDINGS = "bindings"; - private static final String FIELD_PARAMS = "params"; - private static final String FIELD_TAGS = "tags"; +public class OpDef extends OpTemplate { + private final static Logger logger = LogManager.getLogger(OpDef.class); private final RawStmtDef rawStmtDef; private final StmtsBlock block; @@ -55,20 +51,23 @@ public class OpDef implements OpTemplate { } @Override - public Map getOp() { + public Optional> getOp() { Object op = rawStmtDef.getOp(); + if (op == null) { + return null; + } HashMap newmap = new LinkedHashMap<>(); if (op instanceof Map) { - ((Map)op).forEach((k,v) -> { - newmap.put(k.toString(),v); + ((Map) op).forEach((k, v) -> { + newmap.put(k.toString(), v); }); - } else if (op instanceof CharSequence) { - newmap.put("stmt",op.toString()); + } else if (op instanceof CharSequence) { + newmap.put("stmt", op.toString()); } else { throw new BasicError("Unable to coerce a '" + op.getClass().getCanonicalName() + "' into an op template"); } - return newmap; + return Optional.of(newmap); } @Override @@ -83,7 +82,6 @@ public class OpDef implements OpTemplate { } @Override - @SerializedName("params") public Map getParams() { return params; } @@ -94,96 +92,6 @@ public class OpDef implements OpTemplate { return params; } - @Override - public Map getParamsAsValueType(Class type) { - Map map = new LinkedHashMap<>(); - for (String pname : getParams().keySet()) { - Object object = getParams().get(pname); - if (object != null) { - if (type.isAssignableFrom(object.getClass())) { - map.put(pname, type.cast(object)); - } else { - throw new RuntimeException("With param named '" + pname + "" + - "' You can't assign an object of type '" + object.getClass().getSimpleName() + "" + - "' to '" + type.getSimpleName() + "'. Maybe the YAML format is suggesting the wrong type."); - } - } - } - return map; - } - - @Override - public V removeParamOrDefault(String name, V defaultValue) { - Objects.requireNonNull(defaultValue); - - if (!getParams().containsKey(name)) { - return defaultValue; - } - - Object value = getParams().remove(name); - - try { - return (V) defaultValue.getClass().cast(value); - } catch (Exception e) { - throw new RuntimeException("Unable to cast type " + value.getClass().getCanonicalName() + " to " + defaultValue.getClass().getCanonicalName(), e); - } - } - - @Override - @SuppressWarnings("unchecked") - public V getParamOrDefault(String name, V defaultValue) { - Objects.requireNonNull(defaultValue); - - if (!getParams().containsKey(name)) { - return defaultValue; - } - Object value = getParams().get(name); - try { - return (V) defaultValue.getClass().cast(value); - } catch (Exception e) { - throw new RuntimeException("Unable to cast type " + value.getClass().getCanonicalName() + " to " + defaultValue.getClass().getCanonicalName(), e); - } - } - - @Override - public V getParam(String name, Class type) { - Object object = getParams().get(name); - if (object == null) { - return null; - } - if (type.isAssignableFrom(object.getClass())) { - V value = type.cast(object); - return value; - } - throw new RuntimeException("Unable to cast type " + object.getClass().getSimpleName() + " to" + - " " + type.getSimpleName() + ". Perhaps the yaml format is suggesting the wrong type."); - } - - @Override - @SuppressWarnings("unchecked") - public Optional getOptionalStringParam(String name, Class type) { - if (type.isPrimitive()) { - throw new RuntimeException("Do not use primitive types for the target class here. For example, Boolean.class is accepted, but boolean.class is not."); - } - if (getParams().containsKey(name)) { - Object object = getParams().get(name); - if (object == null) { - return Optional.empty(); - } - try { - V reified = type.cast(object); - return Optional.of(reified); - } catch (Exception e) { - throw new RuntimeException("Unable to cast type " + object.getClass().getCanonicalName() + " to " + type.getCanonicalName()); - } - } - return Optional.empty(); - } - - @Override - public Optional getOptionalStringParam(String name) { - return getOptionalStringParam(name, String.class); - } @Override @@ -192,7 +100,9 @@ public class OpDef implements OpTemplate { } private LinkedHashMap composeTags() { - return new LinkedHashMap<>(new MultiMapLookup<>(rawStmtDef.getTags(), block.getTags())); + LinkedHashMap tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawStmtDef.getTags(), block.getTags())); + tagsWithName.put("name",getName()); + return tagsWithName; } @Override @@ -200,51 +110,9 @@ public class OpDef implements OpTemplate { return "stmt(name:" + getName() + ", stmt:" + getOp() + ", tags:(" + getTags() + "), params:(" + getParams() + "), bindings:(" + getBindings() + "))"; } - @Override - public ParsedStmt getParsed(Function... transforms) { - return new ParsedStmt(this, transforms); - } - - @Override public String getDesc() { return rawStmtDef.getDesc(); } - - @Override - public Map asData() { - LinkedHashMap fields = new LinkedHashMap<>(); - - if (this.getDesc() != null && !this.getDesc().isBlank()) { - fields.put(FIELD_DESC, this.getDesc()); - } - - if (this.getBindings().size() > 0) { - fields.put(FIELD_BINDINGS, this.getBindings()); - } - - if (this.getParams().size() > 0) { - fields.put(FIELD_PARAMS, this.getParams()); - } - - if (this.getTags().size() > 0) { - fields.put(FIELD_TAGS, this.getTags()); - } - - fields.put(FIELD_OP, this.getOp()); - - fields.put(FIELD_NAME, this.getName()); - - return fields; - } - - @Override - public String getStmt() { - if (getOp() instanceof CharSequence) { - return getOp().toString(); - } else { - throw new BasicError("tried to access op type '" + getOp().getClass().getSimpleName() + "' as a string statement."); - } - } } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java index e471c1c28..952b457a4 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java @@ -1,9 +1,13 @@ package io.nosqlbench.engine.api.activityconfig.yaml; -import io.nosqlbench.engine.api.activityconfig.ParsedStmt; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.nosqlbench.engine.api.activityconfig.ParsedStmtOp; import io.nosqlbench.engine.api.util.Tagged; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.function.Function; @@ -121,48 +125,186 @@ import java.util.function.Function; * p2: v2 * } */ -public interface OpTemplate extends Tagged { +public abstract class OpTemplate implements Tagged { - String getName(); + private final static Gson gson = new GsonBuilder().setPrettyPrinting().create(); + // TODO: coalesce Gson instances to a few statics on a central NB API class - Map getOp(); + public final static String FIELD_DESC = "description"; + public final static String FIELD_NAME = "name"; + public final static String FIELD_OP = "op"; + public final static String FIELD_BINDINGS = "bindings"; + public final static String FIELD_PARAMS = "params"; + public final static String FIELD_TAGS = "tags"; - Map getBindings(); + /** + * @return a description for the op template, or an empty string + */ + public abstract String getDesc(); - Map getParams(); + /** + * @return a name for the op template, user-specified or auto-generated + */ + public abstract String getName(); - Map getParamsAsValueType(Class type); + /** + * Return a map of tags for this statement. Implementations are required to + * add a tag for "name" automatically when this value is set during construction. + * @return A map of assigned tags for the op, with the name added as an auto-tag. + */ + public abstract Map getTags(); - VT removeParamOrDefault(String name, VT defaultValue); + public abstract Map getBindings(); + + public abstract Map getParams(); + + public Map getParamsAsValueType(Class type) { + Map map = new LinkedHashMap<>(); + for (String pname : getParams().keySet()) { + Object object = getParams().get(pname); + if (object != null) { + if (type.isAssignableFrom(object.getClass())) { + map.put(pname, type.cast(object)); + } else { + throw new RuntimeException("With param named '" + pname + "" + + "' You can't assign an object of type '" + object.getClass().getSimpleName() + "" + + "' to '" + type.getSimpleName() + "'. Maybe the YAML format is suggesting the wrong type."); + } + } + } + return map; + } + + + public V removeParamOrDefault(String name, V defaultValue) { + Objects.requireNonNull(defaultValue); + + if (!getParams().containsKey(name)) { + return defaultValue; + } + + Object value = getParams().remove(name); + + try { + return (V) defaultValue.getClass().cast(value); + } catch (Exception e) { + throw new RuntimeException("Unable to cast type " + value.getClass().getCanonicalName() + " to " + defaultValue.getClass().getCanonicalName(), e); + } + } @SuppressWarnings("unchecked") - V getParamOrDefault(String name, V defaultValue); + public V getParamOrDefault(String name, V defaultValue) { + Objects.requireNonNull(defaultValue); - V getParam(String name, Class type); + if (!getParams().containsKey(name)) { + return defaultValue; + } + Object value = getParams().get(name); + try { + return (V) defaultValue.getClass().cast(value); + } catch (Exception e) { + throw new RuntimeException("Unable to cast type " + value.getClass().getCanonicalName() + " to " + defaultValue.getClass().getCanonicalName(), e); + } + } + + + + public V getParam(String name, Class type) { + Object object = getParams().get(name); + if (object == null) { + return null; + } + if (type.isAssignableFrom(object.getClass())) { + V value = type.cast(object); + return value; + } + throw new RuntimeException("Unable to cast type " + object.getClass().getSimpleName() + " to" + + " " + type.getSimpleName() + ". Perhaps the yaml format is suggesting the wrong type."); + } @SuppressWarnings("unchecked") - Optional getOptionalStringParam(String name, Class type); + public Optional getOptionalStringParam(String name, Class type) { + if (type.isPrimitive()) { + throw new RuntimeException("Do not use primitive types for the target class here. For example, Boolean.class is accepted, but boolean.class is not."); + } + if (getParams().containsKey(name)) { + Object object = getParams().get(name); + if (object == null) { + return Optional.empty(); + } + try { + V reified = type.cast(object); + return Optional.of(reified); + } catch (Exception e) { + throw new RuntimeException("Unable to cast type " + object.getClass().getCanonicalName() + " to " + type.getCanonicalName()); + } + } + return Optional.empty(); + } - Optional getOptionalStringParam(String name); - - Map getTags(); + public Optional getOptionalStringParam(String name) { + return getOptionalStringParam(name, String.class); + } /** * Parse the statement for anchors and return a richer view of the StmtDef which * is simpler to use for most statement configuration needs. * - * @return a new {@link ParsedStmt} + * @return a new {@link ParsedStmtOp} */ - ParsedStmt getParsed(Function... transforms); + public Optional getParsed(Function... rewriters) { + Optional os = getStmt(); + return os.map(s -> { + String result = s; + for (Function rewriter : rewriters) { + result = rewriter.apply(result); + } + return result; + }).map(s -> new ParsedStmtOp(this)); + } - String getDesc(); + public abstract Optional> getOp(); - Map asData(); + public Map asData() { + LinkedHashMap fields = new LinkedHashMap<>(); + + if (this.getDesc() != null && !this.getDesc().isBlank()) { + fields.put(FIELD_DESC, this.getDesc()); + } + + if (this.getBindings().size() > 0) { + fields.put(FIELD_BINDINGS, this.getBindings()); + } + + if (this.getParams().size() > 0) { + fields.put(FIELD_PARAMS, this.getParams()); + } + + if (this.getTags().size() > 0) { + fields.put(FIELD_TAGS, this.getTags()); + } + + this.getOp().ifPresent(o -> fields.put(FIELD_OP,o)); + + fields.put(FIELD_NAME, this.getName()); + + return fields; + } /** - * Legacy support for String form statements. This will be replaced after refactoring. - * @return A string version of the op - * @throws io.nosqlbench.nb.api.errors.BasicError if the op is not a CharSequence + * Legacy support for String form statements. This is left here as a convenience method, + * however it is changed to an Optional to force caller refactorings. + * + * @return An optional string version of the op, empty if there is no 'stmt' property in the op fields, or no op fields at all. */ - String getStmt(); + public Optional getStmt() { + return getOp().map(m->m.get("stmt")).map(s->{ + if (s instanceof CharSequence) { + return s.toString(); + } else { + return gson.toJson(s); + } + }); + } + } diff --git a/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java b/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java index 04a8cc75c..b80995603 100644 --- a/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java +++ b/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java @@ -71,11 +71,11 @@ public class OpDefTest { StmtsDoc doc1 = all.getStmtDocs().get(0); StmtsBlock block1 = doc1.getBlocks().get(0); assertThat(block1.getName()).isEqualTo("doc1--block0"); - List assys = block1.getOps(); - assertThat(assys).hasSize(2); - OpTemplate sdef1 = assys.get(0); + List ops = block1.getOps(); + assertThat(ops).hasSize(2); + OpTemplate sdef1 = ops.get(0); assertThat(sdef1.getName()).isEqualTo("doc1--block0--stmt1"); - assertThat(assys.get(0).getOp()).isEqualTo("s1"); + assertThat(ops.get(0).getOp()).isEqualTo(Map.of("stmt","s1")); } @Test @@ -110,7 +110,7 @@ public class OpDefTest { OpTemplate op1 = block1.getOps().get(1); assertThat(op1.getParams()).containsAllEntriesOf(Map.of()); assertThat(op1.getName()).isEqualTo("map-of-maps--block0--s2"); - assertThat(op1.getOp()).isEqualTo("statement2"); + assertThat(op1.getOp()).isEqualTo(Map.of("stmt","statement2")); } @Test @@ -125,7 +125,7 @@ public class OpDefTest { assertThat(block1.getOps()).hasSize(1); OpTemplate op0 = block1.getOps().get(0); assertThat(op0.getName()).isEqualTo("string-statement--block0--stmt1"); - assertThat(op0.getOp()).isEqualTo("test statement"); + assertThat(op0.getOp()).isEqualTo(Map.of("stmt","test statement")); } @Test