opdef named maps

driver-cql-shaded/src/main/resources/activities/baselines/hotcold.yaml

@@ -0,0 +1,30 @@
+description: |
+  A set of named scenarios for testing cache performance.
+  This is a set of named scenarios packaged as a workload file that can
+  be used to test datasets of different sizes. This workload file
+  contains no specific workloads of its own. It is expected to be used
+  with existing workload definitions. By default, it will use
+  the cql-tabular2 workload.
+  Going forward, some conventions will be established about what parameters
+  mean for dataset sizing. For now, the names used in cql-tabular2 will be
+  the standard. For now, this means partsize and partcount.
+  partsize is used to modulo the selected partition for a row write
+  partcount is used to modulo a pseudo-random row selector to fall within the
+  known dataset size for extant data. These must be calculated together
+  to ensure that reads address valid data by default. Some partially empty
+  read ratio can be configured by adjusting these parameters with respect to
+  known dataset sizes.
+  The scenario names are suggestive of the dataset size with basic exponents.
+  For example 1e5 means 100000.
+  Defaults:
+    rows (dataset size basis)
+    partsize: rows/100
+    partcount: rows/100
+
+
+scenarios:
+  hotcold1e5:
+    schema_1e5: TEMPLATE(workload,cql-tabular2) schema
+    rampup_1e5: TEMPLATE(workload,cql-tabular2) rampup rampup-cycles=TEMPLATE(rows,1e5) partsize=TEMPLATE(partsize,1e3)
+    main_1e5: TEMPLATE(workload,cql-tabular2) main main-cycles=TEMPLATE(rows,1e5) partcount=TEMPLATE(partcount,1e3)
+

engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDef.java

@@ -33,7 +33,6 @@ public class RawStmtDef extends RawStmtFields {
         this.statement = statement;
     }
 
-
     @SuppressWarnings("unchecked")
     public RawStmtDef(String defaultName, Map<String, Object> map) {
 
@@ -63,18 +62,20 @@ public class RawStmtDef extends RawStmtFields {
                 Map values = (Map) firstEntry.getValue();
                 setFieldsByReflection(values);
                 map = values;
-            } else if (firstEntry.getValue() instanceof CharSequence){
+            } else if (firstEntry.getValue() instanceof CharSequence) {
                 setStmt(((CharSequence) firstEntry.getValue()).toString());
             }
-            map.remove(firstEntry.getKey());
             if (getName().isEmpty()) {
+                map.remove(firstEntry.getKey());
                 setName(firstEntry.getKey());
-            } else {
-                throw new RuntimeException("redefined-name-in-statement-tuple: Statement name has already been set by name parameter. Remove the name parameter for a statement definition map." +
-                        " For more details on this error see " +
-                        "the troubleshooting section in the " +
-                        "YAML format docs for redefined-name-statement-tuple");
             }
+            // TODO: Add explicit check condition for this error
+//            else {
+//                throw new RuntimeException("redefined-name-in-statement-tuple: Statement name has already been set by name parameter. Remove the name parameter for a statement definition map." +
+//                        " For more details on this error see " +
+//                        "the troubleshooting section in the " +
+//                        "YAML format docs for redefined-name-statement-tuple");
+//            }
         }
         if (getName().isEmpty()) {
             setName(defaultName);
@@ -90,7 +91,7 @@ public class RawStmtDef extends RawStmtFields {
     private void setStmt(String statement) {
         this.statement = statement;
     }
-    
+
     public String getName() {
         Object name = getParams().get("name");
         if (name!=null) {

engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java

@@ -41,7 +41,7 @@ public class RawStmtsDoc extends StatementsOwner {
 
     public static RawStmtsDoc forSingleStatement(String statement) {
         RawStmtsDoc rawStmtsDoc = new RawStmtsDoc();
-        rawStmtsDoc.setStatementsFieldByObjectType(statement);
+        rawStmtsDoc.setStatementsFieldByType(statement);
         return rawStmtsDoc;
     }
 

engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StatementsOwner.java

@@ -37,13 +37,14 @@ public class StatementsOwner extends RawStmtFields {
     public void setFieldsByReflection(Map<String, Object> propsmap) {
         Object statementsObject = propsmap.remove("statements");
 
-        if (statementsObject==null) {
+        if (statementsObject == null) {
             statementsObject = propsmap.remove("statement");
         }
 
-        if (statementsObject!=null) {
+        if (statementsObject != null) {
-            setStatementsFieldByObjectType(statementsObject);
+            setStatementsFieldByType(statementsObject);
         }
+
 //        if (statementsObject!=null) {
 //            if (statementsObject instanceof List) {
 //                setByObject(statementsObject);
@@ -55,17 +56,17 @@ public class StatementsOwner extends RawStmtFields {
     }
 
     @SuppressWarnings("unchecked")
-    public void setStatementsFieldByObjectType(Object object) {
+    public void setStatementsFieldByType(Object object) {
         if (object instanceof List) {
             List<Object> stmtList = (List<Object>) object;
             List<RawStmtDef> defs = new ArrayList<>(stmtList.size());
             for (int i = 0; i < stmtList.size(); i++) {
-                String defaultName = "stmt"+(i+1);
+                String defaultName = "stmt" + (i + 1);
                 Object o = stmtList.get(i);
                 if (o instanceof String) {
-                    defs.add(new RawStmtDef(defaultName,(String)o));
+                    defs.add(new RawStmtDef(defaultName, (String) o));
                 } else if (o instanceof Map) {
-                    defs.add(new RawStmtDef(defaultName,(Map<String,Object>)o));
+                    defs.add(new RawStmtDef(defaultName, (Map<String, Object>) o));
                 } else {
                     throw new RuntimeException("Can not construct stmt def from object type:" + o.getClass());
                 }
@@ -76,12 +77,16 @@ public class StatementsOwner extends RawStmtFields {
             List<Map<String,Object>> itemizedMaps = new ArrayList<>();
             for (Map.Entry<String, Object> entries : map.entrySet()) {
                 Object value = entries.getValue();
-                if (value instanceof Map) {
+                if (value instanceof LinkedHashMap) {
-                    Map<String,Object> valueMap = ((Map<String,Object>)value);
+                    // reset order to favor naming first
-                    valueMap.put("name", entries.getKey());
+                    LinkedHashMap<String, Object> vmap = (LinkedHashMap<String, Object>) value;
-                    itemizedMaps.add(valueMap);
+                    LinkedHashMap<String, Object> cp = new LinkedHashMap<>(vmap);
+                    vmap.clear();
+                    vmap.put("name", entries.getKey());
+                    vmap.putAll(cp);
+                    itemizedMaps.add(vmap);
                 } else if (value instanceof String) {
-                    Map<String,Object> stmtDetails = new HashMap<>() {{
+                    Map<String, Object> stmtDetails = new HashMap<>() {{
                         put("name", entries.getKey());
                         put("stmt", entries.getValue());
                     }};
@@ -90,9 +95,9 @@ public class StatementsOwner extends RawStmtFields {
                     throw new RuntimeException("Unknown inner value type on map-based statement definition.");
                 }
             }
-            setStatementsFieldByObjectType(itemizedMaps);
+            setStatementsFieldByType(itemizedMaps);
         } else if (object instanceof String) {
-            setStatementsFieldByObjectType(Map.of("stmt1",(String)object));
+            setStatementsFieldByType(Map.of("stmt1", (String) object));
         } else {
             throw new RuntimeException("Unknown object type: " + object.getClass());
         }

engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java

@@ -6,6 +6,120 @@ import io.nosqlbench.engine.api.util.Tagged;
 import java.util.Map;
 import java.util.Optional;
 
+/**
+ * <p>The OpTemplate is the developer's view of the operational templates that users
+ * provide in YAML or some other structured format.</p>
+ *
+ * <H2>Terms</H2>
+ * Within this documentation, the word <i>OpTemplate</i> will refer to the template API and
+ * semantics. The word <i>user template</i> will refer to the configuration data as provided
+ * by a user.
+ *
+ * <p>OpTemplates are the native Java representation of the user templates that specify how to
+ * make an executable operation. OpTemplates are not created for each operation, but are used
+ * to create an mostly-baked intermediate form commonly known as a <i>ready op</i>.
+ * It is the intermediate form which is used to create an instance of an executable
+ * op in whichever way is the most appropriate and efficient for a given driver.</p>
+ *
+ * <p>This class serves as the canonical documentation and API for how user templates
+ * are mapped into a fully resolved OpTemplate. User-provided op templates can be
+ * any basic data structure, and are often provided  as part of a YAML workload file.
+ * The description below will focus on structural rules rather than any particular
+ * encoding format. The types used are fairly universal and easy to map from one
+ * format to another.</p>
+ *
+ *
+ * <p>A long-form introduction to this format is included in the main NoSQLBench docs
+ * at <a href="http://docs.nosqlbench.io">docs.nosqlbench.io</a>
+ * under the <I>Designing Workloads</I> section.</p>
+ *
+ * <p>A few structural variations are allowed -- No specific form enforced. The reasons for this are:
+ * 1) It is generally obvious what as user wants to do from a given layout. 2) Data structure
+ * markup is generally frustrating and difficult to troubleshoot. 3) The conceptual domain of
+ * NB op construction is well-defined enough to avoid ambiguity.</p>
+ *
+ * <H2>Type Conventions</H2>
+ *
+ * For the purposes of simple interoperability, the types used at this interface boundary should
+ * be limited to common scalar types -- numbers and strings, and simple structures like maps and lists.
+ * The basic types defined for ECMAScript should eventually be supported, but no domain-specific
+ * objects which would require special encoding or decoding rules should be used.
+ *
+ * <H2>Standard Properties</H2>
+ *
+ * Each op template can have these standard properties:
+ * <UL>
+ * <LI>name - every op template has a name, even if it is auto generated for you. This is used to
+ * name errors in the log, to name metrics in telemetry, and so on.</LI>
+ * <LI>description - an optional description, defaulted to "".</LI>
+ * <LI>statement - An optional string value which represents an opaque form of the body of
+ * an op template</LI>
+ * <LI>params - A string-object map of zero or more named parameters, where the key is taken as the parameter
+ * name and the value is any simple object form as limited by type conventions above.
+ * <LI>bindings - A map of binding definitions, where the string key is taken as the anchor name, and the
+ * string value is taken as the binding recipe.</LI>
+ * <LI>tags - A map of tags, with string names and values</LI>
+ * </UL>
+ *
+ * The user-provided definition of an op template should capture a blueprint of an operation to be executed by
+ * a native driver. As such, you need either a statement or a set of params which can describe what
+ * specific type should be constructed. The rules on building an executable operation are not enforced
+ * by this API. Yet, responsible NB driver developers will clearly document what the rules
+ * are for specifying each specific type of operation supported by an NB driver with examples in YAML format.
+ *
+ * <H2>OpTemplate Construction Rules</H2>
+ *
+ * <p>The available structural forms follow a basic set of rules for constructing the OpTemplate in a consistent way.
+ * <OL>
+ * <LI>A collection of user-provided op templates is provided as a string, a list or a map.</LI>
+ * <LI>All maps are order-preserving, like {@link java.util.LinkedHashMap}</LI>
+ * <LI>For maps, the keys are taken as the names of the op template instances.</LI>
+ * <LI>The content of each op template can be provided as a string or as a map.</LI>
+ * <OL>
+ * <LI>If the op template entry is provided as a string, then the OpTemplate is constructed as having only a single
+ * <i>statement</i> property (in addition to defaults within scope).
+ * as provided by OpTemplate API.</LI>
+ * <LI>If the op template entry is provided as a map, then the OpTemplate is constructed as having all of the
+ * named properties defined in the standard properties above.
+ * Any entry in the template which is not a reserved word is assigned to the params map as a parameter, in whatever structured
+ * type is appropriate (scalar, lists, maps).</LI>
+ * </LI>
+ * </p>
+ * </OL>
+ *
+ * <H2>Example Forms</H2>
+ * The valid forms are shown below as examples.
+ *
+ * <H3>One String Statement</H3>
+ * <pre>{@code
+ * statement: statement
+ * }</pre>
+ *
+ * <H3>List of Templates</H3>
+ * <pre>{@code
+ * statements:
+ *   - statement1
+ *   - statement2
+ * }</pre>
+ *
+ * <H3>List of Maps</H3>
+ * <pre>{@code
+ * statements:
+ *   - name: name1
+ *     stmt: statement body
+ *     params:
+ *       p1: v1
+ *       p2: v2
+ * }</pre>
+ *
+ * <H3>List Of Condensed Maps</H3>
+ * <pre>{@code
+ * statements:
+ *   - name1: statement body
+ *     p1: v1
+ *     p2: v2
+ * }</pre>
+ */
 public interface OpTemplate extends Tagged {
 
     String getName();
@@ -30,11 +144,12 @@ public interface OpTemplate extends Tagged {
 
     Optional<String> getOptionalStringParam(String name);
 
-    Map<String,String> getTags();
+    Map<String, String> getTags();
 
     /**
      * Parse the statement for anchors and return a richer view of the StmtDef which
      * is simpler to use for most statement configuration needs.
+     *
      * @return a new {@link ParsedStmt}
      */
     ParsedStmt getParsed();

engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java

@@ -31,9 +31,9 @@ import java.util.Map;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-public class StmtDefTest {
+public class OpDefTest {
 
-    private final static Logger logger = LogManager.getLogger(StmtDefTest.class);
+    private final static Logger logger = LogManager.getLogger(OpDefTest.class);
 
     @Test
     public void testLayering() {
@@ -71,7 +71,7 @@ public class StmtDefTest {
         StmtsDoc doc1 = all.getStmtDocs().get(0);
         StmtsBlock block1 = doc1.getBlocks().get(0);
         assertThat(block1.getName()).isEqualTo("doc1--block0");
-        List<OpTemplate> assys = block1.getStmts();
+        List<OpTemplate> assys = block1.getOps();
         assertThat(assys).hasSize(2);
         OpTemplate sdef1 = assys.get(0);
         assertThat(sdef1.getName()).isEqualTo("doc1--block0--stmt1");
@@ -93,4 +93,56 @@ public class StmtDefTest {
         assertThat(stmt1.getParams()).containsAllEntriesOf(Map.of("timeout", 23423, "foobar", "baz"));
     }
 
+    @Test
+    public void testMapOfMaps() {
+        StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
+        List<StmtsDoc> docs = all.getStmtDocs();
+        StmtsDoc doc0 = docs.get(0);
+        assertThat(doc0.getName()).isEqualTo("map-of-maps");
+        assertThat(doc0.getBlocks()).hasSize(1);
+        StmtsBlock block1 = doc0.getBlocks().get(0);
+        assertThat(block1.getName()).isEqualTo("map-of-maps--block0");
+        assertThat(block1.getOps()).hasSize(2);
+        OpTemplate op0 = block1.getOps().get(0);
+        assertThat(op0.getName()).isEqualTo("map-of-maps--block0--s3");
+        assertThat(op0.getParams()).hasSize(2);
+        assertThat(op0.getParams()).containsAllEntriesOf(Map.of("p1", "v7", "p2", "v8"));
+        OpTemplate op1 = block1.getOps().get(1);
+        assertThat(op1.getParams()).containsAllEntriesOf(Map.of());
+        assertThat(op1.getName()).isEqualTo("map-of-maps--block0--s2");
+        assertThat(op1.getStmt()).isEqualTo("statement2");
+    }
+
+    @Test
+    public void testBasicStringStmt() {
+        StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
+        List<StmtsDoc> docs = all.getStmtDocs();
+        StmtsDoc doc1 = docs.get(1);
+        assertThat(doc1.getName()).isEqualTo("string-statement");
+        assertThat(doc1.getBlocks()).hasSize(1);
+        StmtsBlock block1 = doc1.getBlocks().get(0);
+        assertThat(block1.getName()).isEqualTo("string-statement--block0");
+        assertThat(block1.getOps()).hasSize(1);
+        OpTemplate op0 = block1.getOps().get(0);
+        assertThat(op0.getName()).isEqualTo("string-statement--block0--stmt1");
+        assertThat(op0.getStmt()).isEqualTo("test statement");
+    }
+
+    @Test
+    public void testListOfNamedMap() {
+        StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
+        List<StmtsDoc> docs = all.getStmtDocs();
+        StmtsDoc doc2 = docs.get(2);
+        assertThat(doc2.getName()).isEqualTo("list-of-named-map");
+        assertThat(doc2.getBlocks()).hasSize(1);
+        StmtsBlock block1 = doc2.getBlocks().get(0);
+        assertThat(block1.getOps()).hasSize(1);
+        OpTemplate op0 = block1.getOps().get(0);
+        assertThat(op0.getName()).isEqualTo("list-of-named-map--block0--s1");
+        assertThat(op0.getStmt()).isNull();
+        assertThat(op0.getParams()).hasSize(3);
+        assertThat(op0.getParams()).containsExactlyEntriesOf(Map.of("p1", "v2", "p2", "v2", "p3", "v3"));
+    }
+
+
 }

engine-api/src/test/resources/testdocs/docs_blocks_stmts.yaml

@@ -56,3 +56,4 @@ params:
 bindings:
   b11: b11d
   b12: b12d
+

engine-api/src/test/resources/testdocs/statement_variants.yaml

@@ -0,0 +1,16 @@
+name: map-of-maps
+statements:
+  s3:
+    p1: v7
+    p2: v8
+  s2: statement2
+---
+name: string-statement
+statement: "test statement"
+---
+name: list-of-named-map
+statements:
+  - s1:
+      p1: v1
+      p2: v2
+    p3: v3