diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java index d933991d4..011eb3608 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java @@ -45,8 +45,8 @@ public class CGDefaultCqlBindings implements BindingsLibrary { ", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE) ); - OpsDocList stmtsDocs = OpsLoader.loadContent(content, Map.of()); - this.bindings = stmtsDocs.getDocBindings(); + OpsDocList opsDocList = OpsLoader.loadContent(content, Map.of()); + this.bindings = opsDocList.getDocBindings(); } private Optional loadLocal(String path) { diff --git a/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java b/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java index 55065bdb0..fb4baa286 100644 --- a/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java +++ b/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java @@ -103,6 +103,6 @@ public class DiagDriverAdapter extends BaseDriverAdapter impl @Override public List getSyntheticOpTemplates(OpsDocList opsDocList, Map params) { - return OpsLoader.loadString("log:level=INFO", OpTemplateFormat.inline, params,null).getStmts(); + return OpsLoader.loadString("log:level=INFO", OpTemplateFormat.inline, params,null).getOps(); } } diff --git a/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java b/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java index 42b9036d5..a5129d3ca 100644 --- a/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java +++ b/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java @@ -53,8 +53,8 @@ public class HttpOpMapperTest { private static ParsedOp parsedOpFor(String yaml) { OpsDocList docs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null); - OpTemplate stmtDef = docs.getStmts().get(0); - ParsedOp parsedOp = new ParsedOp(stmtDef, cfg, List.of(adapter.getPreprocessor())); + OpTemplate opTemplate = docs.getOps().get(0); + ParsedOp parsedOp = new ParsedOp(opTemplate, cfg, List.of(adapter.getPreprocessor())); return parsedOp; } @@ -102,7 +102,7 @@ public class HttpOpMapperTest { @Test public void testRFCFormBody() { ParsedOp pop = parsedOpFor(""" - statements: + ops: - s1: | get http://localhost/ @@ -118,7 +118,7 @@ public class HttpOpMapperTest { // This can not be fully resolved in the unit testing context, but it could be // in the integrated testing context. It is sufficient to verify parsing here. ParsedOp pop = parsedOpFor(""" - statements: + ops: - s1: | {method} {scheme}://{host}/{path}?{query} {version} Header1: {header1val} diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpsOwner.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpsOwner.java index db817449f..420a57a67 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpsOwner.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpsOwner.java @@ -22,14 +22,14 @@ import java.util.*; public class OpsOwner extends RawOpFields { - private final static List stmtsFieldNames = List.of("op","ops","operation","statement","statements"); + private final static List opsFieldNames = List.of("op","ops","operation","statement","statements"); private List rawOpDefs = new ArrayList<>(); public OpsOwner() { } - public List getRawStmtDefs() { + public List getRawOpDefs() { return rawOpDefs; } @@ -43,7 +43,7 @@ public class OpsOwner extends RawOpFields { super.setFieldsByReflection(propsmap); HashSet found = new HashSet<>(); - for (String fname : stmtsFieldNames) { + for (String fname : opsFieldNames) { if (propsmap.containsKey(fname)) { found.add(fname); } @@ -52,15 +52,15 @@ public class OpsOwner extends RawOpFields { throw new BasicError("You used " + found + " as an op name, but only one of these is allowed."); } if (found.size()==1) { - Object stmtsFieldValue = propsmap.remove(found.iterator().next()); - setStatementsFieldByType(stmtsFieldValue); + Object opsFieldValue = propsmap.remove(found.iterator().next()); + setOpsFieldByType(opsFieldValue); } super.setFieldsByReflection(propsmap); } @SuppressWarnings("unchecked") - public void setStatementsFieldByType(Object object) { + public void setOpsFieldByType(Object object) { if (object instanceof List) { List stmtList = (List) object; List defs = new ArrayList<>(stmtList.size()); @@ -105,9 +105,9 @@ public class OpsOwner extends RawOpFields { entries.getValue().getClass() + "', only maps and strings are recognized."); } } - setStatementsFieldByType(itemizedMaps); + setOpsFieldByType(itemizedMaps); } else if (object instanceof String) { - setStatementsFieldByType(Map.of("stmt1", (String) object)); + setOpsFieldByType(Map.of("stmt1", (String) object)); } else { throw new RuntimeException("Unknown object type: " + object.getClass()); } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDoc.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDoc.java index f0c235a31..ec21a7008 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDoc.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDoc.java @@ -22,15 +22,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -/** - * A statements doc can have both a list of statement blocks and/or a - * list of statements. It can also have all the block parameters - * assignable to {@link RawOpFields}. - *

- * The reason for having support both statements or statement blocks - * is merely convenience. If you do not need or want to deal with the - * full blocks format, the extra structure gets in the way. - */ public class RawOpsDoc extends OpsOwner { private RawScenarios scenarios = new RawScenarios(); @@ -43,7 +34,7 @@ public class RawOpsDoc extends OpsOwner { public static RawOpsDoc forSingleStatement(String statement) { RawOpsDoc rawOpsDoc = new RawOpsDoc(); - rawOpsDoc.setStatementsFieldByType(statement); + rawOpsDoc.setOpsFieldByType(statement); return rawOpsDoc; } @@ -100,19 +91,17 @@ public class RawOpsDoc extends OpsOwner { } /** - * Return the list of statement blocks in this RawStmtsDoc. - * If raw statements are defined on this RawStmtsDoc, then a single - * StmtBlock containing those statements is prepended to the block list. - * Otherwise, the list of StmtBlocks is returned as-is. + * If raw ops are defined on this RawOpsDoc, then a single RawOpsBlock containing those op templates is prepended to + * the block list. Otherwise, the list of RawOpsBlocks is returned as-is. * - * @return all logical statement blocks containing statements + * @return all logical ops blocks */ public List getBlocks() { List stmtBlocks = new ArrayList<>(); - if (!getRawStmtDefs().isEmpty()) { + if (!getRawOpDefs().isEmpty()) { RawOpsBlock rawOpsBlock = new RawOpsBlock(); rawOpsBlock.setName("block0"); - rawOpsBlock.setRawStmtDefs(getRawStmtDefs()); + rawOpsBlock.setRawStmtDefs(getRawOpDefs()); stmtBlocks.add(rawOpsBlock); } stmtBlocks.addAll(this.blocks); diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDocList.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDocList.java index 0446f23c3..6cf742f77 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDocList.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDocList.java @@ -35,14 +35,14 @@ public class RawOpsDocList { return new RawOpsDocList(List.of()); } - public List getStmtsDocs() { + public List getOpsDocs() { return rawOpsDocList; } public String toString() { int docs = rawOpsDocList.size(); int blocks = rawOpsDocList.stream().map(RawOpsDoc::getBlocks).mapToInt(List::size).sum(); - long optemplates = rawOpsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawStmtDefs().stream()).count(); + long optemplates = rawOpsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawOpDefs().stream()).count(); return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates; } } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java index 75e722b21..0a6f6dc76 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java @@ -95,7 +95,7 @@ public class RawOpsLoader { newDocList.add(doc); } else { - throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName()); + throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName()); } } RawOpsDocList rawOpsDocList = new RawOpsDocList(newDocList); diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java index 5358c6171..d5a3d3b72 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java @@ -84,7 +84,7 @@ public class RawYamlLoader { if (object instanceof Map) { maps.add(new LinkedHashMap<>((Map)object)); } else { - throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName()); + throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName()); } } return maps; diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java index 70eb37d5a..9c3bfcedf 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,118 +32,17 @@ import java.util.Optional; import java.util.function.Function; /** - *

The OpTemplate is the developer's view of the operational templates that users - * provide in YAML or some other structured format.

- * - *

Terms

- * Within this documentation, the word OpTemplate will refer to the template API and - * semantics. The word user template will refer to the configuration data as provided - * by a user. - * - *

OpTemplates are the native Java representation of the user templates that specify how to - * make an executable operation. OpTemplates are not created for each operation, but are used - * to create an mostly-baked intermediate form commonly known as a ready op. - * It is the intermediate form which is used to create an instance of an executable - * op in whichever way is the most appropriate and efficient for a given driver.

- * - *

This class serves as the canonical documentation and API for how user templates - * are mapped into a fully resolved OpTemplate. User-provided op templates can be - * any basic data structure, and are often provided as part of a YAML workload file. - * The description below will focus on structural rules rather than any particular - * encoding format. The types used are fairly universal and easy to map from one - * format to another.

- * - * - *

A long-form introduction to this format is included in the main NoSQLBench docs - * at docs.nosqlbench.io - * under the Designing Workloads section.

- * - *

A few structural variations are allowed -- No specific form enforced. The reasons for this are: - * 1) It is generally obvious what as user wants to do from a given layout. 2) Data structure - * markup is generally frustrating and difficult to troubleshoot. 3) The conceptual domain of - * NB op construction is well-defined enough to avoid ambiguity.

- * - *

Type Conventions

- * - * For the purposes of simple interoperability, the types used at this interface boundary should - * be limited to common scalar types -- numbers and strings, and simple structures like maps and lists. - * The basic types defined for ECMAScript should eventually be supported, but no domain-specific - * objects which would require special encoding or decoding rules should be used. - * - *

Standard Properties

- * - * Each op template can have these standard properties: - *
    - *
  • name - every op template has a name, even if it is auto generated for you. This is used to - * name errors in the log, to name metrics in telemetry, and so on.
    • - *
  • description - an optional description, defaulted to "".
    • - *
  • statement - An optional string value which represents an opaque form of the body of - * an op template
    • - *
  • params - A string-object map of zero or more named parameters, where the key is taken as the parameter - * name and the value is any simple object form as limited by type conventions above. - *
  • bindings - A map of binding definitions, where the string key is taken as the anchor name, and the - * string value is taken as the binding recipe.
    • - *
  • tags - A map of tags, with string names and values
    • - *
- * - * The user-provided definition of an op template should capture a blueprint of an operation to be executed by - * a native driver. As such, you need either a statement or a set of params which can describe what - * specific type should be constructed. The rules on building an executable operation are not enforced - * by this API. Yet, responsible NB driver developers will clearly document what the rules - * are for specifying each specific type of operation supported by an NB driver with examples in YAML format. - * - *

OpTemplate Construction Rules

- * - *

The available structural forms follow a basic set of rules for constructing the OpTemplate in a consistent way. - *

    - *
  1. A collection of user-provided op templates is provided as a string, a list or a map.
    2. - *
  2. All maps are order-preserving, like {@link java.util.LinkedHashMap}
    3. - *
  3. For maps, the keys are taken as the names of the op template instances.
    4. - *
  4. The content of each op template can be provided as a string or as a map.
    5. - *
      - *
    1. If the op template entry is provided as a string, then the OpTemplate is constructed as having only a single - * statement property (in addition to defaults within scope). - * as provided by OpTemplate API.
      2. - *
    2. If the op template entry is provided as a map, then the OpTemplate is constructed as having all of the - * named properties defined in the standard properties above. - * Any entry in the template which is not a reserved word is assigned to the params map as a parameter, in whatever structured - * type is appropriate (scalar, lists, maps).
      3. - * + *

      + * The OpTemplate is a structurally normalized type which presents the user-provided op template to the NoSQLBench + * loading and templating mechanisms. This type is not generally used directly for new driver development. It is the + * backing data which is used by {@link io.nosqlbench.engine.api.templating.ParsedOp}, which is used in drivers to map + * op templates to function to be used for a given cycle value. *

      - *
    * - *

    Example Forms

    - * The valid forms are shown below as examples. - * - *

    One String Statement

    - * 
    {@code
- * statement: statement
- * }
    - * - *

    List of Templates

    - * 
    {@code
- * statements:
- *   - statement1
- *   - statement2
- * }
    - * - *

    List of Maps

    - * 
    {@code
- * statements:
- *   - name: name1
- *     stmt: statement body
- *     params:
- *       p1: v1
- *       p2: v2
- * }
    - * - *

    List Of Condensed Maps

    - * 
    {@code
- * statements:
- *   - name1: statement body
- *     p1: v1
- *     p2: v2
- * }
    + *

    + * This is part of the implementation of the NoSQLBench Uniform Workload Specification. Check the tests + * for UniformWorkloadSpecification directly to see how this specification is tested and documented. + *

    */ public abstract class OpTemplate implements Tagged { @@ -170,6 +69,7 @@ public abstract class OpTemplate implements Tagged { /** * Return a map of tags for this statement. Implementations are required to * add a tag for "name" automatically when this value is set during construction. + * * @return A map of assigned tags for the op, with the name added as an auto-tag. */ public abstract Map getTags(); @@ -208,7 +108,7 @@ public abstract class OpTemplate implements Tagged { if (defaultValue.getClass().isAssignableFrom(value.getClass())) { return (V) value; } else { - return NBTypeConverter.convertOr(value,defaultValue); + return NBTypeConverter.convertOr(value, defaultValue); } } @@ -229,7 +129,6 @@ public abstract class OpTemplate implements Tagged { } - public V getParam(String name, Class type) { Object object = getParams().get(name); if (object == null) { @@ -273,7 +172,7 @@ public abstract class OpTemplate implements Tagged { * * @return an optional {@link ParsedStringTemplate} */ - public Optional getParsed(Function... rewriters) { + public Optional getParsed(Function... rewriters) { Optional os = getStmt(); return os.map(s -> { String result = s; @@ -281,7 +180,7 @@ public abstract class OpTemplate implements Tagged { result = rewriter.apply(result); } return result; - }).map(s -> new ParsedStringTemplate(s,getBindings())); + }).map(s -> new ParsedStringTemplate(s, getBindings())); } public Optional getParsed() { @@ -309,7 +208,7 @@ public abstract class OpTemplate implements Tagged { fields.put(FIELD_TAGS, this.getTags()); } - this.getOp().ifPresent(o -> fields.put(FIELD_OP,o)); + this.getOp().ifPresent(o -> fields.put(FIELD_OP, o)); fields.put(FIELD_NAME, this.getName()); @@ -320,10 +219,11 @@ public abstract class OpTemplate implements Tagged { * Legacy support for String form statements. This is left here as a convenience method, * however it is changed to an Optional to force caller refactorings. * - * @return An optional string version of the op, empty if there is no 'stmt' property in the op fields, or no op fields at all. + * @return An optional string version of the op, empty if there is no 'stmt' property in the op fields, or no op + * fields at all. */ public Optional getStmt() { - return getOp().map(m->m.get("stmt")).map(s->{ + return getOp().map(m -> m.get("stmt")).map(s -> { if (s instanceof CharSequence) { return s.toString(); } else { @@ -333,7 +233,7 @@ public abstract class OpTemplate implements Tagged { } public Element getParamReader() { - return NBParams.one(getName(),getParams()); + return NBParams.one(getName(), getParams()); } /** @@ -347,13 +247,13 @@ public abstract class OpTemplate implements Tagged { * @return the map of all remaining fields from the op template and the params map. */ public Map remainingFields() { - Map remaining = new LinkedHashMap<>(getOp().orElse(Map.of())); + Map remaining = new LinkedHashMap<>(getOp().orElse(Map.of())); remaining.putAll(getParams()); return remaining; } public void assertConsumed() { - if (size()>0) { + if (size() > 0) { throw new OpConfigError("The op template named '" + getName() + "' was not fully consumed. These fields are not being applied:" + remainingFields()); } } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsBlock.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsBlock.java index 1f748760f..6143d6a5f 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsBlock.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsBlock.java @@ -40,11 +40,11 @@ public class OpsBlock implements Tagged, Iterable { public List getOps() { List rawOpTemplates = new ArrayList<>(); - List statements = rawOpsBlock.getRawStmtDefs(); + List opDefs = rawOpsBlock.getRawOpDefs(); - for (int i = 0; i < statements.size(); i++) { + for (int i = 0; i < opDefs.size(); i++) { rawOpTemplates.add( - new OpDef(this, statements.get(i)) + new OpDef(this, opDefs.get(i)) ); } return rawOpTemplates; diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDoc.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDoc.java index a09e735a5..4b0d00b2f 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDoc.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDoc.java @@ -27,7 +27,7 @@ import java.util.Map; import java.util.stream.Collectors; /** - * StmtsDoc creates a logical view of a statements doc that includes + * OpsDoc creates a logical view of a RawOpsDoc doc that includes * all inherited and overridden values for bindings, tags, and params. */ public class OpsDoc implements Tagged, Iterable { @@ -86,10 +86,10 @@ public class OpsDoc implements Tagged, Iterable { } /** - * @return The list of all included statements for all included block in this document, - * including the inherited and overridden values from the this doc and the parent block. + * @return The list of all included op templates for all included block in this document, + * including the inherited and overridden values from this doc and the parent block. */ - public List getStmts() { + public List getOpTemplates() { return getBlocks().stream().flatMap(b -> b.getOps().stream()).collect(Collectors.toList()); } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java index 68f79dd79..9c2923041 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java @@ -53,26 +53,26 @@ public class OpsDocList implements Iterable { } public List getStmtDocs() { - return rawOpsDocList.getStmtsDocs().stream() + return rawOpsDocList.getOpsDocs().stream() .map(OpsDoc::new) .collect(Collectors.toList()); } - public List getStmts() { - return getStmts(""); + public List getOps() { + return getOps(""); } /** * @param tagFilterSpec a comma-separated tag filter spec - * @return The list of all included statements for all included blocks of in this document, - * including the inherited and overridden values from the this doc and the parent block. + * @return The list of all included op templates for all included blocks of in this document, + * including the inherited and overridden values from this doc and the parent block. */ - public List getStmts(String tagFilterSpec) { + public List getOps(String tagFilterSpec) { TagFilter ts = new TagFilter(tagFilterSpec); List opTemplates = new ArrayList<>(); getStmtDocs().stream() - .flatMap(d -> d.getStmts().stream()) + .flatMap(d -> d.getOpTemplates().stream()) .filter(ts::matchesTagged) .forEach(opTemplates::add); @@ -155,8 +155,6 @@ public class OpsDocList implements Iterable { } sb.append("docs: " + docscount + " blocks:" + blockscount + " ops:" + opscount); -// String names = this.rawStmtsDocList.getStmtsDocs().stream().flatMap(sd -> sd.getRawStmtDefs().stream()).map(d->d.getName()).collect(Collectors.joining(",")); -// sb.append(", names:").append(names); return sb.toString(); } @@ -166,7 +164,7 @@ public class OpsDocList implements Iterable { }; public Pattern getVersionRegex() { - List stmtDocs = rawOpsDocList.getStmtsDocs(); + List stmtDocs = rawOpsDocList.getOpsDocs(); return Pattern.compile(stmtDocs.size()>0 ? stmtDocs.get(0).getVersionRegex() : ".*"); } } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverSpaceCache.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverSpaceCache.java index f8bc6e4f4..207792538 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverSpaceCache.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverSpaceCache.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,34 +31,24 @@ import java.util.function.Function; * a client API for advanced testing. The default instance should simply be named * {@code default}

    * - *

    Most native drivers use some combination of fluent, functional, and declarative - * patterns. These usually require you to keep access to a set of core - * state-holding objects in order to construct new elements to drive operations with. - * An example of this would be creating a statement from a session. It is necessary - * to keep the session around in for when you create new statements. Maintaining - * the session object is considered an essential part of idiomatic and efficient - * use of the API. Further, you may have builders or factories that are created - * from the session which should be cached as well. Keeping all these objects - * together requires attaching them to a cohesive owning object -- That is the space - * cache.

    + *

    Most native drivers use some combination of fluent, functional, and declarative patterns. These usually require + * you to keep access to a set of core state-holding objects in order to construct new elements to drive operations + * with. An example of this would be creating an executable operation from a session object. It is necessary to keep the + * session around in for when you create new statements. Maintaining the session object is considered an essential part + * of idiomatic and efficient use of the API. Further, you may have builders or factories that are created from the + * session which should be cached as well. Keeping all these objects together requires attaching them to a cohesive + * owning object -- That is the space cache.

    * - *

    You might want to create multiple session contexts in order to test out - * non-trivial behavior in advanced testing scenarios. To do this dynamically, i.e. - * acquire some named space cache, simply call the {@link #get(String)}

    method - * with the name of the space you want to use. This value can be provided as a - * dynamic field in your op mapping ({@link io.nosqlbench.engine.api.activityimpl.OpMapper}) - * or synthesis ({@link io.nosqlbench.engine.api.activityimpl.OpDispenser}) - * implementation. - * - * @param The type which will represent the cache for a given type of adapter. + * @param + * The type which will represent the cache for a given type of adapter. */ public class DriverSpaceCache { - private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); private final Function newSpaceFunction; - public DriverSpaceCache(Function newSpaceFunction) { + public DriverSpaceCache(Function newSpaceFunction) { this.newSpaceFunction = newSpaceFunction; } @@ -66,7 +56,7 @@ public class DriverSpaceCache { return cache.computeIfAbsent(name, newSpaceFunction); } - public Map getElements() { + public Map getElements() { return Collections.unmodifiableMap(cache); } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/decorators/SyntheticOpTemplateProvider.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/decorators/SyntheticOpTemplateProvider.java index 95f5299f6..8aab2f340 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/decorators/SyntheticOpTemplateProvider.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/decorators/SyntheticOpTemplateProvider.java @@ -24,22 +24,27 @@ import java.util.List; import java.util.Map; /** - * In the case that no statements are provided for an activity, but a workload - * description is provided, and the result for no found statements is not - * simply a matter of tag filtering, an activity may provide its own - * synthetic ops. This is here primarily to support the classic behavior of stdout - * until it's functionality is subsumed by standard diagnostic features. - * - * Note that this is only valid while an activity uses a single driver, which will - * change with upcoming API updates. + * It is possible for a DriverAdapter to create op templates using partial information. + * For example, the stdout driver can use only bindings to create CSV or JSON style data renderings. + * This mechanism is only triggered when: + *
      + *
    1. No op templates were provided.
      2. + *
    2. The default driver for an activity implements this method.
      3. + *
    + *

    + * This excludes cases where a workload was provided with op templates, but they were all filtered out. In that case, + * the user should be informed of an error. */ public interface SyntheticOpTemplateProvider extends DriverAdapterDecorators { /** * If a driver adapter supports creating example op templates from bindings, * it must implement this method to do so. - * @param opsDocList The existing doc structure, which should contain no fully defined op templates, but may contain other elements like bindings + * + * @param opsDocList + * The existing doc structure, which should contain no fully defined op templates, but may contain other + * elements like bindings * @return A list of op templates, size zero or more */ - List getSyntheticOpTemplates(OpsDocList opsDocList, Map params); + List getSyntheticOpTemplates(OpsDocList opsDocList, Map params); } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/templating/OpTemplateSupplier.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/templating/OpTemplateSupplier.java deleted file mode 100644 index 1de252158..000000000 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/templating/OpTemplateSupplier.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2022-2023 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.engine.api.templating; - -import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.api.config.standard.NBConfiguration; - -import java.util.List; -import java.util.Optional; - -/** - * An Op Template Supplier can provide its own source of op templates instead - * of relying on the built-in mechanism. By default, the built-in mechanism - * will read op definitions from parameters first, then any ops (statements) - * from yaml files provided in the workload= activity parameters. - */ -public interface OpTemplateSupplier extends DriverAdapterDecorators { - - Optional> loadOpTemplates(NBConfiguration cfg); -} diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java index 53e28d5cb..7bc4fa02f 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java @@ -29,12 +29,12 @@ public class OpsLoaderTest { @Test public void testTemplateVarSubstitution() { - OpsDocList stmtsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources"); - assertThat(stmtsDocs).isNotNull(); - List docs = stmtsDocs.getStmtDocs(); + OpsDocList opsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources"); + assertThat(opsDocs).isNotNull(); + List docs = opsDocs.getStmtDocs(); assertThat(docs).hasSize(1); - OpsDoc stmtsBlocks = docs.get(0); - Map bindings = stmtsBlocks.getBindings(); + OpsDoc opsDoc = docs.get(0); + Map bindings = opsDoc.getBindings(); assertThat(bindings).isEqualTo(Map.of( "b1a","Prefix(\"prefix\")", "b1b","Prefix(\"prefix\")", diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java index b9bc680fc..95b9e5bd1 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java @@ -37,7 +37,7 @@ public class OpDefTest { @Test public void testLayering() { - OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); + OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of()); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(2); OpsDoc doc1 = all.getStmtDocs().get(0); @@ -64,7 +64,7 @@ public class OpDefTest { @Test public void testStatementRendering() { - OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); + OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of()); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(2); OpsDoc doc1 = all.getStmtDocs().get(0); @@ -79,10 +79,10 @@ public class OpDefTest { @Test public void testConsumableMapState() { - OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); + OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of()); List docs = all.getStmtDocs(); OpsDoc block1 = docs.get(1); - List stmts = block1.getStmts(); + List stmts = block1.getOpTemplates(); OpTemplate stmt0 = stmts.get(0); OpTemplate stmt1 = stmts.get(1); assertThat(stmt0.getParams()).containsAllEntriesOf(Map.of("timeout", 23423, "foobar", "baz")); @@ -94,7 +94,7 @@ public class OpDefTest { @Test public void testMapOfMaps() { - OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); + OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of()); List docs = all.getStmtDocs(); OpsDoc doc0 = docs.get(0); assertThat(doc0.getName()).isEqualTo("map-of-maps"); @@ -115,7 +115,7 @@ public class OpDefTest { @Test public void testBasicStringStmt() { - OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); + OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of()); List docs = all.getStmtDocs(); OpsDoc doc1 = docs.get(1); assertThat(doc1.getName()).isEqualTo("string-statement"); @@ -130,7 +130,7 @@ public class OpDefTest { @Test public void testListOfNamedMap() { - OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); + OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of()); List docs = all.getStmtDocs(); OpsDoc doc2 = docs.get(2); assertThat(doc2.getName()).isEqualTo("list-of-named-map"); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpEscapingTest.java similarity index 91% rename from adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java rename to adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpEscapingTest.java index 62235c697..388fcf063 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpEscapingTest.java @@ -28,15 +28,15 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -public class StmtEscapingTest { +public class OpEscapingTest { - private final static Logger logger = LogManager.getLogger(StmtEscapingTest.class); + private final static Logger logger = LogManager.getLogger(OpEscapingTest.class); private static List defs; @BeforeAll public static void testLayering() { - OpsDocList all = OpsLoader.loadPath("testdocs/escaped_stmts.yaml", Map.of()); + OpsDocList all = OpsLoader.loadPath("testdocs/escaped_ops.yaml", Map.of()); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(1); OpsDoc doc1 = all.getStmtDocs().get(0); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtVariationTests.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpVariationTests.java similarity index 52% rename from adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtVariationTests.java rename to adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpVariationTests.java index 1cc2b679f..10cf03548 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtVariationTests.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpVariationTests.java @@ -24,89 +24,93 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; -public class StmtVariationTests { +public class OpVariationTests { - private final static Logger logger = LogManager.getLogger(StmtVariationTests.class); + private final static Logger logger = LogManager.getLogger(OpVariationTests.class); @Test - public void testListStmtsOnly() { + public void testListOpsOnly() { RawOpsLoader ysl = new RawOpsLoader(); RawOpsDocList docs = ysl.loadString( - "statements:\n" + - " - first statement\n" + - " - second statement\n" + "ops:\n" + + " - first op\n" + + " - second op\n" ); - assertThat(docs.getStmtsDocs()).hasSize(1); - RawOpsDoc doc = docs.getStmtsDocs().get(0); - assertThat(doc.getRawStmtDefs()).hasSize(2); - List stmts = doc.getRawStmtDefs(); - RawOpDef s0 = stmts.get(0); + assertThat(docs.getOpsDocs()).hasSize(1); + RawOpsDoc doc = docs.getOpsDocs().get(0); + assertThat(doc.getRawOpDefs()).hasSize(2); + List ops = doc.getRawOpDefs(); + RawOpDef s0 = ops.get(0); assertThat(s0.getName()).isEqualTo("stmt1"); - assertThat(s0.getStmt()).isEqualTo("first statement"); - RawOpDef s1 = stmts.get(1); + assertThat(s0.getStmt()).isEqualTo("first op"); + RawOpDef s1 = ops.get(1); assertThat(s1.getName()).isEqualTo("stmt2"); - assertThat(s1.getStmt()).isEqualTo("second statement"); + assertThat(s1.getStmt()).isEqualTo("second op"); } @Test public void testSingleEntryMapStmtsOnly() { RawOpsLoader ysl = new RawOpsLoader(); RawOpsDocList docs = ysl.loadString( - "statements:\n" + - " - s1: statement one\n" + - " - s2: statement two\n" + "ops:\n" + + " - s1: op one\n" + + " - s2: op two\n" ); - assertThat(docs.getStmtsDocs()).hasSize(1); - RawOpsDoc doc = docs.getStmtsDocs().get(0); - assertThat(doc.getRawStmtDefs()).hasSize(2); - List stmts = doc.getRawStmtDefs(); - assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class); - assertThat(stmts.get(0).getName()).isEqualTo("s1"); - assertThat(stmts.get(0).getStmt()).isEqualTo("statement one"); - assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class); - assertThat(stmts.get(1).getName()).isEqualTo("s2"); - assertThat(stmts.get(1).getStmt()).isEqualTo("statement two"); + assertThat(docs.getOpsDocs()).hasSize(1); + RawOpsDoc doc = docs.getOpsDocs().get(0); + assertThat(doc.getRawOpDefs()).hasSize(2); + List ops = doc.getRawOpDefs(); + assertThat(ops.get(0)).isOfAnyClassIn(RawOpDef.class); + assertThat(ops.get(0).getName()).isEqualTo("s1"); + assertThat(ops.get(0).getStmt()).isEqualTo("op one"); + assertThat(ops.get(1)).isOfAnyClassIn(RawOpDef.class); + assertThat(ops.get(1).getName()).isEqualTo("s2"); + assertThat(ops.get(1).getStmt()).isEqualTo("op two"); } @Test public void testMapStmtsOnly() { RawOpsLoader ysl = new RawOpsLoader(); RawOpsDocList docs = ysl.loadString( - "statements:\n" + - " - name: s1\n" + - " stmt: statement one\n" + - " - name: s2\n" + - " stmt: statement two\n" + "ops:\n" + + " - name: s1\n" + + " stmt: op one\n" + + " - name: s2\n" + + " stmt: op two\n" ); - assertThat(docs.getStmtsDocs()).hasSize(1); - RawOpsDoc doc = docs.getStmtsDocs().get(0); - assertThat(doc.getRawStmtDefs()).hasSize(2); - List stmts = doc.getRawStmtDefs(); + assertThat(docs.getOpsDocs()).hasSize(1); + RawOpsDoc doc = docs.getOpsDocs().get(0); + assertThat(doc.getRawOpDefs()).hasSize(2); + List stmts = doc.getRawOpDefs(); assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(0).getName()).isEqualTo("s1"); - assertThat(stmts.get(0).getStmt()).isEqualTo("statement one"); + assertThat(stmts.get(0).getStmt()).isEqualTo("op one"); assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(1).getName()).isEqualTo("s2"); - assertThat(stmts.get(1).getStmt()).isEqualTo("statement two"); + assertThat(stmts.get(1).getStmt()).isEqualTo("op two"); } + /** + * This test uses the compatible names for the sake of demonstration. Users should know that they can use op and + * statement terms interchangeably in op templates. + */ @Test public void testMixedForms() { RawOpsLoader ysl = new RawOpsLoader(); RawOpsDocList docs = ysl.loadString( "statement:\n" + - " - name: s1\n" + - " stmt: statement one\n" + - " - statement two\n" + - " - s3: statement three\n" + - " - ST4: statement four\n" + - " type: organic\n" + " - name: s1\n" + + " stmt: statement one\n" + + " - statement two\n" + + " - s3: statement three\n" + + " - ST4: statement four\n" + + " type: organic\n" ); - assertThat(docs.getStmtsDocs()).hasSize(1); - RawOpsDoc doc = docs.getStmtsDocs().get(0); - assertThat(doc.getRawStmtDefs()).hasSize(4); - List stmts = doc.getRawStmtDefs(); + assertThat(docs.getOpsDocs()).hasSize(1); + RawOpsDoc doc = docs.getOpsDocs().get(0); + assertThat(doc.getRawOpDefs()).hasSize(4); + List stmts = doc.getRawOpDefs(); assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(0).getName()).isEqualTo("s1"); assertThat(stmts.get(0).getStmt()).isEqualTo("statement one"); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlTemplateLoaderTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlTemplateLoaderTest.java index 04ae15c9b..9a226c3e3 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlTemplateLoaderTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlTemplateLoaderTest.java @@ -32,9 +32,9 @@ public class RawYamlTemplateLoaderTest { public void testLoadPropertiesBlock() { RawOpsLoader ysl = new RawOpsLoader(); RawOpsDocList rawBlockDocs = ysl.loadPath("testdocs/rawblock.yaml"); - assertThat(rawBlockDocs.getStmtsDocs()).hasSize(1); - RawOpsDoc rawBlockDoc = rawBlockDocs.getStmtsDocs().get(0); - assertThat(rawBlockDoc.getRawStmtDefs()).hasSize(1); + assertThat(rawBlockDocs.getOpsDocs()).hasSize(1); + RawOpsDoc rawBlockDoc = rawBlockDocs.getOpsDocs().get(0); + assertThat(rawBlockDoc.getRawOpDefs()).hasSize(1); assertThat(rawBlockDoc.getBindings()).hasSize(1); assertThat(rawBlockDoc.getName()).isEqualTo("name"); assertThat(rawBlockDoc.getTags()).hasSize(1); @@ -44,8 +44,8 @@ public class RawYamlTemplateLoaderTest { @Test public void testLoadFullFormat() { RawOpsLoader ysl = new RawOpsLoader(); - RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_stmts.yaml"); - List rawOpsDocs = erthing.getStmtsDocs(); + RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_ops.yaml"); + List rawOpsDocs = erthing.getOpsDocs(); assertThat(rawOpsDocs).hasSize(2); RawOpsDoc rawOpsDoc = rawOpsDocs.get(0); List blocks = rawOpsDoc.getBlocks(); @@ -58,8 +58,8 @@ public class RawYamlTemplateLoaderTest { @Test public void testLoadScenarios() { RawOpsLoader ysl = new RawOpsLoader(); - RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_stmts.yaml"); - List rawOpsDocs = erthing.getStmtsDocs(); + RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_ops.yaml"); + List rawOpsDocs = erthing.getOpsDocs(); assertThat(rawOpsDocs).hasSize(2); RawOpsDoc rawOpsDoc = rawOpsDocs.get(0); List blocks = rawOpsDoc.getBlocks(); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java index 9bbe834e8..23b1a21cc 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java @@ -109,7 +109,7 @@ public class YamlSpecValidator implements STAssemblyValidator { List> expectedList = gson.fromJson(json, type); OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null); - List stmts = stmtsDocs.getStmts(); + List stmts = stmtsDocs.getOps(); List> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList()); try { @@ -143,8 +143,6 @@ public class YamlSpecValidator implements STAssemblyValidator { private void validateYamlWithJson(String desc, String yaml, String json, STNodeReference testset, boolean debug) { System.out.format("%-40s", "- checking yaml->json"); -// StmtsDocList stmts = StatementsLoader.loadString(yaml); - try { List> docmaps = new RawYamlLoader().loadString(logger, yaml); JsonElement elem = null; diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java index 30da12cce..7b6fea183 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java @@ -32,7 +32,7 @@ public class OpDetailOverrideTest { @Test public void testStmtOverrides() { - OpsDocList doclist = OpsLoader.loadPath("testdocs/stmt_details.yaml", Map.of()); + OpsDocList doclist = OpsLoader.loadPath("testdocs/op_details.yaml", Map.of()); assertThat(doclist).isNotNull(); @@ -50,24 +50,24 @@ public class OpDetailOverrideTest { assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","block0--stmt1")); OpsBlock doc1block1 = doc1.getBlocks().get(1); - List stmts = doc1block1.getOps(); - assertThat(stmts).hasSize(4); + List ops = doc1block1.getOps(); + assertThat(ops).hasSize(4); - s = stmts.get(0); + s = ops.get(0); assertThat(s.getName()).isEqualTo("testblock1--stmt1"); assertThat(s.getStmt()).contains("astatement1"); assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--stmt1")); assertThat(s.getBindings()).hasSize(1); assertThat(s.getParams()).hasSize(1); - s = stmts.get(1); + s = ops.get(1); assertThat(s.getName()).isEqualTo("testblock1--s2name"); assertThat(s.getStmt()).contains("s2statement data"); assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s2name")); assertThat(s.getBindings()).hasSize(1); assertThat(s.getParams()).hasSize(1); - s = stmts.get(2); + s = ops.get(2); assertThat(s.getName()).isEqualTo("testblock1--s3"); assertThat(s.getStmt()).contains("statement three"); assertThat(s.getTags()).containsEntry("tname1", "tval1"); @@ -75,7 +75,7 @@ public class OpDetailOverrideTest { assertThat(s.getBindings()).hasSize(3); assertThat(s.getParams()).hasSize(2); - s = stmts.get(3); + s = ops.get(3); assertThat(s.getName()).isEqualTo("testblock1--s4"); assertThat(s.getStmt()).contains("statement 4"); assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s4")); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java index 436d95f61..aa79fbe10 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java @@ -50,7 +50,7 @@ public class OpsDocListTest { @BeforeAll public static void testLoadYaml() { - doclist = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); + doclist = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of()); } @Test @@ -76,21 +76,21 @@ public class OpsDocListTest { @Test public void testStmtInheritsBlockData() { OpsDoc doc0 = doclist.getStmtDocs().get(0); - List stmts1 = doc0.getBlocks().get(0).getOps(); - assertThat(stmts1).hasSize(2); + List ops1 = doc0.getBlocks().get(0).getOps(); + assertThat(ops1).hasSize(2); OpsBlock block0 = doc0.getBlocks().get(0); assertThat(block0.getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d")); assertThat(block0.getParams()).containsExactly(MapEntry.entry("param1","value1")); assertThat(block0.getTags()).containsExactly(MapEntry.entry("atagname","atagvalue")); - assertThat(stmts1.get(0).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d")); - assertThat(stmts1.get(0).getParams()).containsExactly(MapEntry.entry("param1","value1")); - assertThat(stmts1.get(0).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt1","block","doc1--block0")); + assertThat(ops1.get(0).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d")); + assertThat(ops1.get(0).getParams()).containsExactly(MapEntry.entry("param1","value1")); + assertThat(ops1.get(0).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt1","block","doc1--block0")); - assertThat(stmts1.get(1).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d")); - assertThat(stmts1.get(1).getParams()).containsExactly(MapEntry.entry("param1","value1")); - assertThat(stmts1.get(1).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt2","block","doc1--block0")); + assertThat(ops1.get(1).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d")); + assertThat(ops1.get(1).getParams()).containsExactly(MapEntry.entry("param1","value1")); + assertThat(ops1.get(1).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt2","block","doc1--block0")); } @@ -120,15 +120,15 @@ public class OpsDocListTest { @Test public void testStmtsGetter() { OpsDoc doc1 = doclist.getStmtDocs().get(1); - List stmts = doc1.getStmts(); + List stmts = doc1.getOpTemplates(); assertThat(stmts).hasSize(4); } @Test public void testFilteredStmts() { - List stmts = doclist.getStmts(""); + List stmts = doclist.getOps(""); assertThat(stmts).hasSize(6); - stmts = doclist.getStmts("root1:value23"); + stmts = doclist.getOps("root1:value23"); assertThat(stmts).hasSize(2); } diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java index 96838318a..5140451dc 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java @@ -70,8 +70,8 @@ public class ParsedOpTest { ps1: "param-one" """; OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null); - assertThat(stmtsDocs.getStmts().size()).isEqualTo(1); - OpTemplate opTemplate = stmtsDocs.getStmts().get(0); + assertThat(stmtsDocs.getOps().size()).isEqualTo(1); + OpTemplate opTemplate = stmtsDocs.getOps().get(0); ParsedOp parsedOp = new ParsedOp(opTemplate, cfg); assertThat(parsedOp.getAsFunctionOr("d1","invalid").apply(1L)).isEqualTo("one"); diff --git a/adapters-api/src/test/resources/testdocs/docs_blocks_stmts.yaml b/adapters-api/src/test/resources/testdocs/docs_blocks_ops.yaml similarity index 100% rename from adapters-api/src/test/resources/testdocs/docs_blocks_stmts.yaml rename to adapters-api/src/test/resources/testdocs/docs_blocks_ops.yaml diff --git a/adapters-api/src/test/resources/testdocs/escaped_stmts.yaml b/adapters-api/src/test/resources/testdocs/escaped_ops.yaml similarity index 100% rename from adapters-api/src/test/resources/testdocs/escaped_stmts.yaml rename to adapters-api/src/test/resources/testdocs/escaped_ops.yaml diff --git a/adapters-api/src/test/resources/testdocs/stmt_details.yaml b/adapters-api/src/test/resources/testdocs/op_details.yaml similarity index 100% rename from adapters-api/src/test/resources/testdocs/stmt_details.yaml rename to adapters-api/src/test/resources/testdocs/op_details.yaml diff --git a/adapters-api/src/test/resources/testdocs/statement_variants.yaml b/adapters-api/src/test/resources/testdocs/op_variants.yaml similarity index 100% rename from adapters-api/src/test/resources/testdocs/statement_variants.yaml rename to adapters-api/src/test/resources/testdocs/op_variants.yaml diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java index 9c38f5fab..a05987550 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java @@ -551,12 +551,12 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs OpsDocList opsDocList = loadStmtsDocList(); - List unfilteredOps = opsDocList.getStmts(); - List filteredOps = opsDocList.getStmts(tagfilter); + List unfilteredOps = opsDocList.getOps(); + List filteredOps = opsDocList.getOps(tagfilter); if (filteredOps.size() == 0) { if (unfilteredOps.size() > 0) { // There were no ops, and it was because they were all filtered out - throw new BasicError("There were no active statements with tag filter '" + throw new BasicError("There were no active op templates with tag filter '" + tagfilter + "', since all " + unfilteredOps.size() + " were filtered out."); } else { // There were no ops, and it *wasn't* because they were all filtered out. @@ -578,7 +578,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs } } if (filteredOps.size() == 0) { - throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'"); + throw new BasicError("There were no active op templates with tag filter '" + tagfilter + "'"); } } @@ -613,8 +613,8 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs * taken as the only provided statement. *

  5. If a 'yaml, or 'workload' parameter is provided, then the statements in that file * are taken with their ratios
    6. - *
  6. Any provided tags filter is used to select only the statements which have matching - * tags. If no tags are provided, then all the found statements are included.
    7. + *
  7. Any provided tags filter is used to select only the op templates which have matching + * tags. If no tags are provided, then all the found op templates are included.
    8. *
  8. The ratios and the 'seq' parameter are used to build a sequence of the ready operations, * where the sequence length is the sum of the ratios.
    9. *
diff --git a/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java b/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java index 1644a0d37..29b49ebfa 100644 --- a/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java +++ b/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java @@ -35,11 +35,11 @@ public class CommandTemplateTest { @Test public void testCommandTemplate() { - OpsDocList stmtsDocs = OpsLoader.loadString("" + - "statements:\n" + + OpsDocList opsDocs = OpsLoader.loadString("" + + "ops:\n" + " - s1: test1=foo test2=bar", OpTemplateFormat.yaml, Map.of(), null); - OpTemplate optpl = stmtsDocs.getStmts().get(0); + OpTemplate optpl = opsDocs.getOps().get(0); CommandTemplate ct = new CommandTemplate(optpl); assertThat(ct.isStatic()).isTrue(); } @@ -48,13 +48,13 @@ public class CommandTemplateTest { public void testCommandTemplateFormat() { Gson gson = new GsonBuilder().setPrettyPrinting().create(); OpsDocList stmtsDocs = OpsLoader.loadString("" + - "statements:\n" + + "ops:\n" + " - s1: test1=foo test2={bar}\n" + " bindings:\n" + " bar: NumberNameToString();\n", OpTemplateFormat.yaml, Map.of(), null ); - OpTemplate optpl = stmtsDocs.getStmts().get(0); + OpTemplate optpl = stmtsDocs.getOps().get(0); CommandTemplate ct = new CommandTemplate(optpl); String format = gson.toJson(ct); logger.debug(format); diff --git a/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java index 9b89c8bef..cd3bb627b 100644 --- a/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java +++ b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java @@ -34,12 +34,12 @@ public class NBCLIScenarioParserTemplateVarTest { cmds.forEach(System.out::println); OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams()); - OpTemplate optpl1 = workload1.getStmts().get(0); + OpTemplate optpl1 = workload1.getOps().get(0); System.out.println("op from cmd1:"+optpl1); assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n"); OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArg("workload"),cmds.get(1).getParams()); - OpTemplate optpl2 = workload2.getStmts().get(0); + OpTemplate optpl2 = workload2.getOps().get(0); System.out.println("op from cmd2:"+optpl2); assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n"); } @@ -51,7 +51,7 @@ public class NBCLIScenarioParserTemplateVarTest { cmds.forEach(System.out::println); OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams()); - OpTemplate optpl1 = workload1.getStmts().get(0); + OpTemplate optpl1 = workload1.getOps().get(0); System.out.println("op from cmd1:"+optpl1); assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n"); } diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/services/WorkSpace.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/services/WorkSpace.java index a50bc4d00..705579a55 100644 --- a/engine-rest/src/main/java/io/nosqlbench/engine/rest/services/WorkSpace.java +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/services/WorkSpace.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ package io.nosqlbench.engine.rest.services; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader; import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser; import io.nosqlbench.engine.api.scenarios.WorkloadDesc; import io.nosqlbench.engine.rest.transfertypes.WorkspaceItemView; diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 23260e6b2..86b3f7919 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -576,7 +576,7 @@ prepare-agent - ${build.directory}/jacoco-data.exec + ${project.build.directory}/jacoco-data.exec @@ -586,8 +586,8 @@ report - ${build.directory}/jacoco-data.exec - ${build.directory}/jacoco-reports + ${project.build.directory}/jacoco-data.exec + ${project.build.directory}/jacoco-reports diff --git a/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java b/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java index d542da91f..1ab2d6ebc 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java +++ b/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,7 @@ public class ActivityDef implements NBNamedElement { private final static Logger logger = LogManager.getLogger(ActivityDef.class); // an alias with which to control the activity while it is running private static final String FIELD_ALIAS = "alias"; - // a file or URL containing the activity: statements, generator bindings, ... + // a file or URL containing the activity: op templates, generator bindings, ... private static final String FIELD_ATYPE = "type"; // cycles for this activity in either "M" or "N..M" form. "M" form implies "0..M" private static final String FIELD_CYCLES = "cycles"; diff --git a/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/BindType.java b/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/BindType.java index e25d1468d..5204f2985 100644 --- a/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/BindType.java +++ b/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/BindType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ public enum BindType { /** * A literal template is one which has no bindings that need to be provided to render a specific statement. - * These templates are basically static statements. + * These templates are basically static values. * Example: {@code truncate testks.testtable;} */ literal,