code cleanup and naming updates from "statements" to "ops"

This commit is contained in:
Jonathan Shook 2023-01-29 01:26:19 -06:00
parent 5dfa55bfe4
commit ba2944ce77
35 changed files with 208 additions and 358 deletions

View File

@ -45,8 +45,8 @@ public class CGDefaultCqlBindings implements BindingsLibrary {
", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE) ", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE)
); );
OpsDocList stmtsDocs = OpsLoader.loadContent(content, Map.of()); OpsDocList opsDocList = OpsLoader.loadContent(content, Map.of());
this.bindings = stmtsDocs.getDocBindings(); this.bindings = opsDocList.getDocBindings();
} }
private Optional<String> loadLocal(String path) { private Optional<String> loadLocal(String path) {

View File

@ -103,6 +103,6 @@ public class DiagDriverAdapter extends BaseDriverAdapter<DiagOp, DiagSpace> impl
@Override @Override
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> params) { public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> params) {
return OpsLoader.loadString("log:level=INFO", OpTemplateFormat.inline, params,null).getStmts(); return OpsLoader.loadString("log:level=INFO", OpTemplateFormat.inline, params,null).getOps();
} }
} }

View File

@ -53,8 +53,8 @@ public class HttpOpMapperTest {
private static ParsedOp parsedOpFor(String yaml) { private static ParsedOp parsedOpFor(String yaml) {
OpsDocList docs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null); OpsDocList docs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
OpTemplate stmtDef = docs.getStmts().get(0); OpTemplate opTemplate = docs.getOps().get(0);
ParsedOp parsedOp = new ParsedOp(stmtDef, cfg, List.of(adapter.getPreprocessor())); ParsedOp parsedOp = new ParsedOp(opTemplate, cfg, List.of(adapter.getPreprocessor()));
return parsedOp; return parsedOp;
} }
@ -102,7 +102,7 @@ public class HttpOpMapperTest {
@Test @Test
public void testRFCFormBody() { public void testRFCFormBody() {
ParsedOp pop = parsedOpFor(""" ParsedOp pop = parsedOpFor("""
statements: ops:
- s1: | - s1: |
get http://localhost/ get http://localhost/
@ -118,7 +118,7 @@ public class HttpOpMapperTest {
// This can not be fully resolved in the unit testing context, but it could be // This can not be fully resolved in the unit testing context, but it could be
// in the integrated testing context. It is sufficient to verify parsing here. // in the integrated testing context. It is sufficient to verify parsing here.
ParsedOp pop = parsedOpFor(""" ParsedOp pop = parsedOpFor("""
statements: ops:
- s1: | - s1: |
{method} {scheme}://{host}/{path}?{query} {version} {method} {scheme}://{host}/{path}?{query} {version}
Header1: {header1val} Header1: {header1val}

View File

@ -22,14 +22,14 @@ import java.util.*;
public class OpsOwner extends RawOpFields { public class OpsOwner extends RawOpFields {
private final static List<String> stmtsFieldNames = List.of("op","ops","operation","statement","statements"); private final static List<String> opsFieldNames = List.of("op","ops","operation","statement","statements");
private List<RawOpDef> rawOpDefs = new ArrayList<>(); private List<RawOpDef> rawOpDefs = new ArrayList<>();
public OpsOwner() { public OpsOwner() {
} }
public List<RawOpDef> getRawStmtDefs() { public List<RawOpDef> getRawOpDefs() {
return rawOpDefs; return rawOpDefs;
} }
@ -43,7 +43,7 @@ public class OpsOwner extends RawOpFields {
super.setFieldsByReflection(propsmap); super.setFieldsByReflection(propsmap);
HashSet<String> found = new HashSet<>(); HashSet<String> found = new HashSet<>();
for (String fname : stmtsFieldNames) { for (String fname : opsFieldNames) {
if (propsmap.containsKey(fname)) { if (propsmap.containsKey(fname)) {
found.add(fname); found.add(fname);
} }
@ -52,15 +52,15 @@ public class OpsOwner extends RawOpFields {
throw new BasicError("You used " + found + " as an op name, but only one of these is allowed."); throw new BasicError("You used " + found + " as an op name, but only one of these is allowed.");
} }
if (found.size()==1) { if (found.size()==1) {
Object stmtsFieldValue = propsmap.remove(found.iterator().next()); Object opsFieldValue = propsmap.remove(found.iterator().next());
setStatementsFieldByType(stmtsFieldValue); setOpsFieldByType(opsFieldValue);
} }
super.setFieldsByReflection(propsmap); super.setFieldsByReflection(propsmap);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public void setStatementsFieldByType(Object object) { public void setOpsFieldByType(Object object) {
if (object instanceof List) { if (object instanceof List) {
List<Object> stmtList = (List<Object>) object; List<Object> stmtList = (List<Object>) object;
List<RawOpDef> defs = new ArrayList<>(stmtList.size()); List<RawOpDef> defs = new ArrayList<>(stmtList.size());
@ -105,9 +105,9 @@ public class OpsOwner extends RawOpFields {
entries.getValue().getClass() + "', only maps and strings are recognized."); entries.getValue().getClass() + "', only maps and strings are recognized.");
} }
} }
setStatementsFieldByType(itemizedMaps); setOpsFieldByType(itemizedMaps);
} else if (object instanceof String) { } else if (object instanceof String) {
setStatementsFieldByType(Map.of("stmt1", (String) object)); setOpsFieldByType(Map.of("stmt1", (String) object));
} else { } else {
throw new RuntimeException("Unknown object type: " + object.getClass()); throw new RuntimeException("Unknown object type: " + object.getClass());
} }

View File

@ -22,15 +22,6 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
/**
* A statements doc can have both a list of statement blocks and/or a
* list of statements. It can also have all the block parameters
* assignable to {@link RawOpFields}.
* <p>
* The reason for having support both statements or statement blocks
* is merely convenience. If you do not need or want to deal with the
* full blocks format, the extra structure gets in the way.
*/
public class RawOpsDoc extends OpsOwner { public class RawOpsDoc extends OpsOwner {
private RawScenarios scenarios = new RawScenarios(); private RawScenarios scenarios = new RawScenarios();
@ -43,7 +34,7 @@ public class RawOpsDoc extends OpsOwner {
public static RawOpsDoc forSingleStatement(String statement) { public static RawOpsDoc forSingleStatement(String statement) {
RawOpsDoc rawOpsDoc = new RawOpsDoc(); RawOpsDoc rawOpsDoc = new RawOpsDoc();
rawOpsDoc.setStatementsFieldByType(statement); rawOpsDoc.setOpsFieldByType(statement);
return rawOpsDoc; return rawOpsDoc;
} }
@ -100,19 +91,17 @@ public class RawOpsDoc extends OpsOwner {
} }
/** /**
* Return the list of statement blocks in this RawStmtsDoc. * If raw ops are defined on this RawOpsDoc, then a single RawOpsBlock containing those op templates is prepended to
* If raw statements are defined on this RawStmtsDoc, then a single * the block list. Otherwise, the list of RawOpsBlocks is returned as-is.
* StmtBlock containing those statements is prepended to the block list.
* Otherwise, the list of StmtBlocks is returned as-is.
* *
* @return all logical statement blocks containing statements * @return all logical ops blocks
*/ */
public List<RawOpsBlock> getBlocks() { public List<RawOpsBlock> getBlocks() {
List<RawOpsBlock> stmtBlocks = new ArrayList<>(); List<RawOpsBlock> stmtBlocks = new ArrayList<>();
if (!getRawStmtDefs().isEmpty()) { if (!getRawOpDefs().isEmpty()) {
RawOpsBlock rawOpsBlock = new RawOpsBlock(); RawOpsBlock rawOpsBlock = new RawOpsBlock();
rawOpsBlock.setName("block0"); rawOpsBlock.setName("block0");
rawOpsBlock.setRawStmtDefs(getRawStmtDefs()); rawOpsBlock.setRawStmtDefs(getRawOpDefs());
stmtBlocks.add(rawOpsBlock); stmtBlocks.add(rawOpsBlock);
} }
stmtBlocks.addAll(this.blocks); stmtBlocks.addAll(this.blocks);

View File

@ -35,14 +35,14 @@ public class RawOpsDocList {
return new RawOpsDocList(List.of()); return new RawOpsDocList(List.of());
} }
public List<RawOpsDoc> getStmtsDocs() { public List<RawOpsDoc> getOpsDocs() {
return rawOpsDocList; return rawOpsDocList;
} }
public String toString() { public String toString() {
int docs = rawOpsDocList.size(); int docs = rawOpsDocList.size();
int blocks = rawOpsDocList.stream().map(RawOpsDoc::getBlocks).mapToInt(List::size).sum(); int blocks = rawOpsDocList.stream().map(RawOpsDoc::getBlocks).mapToInt(List::size).sum();
long optemplates = rawOpsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawStmtDefs().stream()).count(); long optemplates = rawOpsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawOpDefs().stream()).count();
return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates; return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates;
} }
} }

View File

@ -95,7 +95,7 @@ public class RawOpsLoader {
newDocList.add(doc); newDocList.add(doc);
} else { } else {
throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName()); throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
} }
} }
RawOpsDocList rawOpsDocList = new RawOpsDocList(newDocList); RawOpsDocList rawOpsDocList = new RawOpsDocList(newDocList);

View File

@ -84,7 +84,7 @@ public class RawYamlLoader {
if (object instanceof Map) { if (object instanceof Map) {
maps.add(new LinkedHashMap<>((Map<String,Object>)object)); maps.add(new LinkedHashMap<>((Map<String,Object>)object));
} else { } else {
throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName()); throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
} }
} }
return maps; return maps;

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -32,118 +32,17 @@ import java.util.Optional;
import java.util.function.Function; import java.util.function.Function;
/** /**
* <p>The OpTemplate is the developer's view of the operational templates that users * <p>
* provide in YAML or some other structured format.</p> * The OpTemplate is a structurally normalized type which presents the user-provided op template to the NoSQLBench
* * loading and templating mechanisms. This type is not generally used directly for new driver development. It is the
* <H2>Terms</H2> * backing data which is used by {@link io.nosqlbench.engine.api.templating.ParsedOp}, which is used in drivers to map
* Within this documentation, the word <i>OpTemplate</i> will refer to the template API and * op templates to function to be used for a given cycle value.
* semantics. The word <i>user template</i> will refer to the configuration data as provided
* by a user.
*
* <p>OpTemplates are the native Java representation of the user templates that specify how to
* make an executable operation. OpTemplates are not created for each operation, but are used
* to create an mostly-baked intermediate form commonly known as a <i>ready op</i>.
* It is the intermediate form which is used to create an instance of an executable
* op in whichever way is the most appropriate and efficient for a given driver.</p>
*
* <p>This class serves as the canonical documentation and API for how user templates
* are mapped into a fully resolved OpTemplate. User-provided op templates can be
* any basic data structure, and are often provided as part of a YAML workload file.
* The description below will focus on structural rules rather than any particular
* encoding format. The types used are fairly universal and easy to map from one
* format to another.</p>
*
*
* <p>A long-form introduction to this format is included in the main NoSQLBench docs
* at <a href="http://docs.nosqlbench.io">docs.nosqlbench.io</a>
* under the <I>Designing Workloads</I> section.</p>
*
* <p>A few structural variations are allowed -- No specific form enforced. The reasons for this are:
* 1) It is generally obvious what as user wants to do from a given layout. 2) Data structure
* markup is generally frustrating and difficult to troubleshoot. 3) The conceptual domain of
* NB op construction is well-defined enough to avoid ambiguity.</p>
*
* <H2>Type Conventions</H2>
*
* For the purposes of simple interoperability, the types used at this interface boundary should
* be limited to common scalar types -- numbers and strings, and simple structures like maps and lists.
* The basic types defined for ECMAScript should eventually be supported, but no domain-specific
* objects which would require special encoding or decoding rules should be used.
*
* <H2>Standard Properties</H2>
*
* Each op template can have these standard properties:
* <UL>
* <LI>name - every op template has a name, even if it is auto generated for you. This is used to
* name errors in the log, to name metrics in telemetry, and so on.</LI>
* <LI>description - an optional description, defaulted to "".</LI>
* <LI>statement - An optional string value which represents an opaque form of the body of
* an op template</LI>
* <LI>params - A string-object map of zero or more named parameters, where the key is taken as the parameter
* name and the value is any simple object form as limited by type conventions above.
* <LI>bindings - A map of binding definitions, where the string key is taken as the anchor name, and the
* string value is taken as the binding recipe.</LI>
* <LI>tags - A map of tags, with string names and values</LI>
* </UL>
*
* The user-provided definition of an op template should capture a blueprint of an operation to be executed by
* a native driver. As such, you need either a statement or a set of params which can describe what
* specific type should be constructed. The rules on building an executable operation are not enforced
* by this API. Yet, responsible NB driver developers will clearly document what the rules
* are for specifying each specific type of operation supported by an NB driver with examples in YAML format.
*
* <H2>OpTemplate Construction Rules</H2>
*
* <p>The available structural forms follow a basic set of rules for constructing the OpTemplate in a consistent way.
* <OL>
* <LI>A collection of user-provided op templates is provided as a string, a list or a map.</LI>
* <LI>All maps are order-preserving, like {@link java.util.LinkedHashMap}</LI>
* <LI>For maps, the keys are taken as the names of the op template instances.</LI>
* <LI>The content of each op template can be provided as a string or as a map.</LI>
* <OL>
* <LI>If the op template entry is provided as a string, then the OpTemplate is constructed as having only a single
* <i>statement</i> property (in addition to defaults within scope).
* as provided by OpTemplate API.</LI>
* <LI>If the op template entry is provided as a map, then the OpTemplate is constructed as having all of the
* named properties defined in the standard properties above.
* Any entry in the template which is not a reserved word is assigned to the params map as a parameter, in whatever structured
* type is appropriate (scalar, lists, maps).</LI>
* </LI>
* </p> * </p>
* </OL>
* *
* <H2>Example Forms</H2> * <p>
* The valid forms are shown below as examples. * This is part of the implementation of the NoSQLBench <em>Uniform Workload Specification</em>. Check the tests
* * for UniformWorkloadSpecification directly to see how this specification is tested and documented.
* <H3>One String Statement</H3> * </p>
* <pre>{@code
* statement: statement
* }</pre>
*
* <H3>List of Templates</H3>
* <pre>{@code
* statements:
* - statement1
* - statement2
* }</pre>
*
* <H3>List of Maps</H3>
* <pre>{@code
* statements:
* - name: name1
* stmt: statement body
* params:
* p1: v1
* p2: v2
* }</pre>
*
* <H3>List Of Condensed Maps</H3>
* <pre>{@code
* statements:
* - name1: statement body
* p1: v1
* p2: v2
* }</pre>
*/ */
public abstract class OpTemplate implements Tagged { public abstract class OpTemplate implements Tagged {
@ -170,6 +69,7 @@ public abstract class OpTemplate implements Tagged {
/** /**
* Return a map of tags for this statement. Implementations are required to * Return a map of tags for this statement. Implementations are required to
* add a tag for "name" automatically when this value is set during construction. * add a tag for "name" automatically when this value is set during construction.
*
* @return A map of assigned tags for the op, with the name added as an auto-tag. * @return A map of assigned tags for the op, with the name added as an auto-tag.
*/ */
public abstract Map<String, String> getTags(); public abstract Map<String, String> getTags();
@ -208,7 +108,7 @@ public abstract class OpTemplate implements Tagged {
if (defaultValue.getClass().isAssignableFrom(value.getClass())) { if (defaultValue.getClass().isAssignableFrom(value.getClass())) {
return (V) value; return (V) value;
} else { } else {
return NBTypeConverter.convertOr(value,defaultValue); return NBTypeConverter.convertOr(value, defaultValue);
} }
} }
@ -229,7 +129,6 @@ public abstract class OpTemplate implements Tagged {
} }
public <V> V getParam(String name, Class<? extends V> type) { public <V> V getParam(String name, Class<? extends V> type) {
Object object = getParams().get(name); Object object = getParams().get(name);
if (object == null) { if (object == null) {
@ -273,7 +172,7 @@ public abstract class OpTemplate implements Tagged {
* *
* @return an optional {@link ParsedStringTemplate} * @return an optional {@link ParsedStringTemplate}
*/ */
public Optional<ParsedStringTemplate> getParsed(Function<String,String>... rewriters) { public Optional<ParsedStringTemplate> getParsed(Function<String, String>... rewriters) {
Optional<String> os = getStmt(); Optional<String> os = getStmt();
return os.map(s -> { return os.map(s -> {
String result = s; String result = s;
@ -281,7 +180,7 @@ public abstract class OpTemplate implements Tagged {
result = rewriter.apply(result); result = rewriter.apply(result);
} }
return result; return result;
}).map(s -> new ParsedStringTemplate(s,getBindings())); }).map(s -> new ParsedStringTemplate(s, getBindings()));
} }
public Optional<ParsedStringTemplate> getParsed() { public Optional<ParsedStringTemplate> getParsed() {
@ -309,7 +208,7 @@ public abstract class OpTemplate implements Tagged {
fields.put(FIELD_TAGS, this.getTags()); fields.put(FIELD_TAGS, this.getTags());
} }
this.getOp().ifPresent(o -> fields.put(FIELD_OP,o)); this.getOp().ifPresent(o -> fields.put(FIELD_OP, o));
fields.put(FIELD_NAME, this.getName()); fields.put(FIELD_NAME, this.getName());
@ -320,10 +219,11 @@ public abstract class OpTemplate implements Tagged {
* Legacy support for String form statements. This is left here as a convenience method, * Legacy support for String form statements. This is left here as a convenience method,
* however it is changed to an Optional to force caller refactorings. * however it is changed to an Optional to force caller refactorings.
* *
* @return An optional string version of the op, empty if there is no 'stmt' property in the op fields, or no op fields at all. * @return An optional string version of the op, empty if there is no 'stmt' property in the op fields, or no op
* fields at all.
*/ */
public Optional<String> getStmt() { public Optional<String> getStmt() {
return getOp().map(m->m.get("stmt")).map(s->{ return getOp().map(m -> m.get("stmt")).map(s -> {
if (s instanceof CharSequence) { if (s instanceof CharSequence) {
return s.toString(); return s.toString();
} else { } else {
@ -333,7 +233,7 @@ public abstract class OpTemplate implements Tagged {
} }
public Element getParamReader() { public Element getParamReader() {
return NBParams.one(getName(),getParams()); return NBParams.one(getName(), getParams());
} }
/** /**
@ -347,13 +247,13 @@ public abstract class OpTemplate implements Tagged {
* @return the map of all remaining fields from the op template and the params map. * @return the map of all remaining fields from the op template and the params map.
*/ */
public Map<String, Object> remainingFields() { public Map<String, Object> remainingFields() {
Map<String,Object> remaining = new LinkedHashMap<>(getOp().orElse(Map.of())); Map<String, Object> remaining = new LinkedHashMap<>(getOp().orElse(Map.of()));
remaining.putAll(getParams()); remaining.putAll(getParams());
return remaining; return remaining;
} }
public void assertConsumed() { public void assertConsumed() {
if (size()>0) { if (size() > 0) {
throw new OpConfigError("The op template named '" + getName() + "' was not fully consumed. These fields are not being applied:" + remainingFields()); throw new OpConfigError("The op template named '" + getName() + "' was not fully consumed. These fields are not being applied:" + remainingFields());
} }
} }

View File

@ -40,11 +40,11 @@ public class OpsBlock implements Tagged, Iterable<OpTemplate> {
public List<OpTemplate> getOps() { public List<OpTemplate> getOps() {
List<OpTemplate> rawOpTemplates = new ArrayList<>(); List<OpTemplate> rawOpTemplates = new ArrayList<>();
List<RawOpDef> statements = rawOpsBlock.getRawStmtDefs(); List<RawOpDef> opDefs = rawOpsBlock.getRawOpDefs();
for (int i = 0; i < statements.size(); i++) { for (int i = 0; i < opDefs.size(); i++) {
rawOpTemplates.add( rawOpTemplates.add(
new OpDef(this, statements.get(i)) new OpDef(this, opDefs.get(i))
); );
} }
return rawOpTemplates; return rawOpTemplates;

View File

@ -27,7 +27,7 @@ import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
/** /**
* StmtsDoc creates a logical view of a statements doc that includes * OpsDoc creates a logical view of a RawOpsDoc doc that includes
* all inherited and overridden values for bindings, tags, and params. * all inherited and overridden values for bindings, tags, and params.
*/ */
public class OpsDoc implements Tagged, Iterable<OpsBlock> { public class OpsDoc implements Tagged, Iterable<OpsBlock> {
@ -86,10 +86,10 @@ public class OpsDoc implements Tagged, Iterable<OpsBlock> {
} }
/** /**
* @return The list of all included statements for all included block in this document, * @return The list of all included op templates for all included block in this document,
* including the inherited and overridden values from the this doc and the parent block. * including the inherited and overridden values from this doc and the parent block.
*/ */
public List<OpTemplate> getStmts() { public List<OpTemplate> getOpTemplates() {
return getBlocks().stream().flatMap(b -> b.getOps().stream()).collect(Collectors.toList()); return getBlocks().stream().flatMap(b -> b.getOps().stream()).collect(Collectors.toList());
} }

View File

@ -53,26 +53,26 @@ public class OpsDocList implements Iterable<OpsDoc> {
} }
public List<OpsDoc> getStmtDocs() { public List<OpsDoc> getStmtDocs() {
return rawOpsDocList.getStmtsDocs().stream() return rawOpsDocList.getOpsDocs().stream()
.map(OpsDoc::new) .map(OpsDoc::new)
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
public List<OpTemplate> getStmts() { public List<OpTemplate> getOps() {
return getStmts(""); return getOps("");
} }
/** /**
* @param tagFilterSpec a comma-separated tag filter spec * @param tagFilterSpec a comma-separated tag filter spec
* @return The list of all included statements for all included blocks of in this document, * @return The list of all included op templates for all included blocks of in this document,
* including the inherited and overridden values from the this doc and the parent block. * including the inherited and overridden values from this doc and the parent block.
*/ */
public List<OpTemplate> getStmts(String tagFilterSpec) { public List<OpTemplate> getOps(String tagFilterSpec) {
TagFilter ts = new TagFilter(tagFilterSpec); TagFilter ts = new TagFilter(tagFilterSpec);
List<OpTemplate> opTemplates = new ArrayList<>(); List<OpTemplate> opTemplates = new ArrayList<>();
getStmtDocs().stream() getStmtDocs().stream()
.flatMap(d -> d.getStmts().stream()) .flatMap(d -> d.getOpTemplates().stream())
.filter(ts::matchesTagged) .filter(ts::matchesTagged)
.forEach(opTemplates::add); .forEach(opTemplates::add);
@ -155,8 +155,6 @@ public class OpsDocList implements Iterable<OpsDoc> {
} }
sb.append("docs: " + docscount + " blocks:" + blockscount + " ops:" + opscount); sb.append("docs: " + docscount + " blocks:" + blockscount + " ops:" + opscount);
// String names = this.rawStmtsDocList.getStmtsDocs().stream().flatMap(sd -> sd.getRawStmtDefs().stream()).map(d->d.getName()).collect(Collectors.joining(","));
// sb.append(", names:").append(names);
return sb.toString(); return sb.toString();
} }
@ -166,7 +164,7 @@ public class OpsDocList implements Iterable<OpsDoc> {
}; };
public Pattern getVersionRegex() { public Pattern getVersionRegex() {
List<RawOpsDoc> stmtDocs = rawOpsDocList.getStmtsDocs(); List<RawOpsDoc> stmtDocs = rawOpsDocList.getOpsDocs();
return Pattern.compile(stmtDocs.size()>0 ? stmtDocs.get(0).getVersionRegex() : ".*"); return Pattern.compile(stmtDocs.size()>0 ? stmtDocs.get(0).getVersionRegex() : ".*");
} }
} }

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -31,34 +31,24 @@ import java.util.function.Function;
* a client API for advanced testing. The default instance should simply be named * a client API for advanced testing. The default instance should simply be named
* {@code default}</p> * {@code default}</p>
* *
* <p>Most native drivers use some combination of fluent, functional, and declarative * <p>Most native drivers use some combination of fluent, functional, and declarative patterns. These usually require
* patterns. These usually require you to keep access to a set of core * you to keep access to a set of core state-holding objects in order to construct new elements to drive operations
* state-holding objects in order to construct new elements to drive operations with. * with. An example of this would be creating an executable operation from a session object. It is necessary to keep the
* An example of this would be creating a statement from a session. It is necessary * session around in for when you create new statements. Maintaining the session object is considered an essential part
* to keep the session around in for when you create new statements. Maintaining * of idiomatic and efficient use of the API. Further, you may have builders or factories that are created from the
* the session object is considered an essential part of idiomatic and efficient * session which should be cached as well. Keeping all these objects together requires attaching them to a cohesive
* use of the API. Further, you may have builders or factories that are created * owning object -- That is the space cache.</p>
* from the session which should be cached as well. Keeping all these objects
* together requires attaching them to a cohesive owning object -- That is the space
* cache.</p>
* *
* <p>You might want to create multiple session contexts in order to test out * @param <S>
* non-trivial behavior in advanced testing scenarios. To do this dynamically, i.e. * The type which will represent the cache for a given type of adapter.
* acquire some named space cache, simply call the {@link #get(String)}</p> method
* with the name of the space you want to use. This value can be provided as a
* dynamic field in your op mapping ({@link io.nosqlbench.engine.api.activityimpl.OpMapper})
* or synthesis ({@link io.nosqlbench.engine.api.activityimpl.OpDispenser})
* implementation.
*
* @param <S> The type which will represent the cache for a given type of adapter.
*/ */
public class DriverSpaceCache<S> { public class DriverSpaceCache<S> {
private final ConcurrentHashMap<String,S> cache = new ConcurrentHashMap<>(); private final ConcurrentHashMap<String, S> cache = new ConcurrentHashMap<>();
private final Function<String, S> newSpaceFunction; private final Function<String, S> newSpaceFunction;
public DriverSpaceCache(Function<String,S> newSpaceFunction) { public DriverSpaceCache(Function<String, S> newSpaceFunction) {
this.newSpaceFunction = newSpaceFunction; this.newSpaceFunction = newSpaceFunction;
} }
@ -66,7 +56,7 @@ public class DriverSpaceCache<S> {
return cache.computeIfAbsent(name, newSpaceFunction); return cache.computeIfAbsent(name, newSpaceFunction);
} }
public Map<String,S> getElements() { public Map<String, S> getElements() {
return Collections.unmodifiableMap(cache); return Collections.unmodifiableMap(cache);
} }

View File

@ -24,22 +24,27 @@ import java.util.List;
import java.util.Map; import java.util.Map;
/** /**
* In the case that no statements are provided for an activity, but a workload * It is possible for a DriverAdapter to create op templates using partial information.
* description is provided, and the result for no found statements is not * For example, the stdout driver can use only bindings to create CSV or JSON style data renderings.
* simply a matter of tag filtering, an activity may provide its own * This mechanism is only triggered when:
* synthetic ops. This is here primarily to support the classic behavior of stdout * <OL>
* until it's functionality is subsumed by standard diagnostic features. * <LI>No op templates were provided.</LI>
* * <LI>The default driver for an activity implements this method.</LI>
* Note that this is only valid while an activity uses a single driver, which will * </OL>
* change with upcoming API updates. * <p>
* This excludes cases where a workload was provided with op templates, but they were all filtered out. In that case,
* the user should be informed of an error.
*/ */
public interface SyntheticOpTemplateProvider extends DriverAdapterDecorators { public interface SyntheticOpTemplateProvider extends DriverAdapterDecorators {
/** /**
* If a driver adapter supports creating example op templates from bindings, * If a driver adapter supports creating example op templates from bindings,
* it must implement this method to do so. * it must implement this method to do so.
* @param opsDocList The existing doc structure, which should contain no fully defined op templates, but may contain other elements like bindings *
* @param opsDocList
* The existing doc structure, which should contain no fully defined op templates, but may contain other
* elements like bindings
* @return A list of op templates, size zero or more * @return A list of op templates, size zero or more
*/ */
List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String,Object> params); List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> params);
} }

View File

@ -1,34 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.api.config.standard.NBConfiguration;
import java.util.List;
import java.util.Optional;
/**
* An Op Template Supplier can provide its own source of op templates instead
* of relying on the built-in mechanism. By default, the built-in mechanism
* will read op definitions from parameters first, then any ops (statements)
* from yaml files provided in the workload= activity parameters.
*/
public interface OpTemplateSupplier extends DriverAdapterDecorators {
Optional<List<OpTemplate>> loadOpTemplates(NBConfiguration cfg);
}

View File

@ -29,12 +29,12 @@ public class OpsLoaderTest {
@Test @Test
public void testTemplateVarSubstitution() { public void testTemplateVarSubstitution() {
OpsDocList stmtsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources"); OpsDocList opsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources");
assertThat(stmtsDocs).isNotNull(); assertThat(opsDocs).isNotNull();
List<OpsDoc> docs = stmtsDocs.getStmtDocs(); List<OpsDoc> docs = opsDocs.getStmtDocs();
assertThat(docs).hasSize(1); assertThat(docs).hasSize(1);
OpsDoc stmtsBlocks = docs.get(0); OpsDoc opsDoc = docs.get(0);
Map<String, String> bindings = stmtsBlocks.getBindings(); Map<String, String> bindings = opsDoc.getBindings();
assertThat(bindings).isEqualTo(Map.of( assertThat(bindings).isEqualTo(Map.of(
"b1a","Prefix(\"prefix\")", "b1a","Prefix(\"prefix\")",
"b1b","Prefix(\"prefix\")", "b1b","Prefix(\"prefix\")",

View File

@ -37,7 +37,7 @@ public class OpDefTest {
@Test @Test
public void testLayering() { public void testLayering() {
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
assertThat(all).isNotNull(); assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2); assertThat(all.getStmtDocs()).hasSize(2);
OpsDoc doc1 = all.getStmtDocs().get(0); OpsDoc doc1 = all.getStmtDocs().get(0);
@ -64,7 +64,7 @@ public class OpDefTest {
@Test @Test
public void testStatementRendering() { public void testStatementRendering() {
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
assertThat(all).isNotNull(); assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2); assertThat(all.getStmtDocs()).hasSize(2);
OpsDoc doc1 = all.getStmtDocs().get(0); OpsDoc doc1 = all.getStmtDocs().get(0);
@ -79,10 +79,10 @@ public class OpDefTest {
@Test @Test
public void testConsumableMapState() { public void testConsumableMapState() {
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs(); List<OpsDoc> docs = all.getStmtDocs();
OpsDoc block1 = docs.get(1); OpsDoc block1 = docs.get(1);
List<OpTemplate> stmts = block1.getStmts(); List<OpTemplate> stmts = block1.getOpTemplates();
OpTemplate stmt0 = stmts.get(0); OpTemplate stmt0 = stmts.get(0);
OpTemplate stmt1 = stmts.get(1); OpTemplate stmt1 = stmts.get(1);
assertThat(stmt0.getParams()).containsAllEntriesOf(Map.of("timeout", 23423, "foobar", "baz")); assertThat(stmt0.getParams()).containsAllEntriesOf(Map.of("timeout", 23423, "foobar", "baz"));
@ -94,7 +94,7 @@ public class OpDefTest {
@Test @Test
public void testMapOfMaps() { public void testMapOfMaps() {
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs(); List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc0 = docs.get(0); OpsDoc doc0 = docs.get(0);
assertThat(doc0.getName()).isEqualTo("map-of-maps"); assertThat(doc0.getName()).isEqualTo("map-of-maps");
@ -115,7 +115,7 @@ public class OpDefTest {
@Test @Test
public void testBasicStringStmt() { public void testBasicStringStmt() {
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs(); List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc1 = docs.get(1); OpsDoc doc1 = docs.get(1);
assertThat(doc1.getName()).isEqualTo("string-statement"); assertThat(doc1.getName()).isEqualTo("string-statement");
@ -130,7 +130,7 @@ public class OpDefTest {
@Test @Test
public void testListOfNamedMap() { public void testListOfNamedMap() {
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs(); List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc2 = docs.get(2); OpsDoc doc2 = docs.get(2);
assertThat(doc2.getName()).isEqualTo("list-of-named-map"); assertThat(doc2.getName()).isEqualTo("list-of-named-map");

View File

@ -28,15 +28,15 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
public class StmtEscapingTest { public class OpEscapingTest {
private final static Logger logger = LogManager.getLogger(StmtEscapingTest.class); private final static Logger logger = LogManager.getLogger(OpEscapingTest.class);
private static List<OpTemplate> defs; private static List<OpTemplate> defs;
@BeforeAll @BeforeAll
public static void testLayering() { public static void testLayering() {
OpsDocList all = OpsLoader.loadPath("testdocs/escaped_stmts.yaml", Map.of()); OpsDocList all = OpsLoader.loadPath("testdocs/escaped_ops.yaml", Map.of());
assertThat(all).isNotNull(); assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(1); assertThat(all.getStmtDocs()).hasSize(1);
OpsDoc doc1 = all.getStmtDocs().get(0); OpsDoc doc1 = all.getStmtDocs().get(0);

View File

@ -24,89 +24,93 @@ import java.util.List;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
public class StmtVariationTests { public class OpVariationTests {
private final static Logger logger = LogManager.getLogger(StmtVariationTests.class); private final static Logger logger = LogManager.getLogger(OpVariationTests.class);
@Test @Test
public void testListStmtsOnly() { public void testListOpsOnly() {
RawOpsLoader ysl = new RawOpsLoader(); RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString( RawOpsDocList docs = ysl.loadString(
"statements:\n" + "ops:\n" +
" - first statement\n" + " - first op\n" +
" - second statement\n" " - second op\n"
); );
assertThat(docs.getStmtsDocs()).hasSize(1); assertThat(docs.getOpsDocs()).hasSize(1);
RawOpsDoc doc = docs.getStmtsDocs().get(0); RawOpsDoc doc = docs.getOpsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(2); assertThat(doc.getRawOpDefs()).hasSize(2);
List<RawOpDef> stmts = doc.getRawStmtDefs(); List<RawOpDef> ops = doc.getRawOpDefs();
RawOpDef s0 = stmts.get(0); RawOpDef s0 = ops.get(0);
assertThat(s0.getName()).isEqualTo("stmt1"); assertThat(s0.getName()).isEqualTo("stmt1");
assertThat(s0.getStmt()).isEqualTo("first statement"); assertThat(s0.getStmt()).isEqualTo("first op");
RawOpDef s1 = stmts.get(1); RawOpDef s1 = ops.get(1);
assertThat(s1.getName()).isEqualTo("stmt2"); assertThat(s1.getName()).isEqualTo("stmt2");
assertThat(s1.getStmt()).isEqualTo("second statement"); assertThat(s1.getStmt()).isEqualTo("second op");
} }
@Test @Test
public void testSingleEntryMapStmtsOnly() { public void testSingleEntryMapStmtsOnly() {
RawOpsLoader ysl = new RawOpsLoader(); RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString( RawOpsDocList docs = ysl.loadString(
"statements:\n" + "ops:\n" +
" - s1: statement one\n" + " - s1: op one\n" +
" - s2: statement two\n" " - s2: op two\n"
); );
assertThat(docs.getStmtsDocs()).hasSize(1); assertThat(docs.getOpsDocs()).hasSize(1);
RawOpsDoc doc = docs.getStmtsDocs().get(0); RawOpsDoc doc = docs.getOpsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(2); assertThat(doc.getRawOpDefs()).hasSize(2);
List<RawOpDef> stmts = doc.getRawStmtDefs(); List<RawOpDef> ops = doc.getRawOpDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class); assertThat(ops.get(0)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(0).getName()).isEqualTo("s1"); assertThat(ops.get(0).getName()).isEqualTo("s1");
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one"); assertThat(ops.get(0).getStmt()).isEqualTo("op one");
assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class); assertThat(ops.get(1)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(1).getName()).isEqualTo("s2"); assertThat(ops.get(1).getName()).isEqualTo("s2");
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two"); assertThat(ops.get(1).getStmt()).isEqualTo("op two");
} }
@Test @Test
public void testMapStmtsOnly() { public void testMapStmtsOnly() {
RawOpsLoader ysl = new RawOpsLoader(); RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString( RawOpsDocList docs = ysl.loadString(
"statements:\n" + "ops:\n" +
" - name: s1\n" + " - name: s1\n" +
" stmt: statement one\n" + " stmt: op one\n" +
" - name: s2\n" + " - name: s2\n" +
" stmt: statement two\n" " stmt: op two\n"
); );
assertThat(docs.getStmtsDocs()).hasSize(1); assertThat(docs.getOpsDocs()).hasSize(1);
RawOpsDoc doc = docs.getStmtsDocs().get(0); RawOpsDoc doc = docs.getOpsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(2); assertThat(doc.getRawOpDefs()).hasSize(2);
List<RawOpDef> stmts = doc.getRawStmtDefs(); List<RawOpDef> stmts = doc.getRawOpDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(0).getName()).isEqualTo("s1"); assertThat(stmts.get(0).getName()).isEqualTo("s1");
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one"); assertThat(stmts.get(0).getStmt()).isEqualTo("op one");
assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(1).getName()).isEqualTo("s2"); assertThat(stmts.get(1).getName()).isEqualTo("s2");
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two"); assertThat(stmts.get(1).getStmt()).isEqualTo("op two");
} }
/**
* This test uses the compatible names for the sake of demonstration. Users should know that they can use op and
* statement terms interchangeably in op templates.
*/
@Test @Test
public void testMixedForms() { public void testMixedForms() {
RawOpsLoader ysl = new RawOpsLoader(); RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString( RawOpsDocList docs = ysl.loadString(
"statement:\n" + "statement:\n" +
" - name: s1\n" + " - name: s1\n" +
" stmt: statement one\n" + " stmt: statement one\n" +
" - statement two\n" + " - statement two\n" +
" - s3: statement three\n" + " - s3: statement three\n" +
" - ST4: statement four\n" + " - ST4: statement four\n" +
" type: organic\n" " type: organic\n"
); );
assertThat(docs.getStmtsDocs()).hasSize(1); assertThat(docs.getOpsDocs()).hasSize(1);
RawOpsDoc doc = docs.getStmtsDocs().get(0); RawOpsDoc doc = docs.getOpsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(4); assertThat(doc.getRawOpDefs()).hasSize(4);
List<RawOpDef> stmts = doc.getRawStmtDefs(); List<RawOpDef> stmts = doc.getRawOpDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(0).getName()).isEqualTo("s1"); assertThat(stmts.get(0).getName()).isEqualTo("s1");
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one"); assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");

View File

@ -32,9 +32,9 @@ public class RawYamlTemplateLoaderTest {
public void testLoadPropertiesBlock() { public void testLoadPropertiesBlock() {
RawOpsLoader ysl = new RawOpsLoader(); RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList rawBlockDocs = ysl.loadPath("testdocs/rawblock.yaml"); RawOpsDocList rawBlockDocs = ysl.loadPath("testdocs/rawblock.yaml");
assertThat(rawBlockDocs.getStmtsDocs()).hasSize(1); assertThat(rawBlockDocs.getOpsDocs()).hasSize(1);
RawOpsDoc rawBlockDoc = rawBlockDocs.getStmtsDocs().get(0); RawOpsDoc rawBlockDoc = rawBlockDocs.getOpsDocs().get(0);
assertThat(rawBlockDoc.getRawStmtDefs()).hasSize(1); assertThat(rawBlockDoc.getRawOpDefs()).hasSize(1);
assertThat(rawBlockDoc.getBindings()).hasSize(1); assertThat(rawBlockDoc.getBindings()).hasSize(1);
assertThat(rawBlockDoc.getName()).isEqualTo("name"); assertThat(rawBlockDoc.getName()).isEqualTo("name");
assertThat(rawBlockDoc.getTags()).hasSize(1); assertThat(rawBlockDoc.getTags()).hasSize(1);
@ -44,8 +44,8 @@ public class RawYamlTemplateLoaderTest {
@Test @Test
public void testLoadFullFormat() { public void testLoadFullFormat() {
RawOpsLoader ysl = new RawOpsLoader(); RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_stmts.yaml"); RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_ops.yaml");
List<RawOpsDoc> rawOpsDocs = erthing.getStmtsDocs(); List<RawOpsDoc> rawOpsDocs = erthing.getOpsDocs();
assertThat(rawOpsDocs).hasSize(2); assertThat(rawOpsDocs).hasSize(2);
RawOpsDoc rawOpsDoc = rawOpsDocs.get(0); RawOpsDoc rawOpsDoc = rawOpsDocs.get(0);
List<RawOpsBlock> blocks = rawOpsDoc.getBlocks(); List<RawOpsBlock> blocks = rawOpsDoc.getBlocks();
@ -58,8 +58,8 @@ public class RawYamlTemplateLoaderTest {
@Test @Test
public void testLoadScenarios() { public void testLoadScenarios() {
RawOpsLoader ysl = new RawOpsLoader(); RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_stmts.yaml"); RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_ops.yaml");
List<RawOpsDoc> rawOpsDocs = erthing.getStmtsDocs(); List<RawOpsDoc> rawOpsDocs = erthing.getOpsDocs();
assertThat(rawOpsDocs).hasSize(2); assertThat(rawOpsDocs).hasSize(2);
RawOpsDoc rawOpsDoc = rawOpsDocs.get(0); RawOpsDoc rawOpsDoc = rawOpsDocs.get(0);
List<RawOpsBlock> blocks = rawOpsDoc.getBlocks(); List<RawOpsBlock> blocks = rawOpsDoc.getBlocks();

View File

@ -109,7 +109,7 @@ public class YamlSpecValidator implements STAssemblyValidator {
List<Map<String, Object>> expectedList = gson.fromJson(json, type); List<Map<String, Object>> expectedList = gson.fromJson(json, type);
OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null); OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
List<OpTemplate> stmts = stmtsDocs.getStmts(); List<OpTemplate> stmts = stmtsDocs.getOps();
List<Map<String, Object>> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList()); List<Map<String, Object>> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList());
try { try {
@ -143,8 +143,6 @@ public class YamlSpecValidator implements STAssemblyValidator {
private void validateYamlWithJson(String desc, String yaml, String json, STNodeReference testset, boolean debug) { private void validateYamlWithJson(String desc, String yaml, String json, STNodeReference testset, boolean debug) {
System.out.format("%-40s", "- checking yaml->json"); System.out.format("%-40s", "- checking yaml->json");
// StmtsDocList stmts = StatementsLoader.loadString(yaml);
try { try {
List<Map<String, Object>> docmaps = new RawYamlLoader().loadString(logger, yaml); List<Map<String, Object>> docmaps = new RawYamlLoader().loadString(logger, yaml);
JsonElement elem = null; JsonElement elem = null;

View File

@ -32,7 +32,7 @@ public class OpDetailOverrideTest {
@Test @Test
public void testStmtOverrides() { public void testStmtOverrides() {
OpsDocList doclist = OpsLoader.loadPath("testdocs/stmt_details.yaml", Map.of()); OpsDocList doclist = OpsLoader.loadPath("testdocs/op_details.yaml", Map.of());
assertThat(doclist).isNotNull(); assertThat(doclist).isNotNull();
@ -50,24 +50,24 @@ public class OpDetailOverrideTest {
assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","block0--stmt1")); assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","block0--stmt1"));
OpsBlock doc1block1 = doc1.getBlocks().get(1); OpsBlock doc1block1 = doc1.getBlocks().get(1);
List<OpTemplate> stmts = doc1block1.getOps(); List<OpTemplate> ops = doc1block1.getOps();
assertThat(stmts).hasSize(4); assertThat(ops).hasSize(4);
s = stmts.get(0); s = ops.get(0);
assertThat(s.getName()).isEqualTo("testblock1--stmt1"); assertThat(s.getName()).isEqualTo("testblock1--stmt1");
assertThat(s.getStmt()).contains("astatement1"); assertThat(s.getStmt()).contains("astatement1");
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--stmt1")); assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--stmt1"));
assertThat(s.getBindings()).hasSize(1); assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1); assertThat(s.getParams()).hasSize(1);
s = stmts.get(1); s = ops.get(1);
assertThat(s.getName()).isEqualTo("testblock1--s2name"); assertThat(s.getName()).isEqualTo("testblock1--s2name");
assertThat(s.getStmt()).contains("s2statement data"); assertThat(s.getStmt()).contains("s2statement data");
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s2name")); assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s2name"));
assertThat(s.getBindings()).hasSize(1); assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1); assertThat(s.getParams()).hasSize(1);
s = stmts.get(2); s = ops.get(2);
assertThat(s.getName()).isEqualTo("testblock1--s3"); assertThat(s.getName()).isEqualTo("testblock1--s3");
assertThat(s.getStmt()).contains("statement three"); assertThat(s.getStmt()).contains("statement three");
assertThat(s.getTags()).containsEntry("tname1", "tval1"); assertThat(s.getTags()).containsEntry("tname1", "tval1");
@ -75,7 +75,7 @@ public class OpDetailOverrideTest {
assertThat(s.getBindings()).hasSize(3); assertThat(s.getBindings()).hasSize(3);
assertThat(s.getParams()).hasSize(2); assertThat(s.getParams()).hasSize(2);
s = stmts.get(3); s = ops.get(3);
assertThat(s.getName()).isEqualTo("testblock1--s4"); assertThat(s.getName()).isEqualTo("testblock1--s4");
assertThat(s.getStmt()).contains("statement 4"); assertThat(s.getStmt()).contains("statement 4");
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s4")); assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s4"));

View File

@ -50,7 +50,7 @@ public class OpsDocListTest {
@BeforeAll @BeforeAll
public static void testLoadYaml() { public static void testLoadYaml() {
doclist = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); doclist = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
} }
@Test @Test
@ -76,21 +76,21 @@ public class OpsDocListTest {
@Test @Test
public void testStmtInheritsBlockData() { public void testStmtInheritsBlockData() {
OpsDoc doc0 = doclist.getStmtDocs().get(0); OpsDoc doc0 = doclist.getStmtDocs().get(0);
List<OpTemplate> stmts1 = doc0.getBlocks().get(0).getOps(); List<OpTemplate> ops1 = doc0.getBlocks().get(0).getOps();
assertThat(stmts1).hasSize(2); assertThat(ops1).hasSize(2);
OpsBlock block0 = doc0.getBlocks().get(0); OpsBlock block0 = doc0.getBlocks().get(0);
assertThat(block0.getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d")); assertThat(block0.getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
assertThat(block0.getParams()).containsExactly(MapEntry.entry("param1","value1")); assertThat(block0.getParams()).containsExactly(MapEntry.entry("param1","value1"));
assertThat(block0.getTags()).containsExactly(MapEntry.entry("atagname","atagvalue")); assertThat(block0.getTags()).containsExactly(MapEntry.entry("atagname","atagvalue"));
assertThat(stmts1.get(0).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d")); assertThat(ops1.get(0).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
assertThat(stmts1.get(0).getParams()).containsExactly(MapEntry.entry("param1","value1")); assertThat(ops1.get(0).getParams()).containsExactly(MapEntry.entry("param1","value1"));
assertThat(stmts1.get(0).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt1","block","doc1--block0")); assertThat(ops1.get(0).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt1","block","doc1--block0"));
assertThat(stmts1.get(1).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d")); assertThat(ops1.get(1).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
assertThat(stmts1.get(1).getParams()).containsExactly(MapEntry.entry("param1","value1")); assertThat(ops1.get(1).getParams()).containsExactly(MapEntry.entry("param1","value1"));
assertThat(stmts1.get(1).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt2","block","doc1--block0")); assertThat(ops1.get(1).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt2","block","doc1--block0"));
} }
@ -120,15 +120,15 @@ public class OpsDocListTest {
@Test @Test
public void testStmtsGetter() { public void testStmtsGetter() {
OpsDoc doc1 = doclist.getStmtDocs().get(1); OpsDoc doc1 = doclist.getStmtDocs().get(1);
List<OpTemplate> stmts = doc1.getStmts(); List<OpTemplate> stmts = doc1.getOpTemplates();
assertThat(stmts).hasSize(4); assertThat(stmts).hasSize(4);
} }
@Test @Test
public void testFilteredStmts() { public void testFilteredStmts() {
List<OpTemplate> stmts = doclist.getStmts(""); List<OpTemplate> stmts = doclist.getOps("");
assertThat(stmts).hasSize(6); assertThat(stmts).hasSize(6);
stmts = doclist.getStmts("root1:value23"); stmts = doclist.getOps("root1:value23");
assertThat(stmts).hasSize(2); assertThat(stmts).hasSize(2);
} }

View File

@ -70,8 +70,8 @@ public class ParsedOpTest {
ps1: "param-one" ps1: "param-one"
"""; """;
OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null); OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null);
assertThat(stmtsDocs.getStmts().size()).isEqualTo(1); assertThat(stmtsDocs.getOps().size()).isEqualTo(1);
OpTemplate opTemplate = stmtsDocs.getStmts().get(0); OpTemplate opTemplate = stmtsDocs.getOps().get(0);
ParsedOp parsedOp = new ParsedOp(opTemplate, cfg); ParsedOp parsedOp = new ParsedOp(opTemplate, cfg);
assertThat(parsedOp.getAsFunctionOr("d1","invalid").apply(1L)).isEqualTo("one"); assertThat(parsedOp.getAsFunctionOr("d1","invalid").apply(1L)).isEqualTo("one");

View File

@ -551,12 +551,12 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
OpsDocList opsDocList = loadStmtsDocList(); OpsDocList opsDocList = loadStmtsDocList();
List<OpTemplate> unfilteredOps = opsDocList.getStmts(); List<OpTemplate> unfilteredOps = opsDocList.getOps();
List<OpTemplate> filteredOps = opsDocList.getStmts(tagfilter); List<OpTemplate> filteredOps = opsDocList.getOps(tagfilter);
if (filteredOps.size() == 0) { if (filteredOps.size() == 0) {
if (unfilteredOps.size() > 0) { // There were no ops, and it was because they were all filtered out if (unfilteredOps.size() > 0) { // There were no ops, and it was because they were all filtered out
throw new BasicError("There were no active statements with tag filter '" throw new BasicError("There were no active op templates with tag filter '"
+ tagfilter + "', since all " + unfilteredOps.size() + " were filtered out."); + tagfilter + "', since all " + unfilteredOps.size() + " were filtered out.");
} else { } else {
// There were no ops, and it *wasn't* because they were all filtered out. // There were no ops, and it *wasn't* because they were all filtered out.
@ -578,7 +578,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
} }
} }
if (filteredOps.size() == 0) { if (filteredOps.size() == 0) {
throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'"); throw new BasicError("There were no active op templates with tag filter '" + tagfilter + "'");
} }
} }
@ -613,8 +613,8 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
* taken as the only provided statement.</LI> * taken as the only provided statement.</LI>
* <LI>If a 'yaml, or 'workload' parameter is provided, then the statements in that file * <LI>If a 'yaml, or 'workload' parameter is provided, then the statements in that file
* are taken with their ratios </LI> * are taken with their ratios </LI>
* <LI>Any provided tags filter is used to select only the statements which have matching * <LI>Any provided tags filter is used to select only the op templates which have matching
* tags. If no tags are provided, then all the found statements are included.</LI> * tags. If no tags are provided, then all the found op templates are included.</LI>
* <LI>The ratios and the 'seq' parameter are used to build a sequence of the ready operations, * <LI>The ratios and the 'seq' parameter are used to build a sequence of the ready operations,
* where the sequence length is the sum of the ratios.</LI> * where the sequence length is the sum of the ratios.</LI>
* </OL> * </OL>

View File

@ -35,11 +35,11 @@ public class CommandTemplateTest {
@Test @Test
public void testCommandTemplate() { public void testCommandTemplate() {
OpsDocList stmtsDocs = OpsLoader.loadString("" + OpsDocList opsDocs = OpsLoader.loadString("" +
"statements:\n" + "ops:\n" +
" - s1: test1=foo test2=bar", " - s1: test1=foo test2=bar",
OpTemplateFormat.yaml, Map.of(), null); OpTemplateFormat.yaml, Map.of(), null);
OpTemplate optpl = stmtsDocs.getStmts().get(0); OpTemplate optpl = opsDocs.getOps().get(0);
CommandTemplate ct = new CommandTemplate(optpl); CommandTemplate ct = new CommandTemplate(optpl);
assertThat(ct.isStatic()).isTrue(); assertThat(ct.isStatic()).isTrue();
} }
@ -48,13 +48,13 @@ public class CommandTemplateTest {
public void testCommandTemplateFormat() { public void testCommandTemplateFormat() {
Gson gson = new GsonBuilder().setPrettyPrinting().create(); Gson gson = new GsonBuilder().setPrettyPrinting().create();
OpsDocList stmtsDocs = OpsLoader.loadString("" + OpsDocList stmtsDocs = OpsLoader.loadString("" +
"statements:\n" + "ops:\n" +
" - s1: test1=foo test2={bar}\n" + " - s1: test1=foo test2={bar}\n" +
" bindings:\n" + " bindings:\n" +
" bar: NumberNameToString();\n", " bar: NumberNameToString();\n",
OpTemplateFormat.yaml, Map.of(), null OpTemplateFormat.yaml, Map.of(), null
); );
OpTemplate optpl = stmtsDocs.getStmts().get(0); OpTemplate optpl = stmtsDocs.getOps().get(0);
CommandTemplate ct = new CommandTemplate(optpl); CommandTemplate ct = new CommandTemplate(optpl);
String format = gson.toJson(ct); String format = gson.toJson(ct);
logger.debug(format); logger.debug(format);

View File

@ -34,12 +34,12 @@ public class NBCLIScenarioParserTemplateVarTest {
cmds.forEach(System.out::println); cmds.forEach(System.out::println);
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams()); OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0); OpTemplate optpl1 = workload1.getOps().get(0);
System.out.println("op from cmd1:"+optpl1); System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n"); assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n");
OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArg("workload"),cmds.get(1).getParams()); OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArg("workload"),cmds.get(1).getParams());
OpTemplate optpl2 = workload2.getStmts().get(0); OpTemplate optpl2 = workload2.getOps().get(0);
System.out.println("op from cmd2:"+optpl2); System.out.println("op from cmd2:"+optpl2);
assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n"); assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n");
} }
@ -51,7 +51,7 @@ public class NBCLIScenarioParserTemplateVarTest {
cmds.forEach(System.out::println); cmds.forEach(System.out::println);
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams()); OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0); OpTemplate optpl1 = workload1.getOps().get(0);
System.out.println("op from cmd1:"+optpl1); System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n"); assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n");
} }

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -16,7 +16,7 @@
package io.nosqlbench.engine.rest.services; package io.nosqlbench.engine.rest.services;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader;
import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser; import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
import io.nosqlbench.engine.api.scenarios.WorkloadDesc; import io.nosqlbench.engine.api.scenarios.WorkloadDesc;
import io.nosqlbench.engine.rest.transfertypes.WorkspaceItemView; import io.nosqlbench.engine.rest.transfertypes.WorkspaceItemView;

View File

@ -576,7 +576,7 @@
<goal>prepare-agent</goal> <goal>prepare-agent</goal>
</goals> </goals>
<configuration> <configuration>
<destFile>${build.directory}/jacoco-data.exec</destFile> <destFile>${project.build.directory}/jacoco-data.exec</destFile>
</configuration> </configuration>
</execution> </execution>
<execution> <execution>
@ -586,8 +586,8 @@
<goal>report</goal> <goal>report</goal>
</goals> </goals>
<configuration> <configuration>
<dataFile>${build.directory}/jacoco-data.exec</dataFile> <dataFile>${project.build.directory}/jacoco-data.exec</dataFile>
<outputDirectory>${build.directory}/jacoco-reports</outputDirectory> <outputDirectory>${project.build.directory}/jacoco-reports</outputDirectory>
</configuration> </configuration>
</execution> </execution>
<execution> <execution>

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -45,7 +45,7 @@ public class ActivityDef implements NBNamedElement {
private final static Logger logger = LogManager.getLogger(ActivityDef.class); private final static Logger logger = LogManager.getLogger(ActivityDef.class);
// an alias with which to control the activity while it is running // an alias with which to control the activity while it is running
private static final String FIELD_ALIAS = "alias"; private static final String FIELD_ALIAS = "alias";
// a file or URL containing the activity: statements, generator bindings, ... // a file or URL containing the activity: op templates, generator bindings, ...
private static final String FIELD_ATYPE = "type"; private static final String FIELD_ATYPE = "type";
// cycles for this activity in either "M" or "N..M" form. "M" form implies "0..M" // cycles for this activity in either "M" or "N..M" form. "M" form implies "0..M"
private static final String FIELD_CYCLES = "cycles"; private static final String FIELD_CYCLES = "cycles";

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -23,7 +23,7 @@ public enum BindType {
/** /**
* A literal template is one which has no bindings that need to be provided to render a specific statement. * A literal template is one which has no bindings that need to be provided to render a specific statement.
* These templates are basically static statements. * These templates are basically static values.
* Example: <em>{@code truncate testks.testtable;}</em> * Example: <em>{@code truncate testks.testtable;}</em>
*/ */
literal, literal,