mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2024-11-22 00:38:05 -06:00
code cleanup and naming updates from "statements" to "ops"
This commit is contained in:
Jonathan Shook
parent
5dfa55bfe4
commit
ba2944ce77
@ -45,8 +45,8 @@ public class CGDefaultCqlBindings implements BindingsLibrary {
|
||||
", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE)
|
||||
);
|
||||
|
||||
OpsDocList stmtsDocs = OpsLoader.loadContent(content, Map.of());
|
||||
this.bindings = stmtsDocs.getDocBindings();
|
||||
OpsDocList opsDocList = OpsLoader.loadContent(content, Map.of());
|
||||
this.bindings = opsDocList.getDocBindings();
|
||||
}
|
||||
|
||||
private Optional<String> loadLocal(String path) {
|
||||
|
||||
@ -103,6 +103,6 @@ public class DiagDriverAdapter extends BaseDriverAdapter<DiagOp, DiagSpace> impl
|
||||
|
||||
@Override
|
||||
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> params) {
|
||||
return OpsLoader.loadString("log:level=INFO", OpTemplateFormat.inline, params,null).getStmts();
|
||||
return OpsLoader.loadString("log:level=INFO", OpTemplateFormat.inline, params,null).getOps();
|
||||
}
|
||||
}
|
||||
|
||||
@ -53,8 +53,8 @@ public class HttpOpMapperTest {
|
||||
|
||||
private static ParsedOp parsedOpFor(String yaml) {
|
||||
OpsDocList docs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
|
||||
OpTemplate stmtDef = docs.getStmts().get(0);
|
||||
ParsedOp parsedOp = new ParsedOp(stmtDef, cfg, List.of(adapter.getPreprocessor()));
|
||||
OpTemplate opTemplate = docs.getOps().get(0);
|
||||
ParsedOp parsedOp = new ParsedOp(opTemplate, cfg, List.of(adapter.getPreprocessor()));
|
||||
return parsedOp;
|
||||
}
|
||||
|
||||
@ -102,7 +102,7 @@ public class HttpOpMapperTest {
|
||||
@Test
|
||||
public void testRFCFormBody() {
|
||||
ParsedOp pop = parsedOpFor("""
|
||||
statements:
|
||||
ops:
|
||||
- s1: |
|
||||
get http://localhost/
|
||||
|
||||
@ -118,7 +118,7 @@ public class HttpOpMapperTest {
|
||||
// This can not be fully resolved in the unit testing context, but it could be
|
||||
// in the integrated testing context. It is sufficient to verify parsing here.
|
||||
ParsedOp pop = parsedOpFor("""
|
||||
statements:
|
||||
ops:
|
||||
- s1: |
|
||||
{method} {scheme}://{host}/{path}?{query} {version}
|
||||
Header1: {header1val}
|
||||
|
||||
@ -22,14 +22,14 @@ import java.util.*;
|
||||
|
||||
public class OpsOwner extends RawOpFields {
|
||||
|
||||
private final static List<String> stmtsFieldNames = List.of("op","ops","operation","statement","statements");
|
||||
private final static List<String> opsFieldNames = List.of("op","ops","operation","statement","statements");
|
||||
|
||||
private List<RawOpDef> rawOpDefs = new ArrayList<>();
|
||||
|
||||
public OpsOwner() {
|
||||
}
|
||||
|
||||
public List<RawOpDef> getRawStmtDefs() {
|
||||
public List<RawOpDef> getRawOpDefs() {
|
||||
return rawOpDefs;
|
||||
}
|
||||
|
||||
@ -43,7 +43,7 @@ public class OpsOwner extends RawOpFields {
|
||||
super.setFieldsByReflection(propsmap);
|
||||
|
||||
HashSet<String> found = new HashSet<>();
|
||||
for (String fname : stmtsFieldNames) {
|
||||
for (String fname : opsFieldNames) {
|
||||
if (propsmap.containsKey(fname)) {
|
||||
found.add(fname);
|
||||
}
|
||||
@ -52,15 +52,15 @@ public class OpsOwner extends RawOpFields {
|
||||
throw new BasicError("You used " + found + " as an op name, but only one of these is allowed.");
|
||||
}
|
||||
if (found.size()==1) {
|
||||
Object stmtsFieldValue = propsmap.remove(found.iterator().next());
|
||||
setStatementsFieldByType(stmtsFieldValue);
|
||||
Object opsFieldValue = propsmap.remove(found.iterator().next());
|
||||
setOpsFieldByType(opsFieldValue);
|
||||
}
|
||||
|
||||
super.setFieldsByReflection(propsmap);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void setStatementsFieldByType(Object object) {
|
||||
public void setOpsFieldByType(Object object) {
|
||||
if (object instanceof List) {
|
||||
List<Object> stmtList = (List<Object>) object;
|
||||
List<RawOpDef> defs = new ArrayList<>(stmtList.size());
|
||||
@ -105,9 +105,9 @@ public class OpsOwner extends RawOpFields {
|
||||
entries.getValue().getClass() + "', only maps and strings are recognized.");
|
||||
}
|
||||
}
|
||||
setStatementsFieldByType(itemizedMaps);
|
||||
setOpsFieldByType(itemizedMaps);
|
||||
} else if (object instanceof String) {
|
||||
setStatementsFieldByType(Map.of("stmt1", (String) object));
|
||||
setOpsFieldByType(Map.of("stmt1", (String) object));
|
||||
} else {
|
||||
throw new RuntimeException("Unknown object type: " + object.getClass());
|
||||
}
|
||||
|
||||
@ -22,15 +22,6 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* A statements doc can have both a list of statement blocks and/or a
|
||||
* list of statements. It can also have all the block parameters
|
||||
* assignable to {@link RawOpFields}.
|
||||
* <p>
|
||||
* The reason for having support both statements or statement blocks
|
||||
* is merely convenience. If you do not need or want to deal with the
|
||||
* full blocks format, the extra structure gets in the way.
|
||||
*/
|
||||
public class RawOpsDoc extends OpsOwner {
|
||||
|
||||
private RawScenarios scenarios = new RawScenarios();
|
||||
@ -43,7 +34,7 @@ public class RawOpsDoc extends OpsOwner {
|
||||
|
||||
public static RawOpsDoc forSingleStatement(String statement) {
|
||||
RawOpsDoc rawOpsDoc = new RawOpsDoc();
|
||||
rawOpsDoc.setStatementsFieldByType(statement);
|
||||
rawOpsDoc.setOpsFieldByType(statement);
|
||||
return rawOpsDoc;
|
||||
}
|
||||
|
||||
@ -100,19 +91,17 @@ public class RawOpsDoc extends OpsOwner {
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the list of statement blocks in this RawStmtsDoc.
|
||||
* If raw statements are defined on this RawStmtsDoc, then a single
|
||||
* StmtBlock containing those statements is prepended to the block list.
|
||||
* Otherwise, the list of StmtBlocks is returned as-is.
|
||||
* If raw ops are defined on this RawOpsDoc, then a single RawOpsBlock containing those op templates is prepended to
|
||||
* the block list. Otherwise, the list of RawOpsBlocks is returned as-is.
|
||||
*
|
||||
* @return all logical statement blocks containing statements
|
||||
* @return all logical ops blocks
|
||||
*/
|
||||
public List<RawOpsBlock> getBlocks() {
|
||||
List<RawOpsBlock> stmtBlocks = new ArrayList<>();
|
||||
if (!getRawStmtDefs().isEmpty()) {
|
||||
if (!getRawOpDefs().isEmpty()) {
|
||||
RawOpsBlock rawOpsBlock = new RawOpsBlock();
|
||||
rawOpsBlock.setName("block0");
|
||||
rawOpsBlock.setRawStmtDefs(getRawStmtDefs());
|
||||
rawOpsBlock.setRawStmtDefs(getRawOpDefs());
|
||||
stmtBlocks.add(rawOpsBlock);
|
||||
}
|
||||
stmtBlocks.addAll(this.blocks);
|
||||
|
||||
@ -35,14 +35,14 @@ public class RawOpsDocList {
|
||||
return new RawOpsDocList(List.of());
|
||||
}
|
||||
|
||||
public List<RawOpsDoc> getStmtsDocs() {
|
||||
public List<RawOpsDoc> getOpsDocs() {
|
||||
return rawOpsDocList;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
int docs = rawOpsDocList.size();
|
||||
int blocks = rawOpsDocList.stream().map(RawOpsDoc::getBlocks).mapToInt(List::size).sum();
|
||||
long optemplates = rawOpsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawStmtDefs().stream()).count();
|
||||
long optemplates = rawOpsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawOpDefs().stream()).count();
|
||||
return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates;
|
||||
}
|
||||
}
|
||||
|
||||
@ -95,7 +95,7 @@ public class RawOpsLoader {
|
||||
|
||||
newDocList.add(doc);
|
||||
} else {
|
||||
throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName());
|
||||
throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
|
||||
}
|
||||
}
|
||||
RawOpsDocList rawOpsDocList = new RawOpsDocList(newDocList);
|
||||
|
||||
@ -84,7 +84,7 @@ public class RawYamlLoader {
|
||||
if (object instanceof Map) {
|
||||
maps.add(new LinkedHashMap<>((Map<String,Object>)object));
|
||||
} else {
|
||||
throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName());
|
||||
throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
|
||||
}
|
||||
}
|
||||
return maps;
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -32,118 +32,17 @@ import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* <p>The OpTemplate is the developer's view of the operational templates that users
|
||||
* provide in YAML or some other structured format.</p>
|
||||
*
|
||||
* <H2>Terms</H2>
|
||||
* Within this documentation, the word <i>OpTemplate</i> will refer to the template API and
|
||||
* semantics. The word <i>user template</i> will refer to the configuration data as provided
|
||||
* by a user.
|
||||
*
|
||||
* <p>OpTemplates are the native Java representation of the user templates that specify how to
|
||||
* make an executable operation. OpTemplates are not created for each operation, but are used
|
||||
* to create an mostly-baked intermediate form commonly known as a <i>ready op</i>.
|
||||
* It is the intermediate form which is used to create an instance of an executable
|
||||
* op in whichever way is the most appropriate and efficient for a given driver.</p>
|
||||
*
|
||||
* <p>This class serves as the canonical documentation and API for how user templates
|
||||
* are mapped into a fully resolved OpTemplate. User-provided op templates can be
|
||||
* any basic data structure, and are often provided as part of a YAML workload file.
|
||||
* The description below will focus on structural rules rather than any particular
|
||||
* encoding format. The types used are fairly universal and easy to map from one
|
||||
* format to another.</p>
|
||||
*
|
||||
*
|
||||
* <p>A long-form introduction to this format is included in the main NoSQLBench docs
|
||||
* at <a href="http://docs.nosqlbench.io">docs.nosqlbench.io</a>
|
||||
* under the <I>Designing Workloads</I> section.</p>
|
||||
*
|
||||
* <p>A few structural variations are allowed -- No specific form enforced. The reasons for this are:
|
||||
* 1) It is generally obvious what as user wants to do from a given layout. 2) Data structure
|
||||
* markup is generally frustrating and difficult to troubleshoot. 3) The conceptual domain of
|
||||
* NB op construction is well-defined enough to avoid ambiguity.</p>
|
||||
*
|
||||
* <H2>Type Conventions</H2>
|
||||
*
|
||||
* For the purposes of simple interoperability, the types used at this interface boundary should
|
||||
* be limited to common scalar types -- numbers and strings, and simple structures like maps and lists.
|
||||
* The basic types defined for ECMAScript should eventually be supported, but no domain-specific
|
||||
* objects which would require special encoding or decoding rules should be used.
|
||||
*
|
||||
* <H2>Standard Properties</H2>
|
||||
*
|
||||
* Each op template can have these standard properties:
|
||||
* <UL>
|
||||
* <LI>name - every op template has a name, even if it is auto generated for you. This is used to
|
||||
* name errors in the log, to name metrics in telemetry, and so on.</LI>
|
||||
* <LI>description - an optional description, defaulted to "".</LI>
|
||||
* <LI>statement - An optional string value which represents an opaque form of the body of
|
||||
* an op template</LI>
|
||||
* <LI>params - A string-object map of zero or more named parameters, where the key is taken as the parameter
|
||||
* name and the value is any simple object form as limited by type conventions above.
|
||||
* <LI>bindings - A map of binding definitions, where the string key is taken as the anchor name, and the
|
||||
* string value is taken as the binding recipe.</LI>
|
||||
* <LI>tags - A map of tags, with string names and values</LI>
|
||||
* </UL>
|
||||
*
|
||||
* The user-provided definition of an op template should capture a blueprint of an operation to be executed by
|
||||
* a native driver. As such, you need either a statement or a set of params which can describe what
|
||||
* specific type should be constructed. The rules on building an executable operation are not enforced
|
||||
* by this API. Yet, responsible NB driver developers will clearly document what the rules
|
||||
* are for specifying each specific type of operation supported by an NB driver with examples in YAML format.
|
||||
*
|
||||
* <H2>OpTemplate Construction Rules</H2>
|
||||
*
|
||||
* <p>The available structural forms follow a basic set of rules for constructing the OpTemplate in a consistent way.
|
||||
* <OL>
|
||||
* <LI>A collection of user-provided op templates is provided as a string, a list or a map.</LI>
|
||||
* <LI>All maps are order-preserving, like {@link java.util.LinkedHashMap}</LI>
|
||||
* <LI>For maps, the keys are taken as the names of the op template instances.</LI>
|
||||
* <LI>The content of each op template can be provided as a string or as a map.</LI>
|
||||
* <OL>
|
||||
* <LI>If the op template entry is provided as a string, then the OpTemplate is constructed as having only a single
|
||||
* <i>statement</i> property (in addition to defaults within scope).
|
||||
* as provided by OpTemplate API.</LI>
|
||||
* <LI>If the op template entry is provided as a map, then the OpTemplate is constructed as having all of the
|
||||
* named properties defined in the standard properties above.
|
||||
* Any entry in the template which is not a reserved word is assigned to the params map as a parameter, in whatever structured
|
||||
* type is appropriate (scalar, lists, maps).</LI>
|
||||
* </LI>
|
||||
* <p>
|
||||
* The OpTemplate is a structurally normalized type which presents the user-provided op template to the NoSQLBench
|
||||
* loading and templating mechanisms. This type is not generally used directly for new driver development. It is the
|
||||
* backing data which is used by {@link io.nosqlbench.engine.api.templating.ParsedOp}, which is used in drivers to map
|
||||
* op templates to function to be used for a given cycle value.
|
||||
* </p>
|
||||
* </OL>
|
||||
*
|
||||
* <H2>Example Forms</H2>
|
||||
* The valid forms are shown below as examples.
|
||||
*
|
||||
* <H3>One String Statement</H3>
|
||||
* <pre>{@code
|
||||
* statement: statement
|
||||
* }</pre>
|
||||
*
|
||||
* <H3>List of Templates</H3>
|
||||
* <pre>{@code
|
||||
* statements:
|
||||
* - statement1
|
||||
* - statement2
|
||||
* }</pre>
|
||||
*
|
||||
* <H3>List of Maps</H3>
|
||||
* <pre>{@code
|
||||
* statements:
|
||||
* - name: name1
|
||||
* stmt: statement body
|
||||
* params:
|
||||
* p1: v1
|
||||
* p2: v2
|
||||
* }</pre>
|
||||
*
|
||||
* <H3>List Of Condensed Maps</H3>
|
||||
* <pre>{@code
|
||||
* statements:
|
||||
* - name1: statement body
|
||||
* p1: v1
|
||||
* p2: v2
|
||||
* }</pre>
|
||||
* <p>
|
||||
* This is part of the implementation of the NoSQLBench <em>Uniform Workload Specification</em>. Check the tests
|
||||
* for UniformWorkloadSpecification directly to see how this specification is tested and documented.
|
||||
* </p>
|
||||
*/
|
||||
public abstract class OpTemplate implements Tagged {
|
||||
|
||||
@ -170,6 +69,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
/**
|
||||
* Return a map of tags for this statement. Implementations are required to
|
||||
* add a tag for "name" automatically when this value is set during construction.
|
||||
*
|
||||
* @return A map of assigned tags for the op, with the name added as an auto-tag.
|
||||
*/
|
||||
public abstract Map<String, String> getTags();
|
||||
@ -208,7 +108,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
if (defaultValue.getClass().isAssignableFrom(value.getClass())) {
|
||||
return (V) value;
|
||||
} else {
|
||||
return NBTypeConverter.convertOr(value,defaultValue);
|
||||
return NBTypeConverter.convertOr(value, defaultValue);
|
||||
}
|
||||
|
||||
}
|
||||
@ -229,7 +129,6 @@ public abstract class OpTemplate implements Tagged {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public <V> V getParam(String name, Class<? extends V> type) {
|
||||
Object object = getParams().get(name);
|
||||
if (object == null) {
|
||||
@ -273,7 +172,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
*
|
||||
* @return an optional {@link ParsedStringTemplate}
|
||||
*/
|
||||
public Optional<ParsedStringTemplate> getParsed(Function<String,String>... rewriters) {
|
||||
public Optional<ParsedStringTemplate> getParsed(Function<String, String>... rewriters) {
|
||||
Optional<String> os = getStmt();
|
||||
return os.map(s -> {
|
||||
String result = s;
|
||||
@ -281,7 +180,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
result = rewriter.apply(result);
|
||||
}
|
||||
return result;
|
||||
}).map(s -> new ParsedStringTemplate(s,getBindings()));
|
||||
}).map(s -> new ParsedStringTemplate(s, getBindings()));
|
||||
}
|
||||
|
||||
public Optional<ParsedStringTemplate> getParsed() {
|
||||
@ -309,7 +208,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
fields.put(FIELD_TAGS, this.getTags());
|
||||
}
|
||||
|
||||
this.getOp().ifPresent(o -> fields.put(FIELD_OP,o));
|
||||
this.getOp().ifPresent(o -> fields.put(FIELD_OP, o));
|
||||
|
||||
fields.put(FIELD_NAME, this.getName());
|
||||
|
||||
@ -320,10 +219,11 @@ public abstract class OpTemplate implements Tagged {
|
||||
* Legacy support for String form statements. This is left here as a convenience method,
|
||||
* however it is changed to an Optional to force caller refactorings.
|
||||
*
|
||||
* @return An optional string version of the op, empty if there is no 'stmt' property in the op fields, or no op fields at all.
|
||||
* @return An optional string version of the op, empty if there is no 'stmt' property in the op fields, or no op
|
||||
* fields at all.
|
||||
*/
|
||||
public Optional<String> getStmt() {
|
||||
return getOp().map(m->m.get("stmt")).map(s->{
|
||||
return getOp().map(m -> m.get("stmt")).map(s -> {
|
||||
if (s instanceof CharSequence) {
|
||||
return s.toString();
|
||||
} else {
|
||||
@ -333,7 +233,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
}
|
||||
|
||||
public Element getParamReader() {
|
||||
return NBParams.one(getName(),getParams());
|
||||
return NBParams.one(getName(), getParams());
|
||||
}
|
||||
|
||||
/**
|
||||
@ -347,13 +247,13 @@ public abstract class OpTemplate implements Tagged {
|
||||
* @return the map of all remaining fields from the op template and the params map.
|
||||
*/
|
||||
public Map<String, Object> remainingFields() {
|
||||
Map<String,Object> remaining = new LinkedHashMap<>(getOp().orElse(Map.of()));
|
||||
Map<String, Object> remaining = new LinkedHashMap<>(getOp().orElse(Map.of()));
|
||||
remaining.putAll(getParams());
|
||||
return remaining;
|
||||
}
|
||||
|
||||
public void assertConsumed() {
|
||||
if (size()>0) {
|
||||
if (size() > 0) {
|
||||
throw new OpConfigError("The op template named '" + getName() + "' was not fully consumed. These fields are not being applied:" + remainingFields());
|
||||
}
|
||||
}
|
||||
|
||||
@ -40,11 +40,11 @@ public class OpsBlock implements Tagged, Iterable<OpTemplate> {
|
||||
public List<OpTemplate> getOps() {
|
||||
|
||||
List<OpTemplate> rawOpTemplates = new ArrayList<>();
|
||||
List<RawOpDef> statements = rawOpsBlock.getRawStmtDefs();
|
||||
List<RawOpDef> opDefs = rawOpsBlock.getRawOpDefs();
|
||||
|
||||
for (int i = 0; i < statements.size(); i++) {
|
||||
for (int i = 0; i < opDefs.size(); i++) {
|
||||
rawOpTemplates.add(
|
||||
new OpDef(this, statements.get(i))
|
||||
new OpDef(this, opDefs.get(i))
|
||||
);
|
||||
}
|
||||
return rawOpTemplates;
|
||||
|
||||
@ -27,7 +27,7 @@ import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* StmtsDoc creates a logical view of a statements doc that includes
|
||||
* OpsDoc creates a logical view of a RawOpsDoc doc that includes
|
||||
* all inherited and overridden values for bindings, tags, and params.
|
||||
*/
|
||||
public class OpsDoc implements Tagged, Iterable<OpsBlock> {
|
||||
@ -86,10 +86,10 @@ public class OpsDoc implements Tagged, Iterable<OpsBlock> {
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The list of all included statements for all included block in this document,
|
||||
* including the inherited and overridden values from the this doc and the parent block.
|
||||
* @return The list of all included op templates for all included block in this document,
|
||||
* including the inherited and overridden values from this doc and the parent block.
|
||||
*/
|
||||
public List<OpTemplate> getStmts() {
|
||||
public List<OpTemplate> getOpTemplates() {
|
||||
return getBlocks().stream().flatMap(b -> b.getOps().stream()).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@ -53,26 +53,26 @@ public class OpsDocList implements Iterable<OpsDoc> {
|
||||
}
|
||||
|
||||
public List<OpsDoc> getStmtDocs() {
|
||||
return rawOpsDocList.getStmtsDocs().stream()
|
||||
return rawOpsDocList.getOpsDocs().stream()
|
||||
.map(OpsDoc::new)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<OpTemplate> getStmts() {
|
||||
return getStmts("");
|
||||
public List<OpTemplate> getOps() {
|
||||
return getOps("");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param tagFilterSpec a comma-separated tag filter spec
|
||||
* @return The list of all included statements for all included blocks of in this document,
|
||||
* including the inherited and overridden values from the this doc and the parent block.
|
||||
* @return The list of all included op templates for all included blocks of in this document,
|
||||
* including the inherited and overridden values from this doc and the parent block.
|
||||
*/
|
||||
public List<OpTemplate> getStmts(String tagFilterSpec) {
|
||||
public List<OpTemplate> getOps(String tagFilterSpec) {
|
||||
TagFilter ts = new TagFilter(tagFilterSpec);
|
||||
List<OpTemplate> opTemplates = new ArrayList<>();
|
||||
|
||||
getStmtDocs().stream()
|
||||
.flatMap(d -> d.getStmts().stream())
|
||||
.flatMap(d -> d.getOpTemplates().stream())
|
||||
.filter(ts::matchesTagged)
|
||||
.forEach(opTemplates::add);
|
||||
|
||||
@ -155,8 +155,6 @@ public class OpsDocList implements Iterable<OpsDoc> {
|
||||
}
|
||||
|
||||
sb.append("docs: " + docscount + " blocks:" + blockscount + " ops:" + opscount);
|
||||
// String names = this.rawStmtsDocList.getStmtsDocs().stream().flatMap(sd -> sd.getRawStmtDefs().stream()).map(d->d.getName()).collect(Collectors.joining(","));
|
||||
// sb.append(", names:").append(names);
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@ -166,7 +164,7 @@ public class OpsDocList implements Iterable<OpsDoc> {
|
||||
};
|
||||
|
||||
public Pattern getVersionRegex() {
|
||||
List<RawOpsDoc> stmtDocs = rawOpsDocList.getStmtsDocs();
|
||||
List<RawOpsDoc> stmtDocs = rawOpsDocList.getOpsDocs();
|
||||
return Pattern.compile(stmtDocs.size()>0 ? stmtDocs.get(0).getVersionRegex() : ".*");
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -31,34 +31,24 @@ import java.util.function.Function;
|
||||
* a client API for advanced testing. The default instance should simply be named
|
||||
* {@code default}</p>
|
||||
*
|
||||
* <p>Most native drivers use some combination of fluent, functional, and declarative
|
||||
* patterns. These usually require you to keep access to a set of core
|
||||
* state-holding objects in order to construct new elements to drive operations with.
|
||||
* An example of this would be creating a statement from a session. It is necessary
|
||||
* to keep the session around in for when you create new statements. Maintaining
|
||||
* the session object is considered an essential part of idiomatic and efficient
|
||||
* use of the API. Further, you may have builders or factories that are created
|
||||
* from the session which should be cached as well. Keeping all these objects
|
||||
* together requires attaching them to a cohesive owning object -- That is the space
|
||||
* cache.</p>
|
||||
* <p>Most native drivers use some combination of fluent, functional, and declarative patterns. These usually require
|
||||
* you to keep access to a set of core state-holding objects in order to construct new elements to drive operations
|
||||
* with. An example of this would be creating an executable operation from a session object. It is necessary to keep the
|
||||
* session around in for when you create new statements. Maintaining the session object is considered an essential part
|
||||
* of idiomatic and efficient use of the API. Further, you may have builders or factories that are created from the
|
||||
* session which should be cached as well. Keeping all these objects together requires attaching them to a cohesive
|
||||
* owning object -- That is the space cache.</p>
|
||||
*
|
||||
* <p>You might want to create multiple session contexts in order to test out
|
||||
* non-trivial behavior in advanced testing scenarios. To do this dynamically, i.e.
|
||||
* acquire some named space cache, simply call the {@link #get(String)}</p> method
|
||||
* with the name of the space you want to use. This value can be provided as a
|
||||
* dynamic field in your op mapping ({@link io.nosqlbench.engine.api.activityimpl.OpMapper})
|
||||
* or synthesis ({@link io.nosqlbench.engine.api.activityimpl.OpDispenser})
|
||||
* implementation.
|
||||
*
|
||||
* @param <S> The type which will represent the cache for a given type of adapter.
|
||||
* @param <S>
|
||||
* The type which will represent the cache for a given type of adapter.
|
||||
*/
|
||||
public class DriverSpaceCache<S> {
|
||||
|
||||
private final ConcurrentHashMap<String,S> cache = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, S> cache = new ConcurrentHashMap<>();
|
||||
|
||||
private final Function<String, S> newSpaceFunction;
|
||||
|
||||
public DriverSpaceCache(Function<String,S> newSpaceFunction) {
|
||||
public DriverSpaceCache(Function<String, S> newSpaceFunction) {
|
||||
this.newSpaceFunction = newSpaceFunction;
|
||||
}
|
||||
|
||||
@ -66,7 +56,7 @@ public class DriverSpaceCache<S> {
|
||||
return cache.computeIfAbsent(name, newSpaceFunction);
|
||||
}
|
||||
|
||||
public Map<String,S> getElements() {
|
||||
public Map<String, S> getElements() {
|
||||
return Collections.unmodifiableMap(cache);
|
||||
}
|
||||
|
||||
|
||||
@ -24,22 +24,27 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* In the case that no statements are provided for an activity, but a workload
|
||||
* description is provided, and the result for no found statements is not
|
||||
* simply a matter of tag filtering, an activity may provide its own
|
||||
* synthetic ops. This is here primarily to support the classic behavior of stdout
|
||||
* until it's functionality is subsumed by standard diagnostic features.
|
||||
*
|
||||
* Note that this is only valid while an activity uses a single driver, which will
|
||||
* change with upcoming API updates.
|
||||
* It is possible for a DriverAdapter to create op templates using partial information.
|
||||
* For example, the stdout driver can use only bindings to create CSV or JSON style data renderings.
|
||||
* This mechanism is only triggered when:
|
||||
* <OL>
|
||||
* <LI>No op templates were provided.</LI>
|
||||
* <LI>The default driver for an activity implements this method.</LI>
|
||||
* </OL>
|
||||
* <p>
|
||||
* This excludes cases where a workload was provided with op templates, but they were all filtered out. In that case,
|
||||
* the user should be informed of an error.
|
||||
*/
|
||||
public interface SyntheticOpTemplateProvider extends DriverAdapterDecorators {
|
||||
|
||||
/**
|
||||
* If a driver adapter supports creating example op templates from bindings,
|
||||
* it must implement this method to do so.
|
||||
* @param opsDocList The existing doc structure, which should contain no fully defined op templates, but may contain other elements like bindings
|
||||
*
|
||||
* @param opsDocList
|
||||
* The existing doc structure, which should contain no fully defined op templates, but may contain other
|
||||
* elements like bindings
|
||||
* @return A list of op templates, size zero or more
|
||||
*/
|
||||
List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String,Object> params);
|
||||
List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> params);
|
||||
}
|
||||
|
||||
@ -1,34 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.engine.api.templating;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.api.config.standard.NBConfiguration;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* An Op Template Supplier can provide its own source of op templates instead
|
||||
* of relying on the built-in mechanism. By default, the built-in mechanism
|
||||
* will read op definitions from parameters first, then any ops (statements)
|
||||
* from yaml files provided in the workload= activity parameters.
|
||||
*/
|
||||
public interface OpTemplateSupplier extends DriverAdapterDecorators {
|
||||
|
||||
Optional<List<OpTemplate>> loadOpTemplates(NBConfiguration cfg);
|
||||
}
|
||||
@ -29,12 +29,12 @@ public class OpsLoaderTest {
|
||||
|
||||
@Test
|
||||
public void testTemplateVarSubstitution() {
|
||||
OpsDocList stmtsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources");
|
||||
assertThat(stmtsDocs).isNotNull();
|
||||
List<OpsDoc> docs = stmtsDocs.getStmtDocs();
|
||||
OpsDocList opsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources");
|
||||
assertThat(opsDocs).isNotNull();
|
||||
List<OpsDoc> docs = opsDocs.getStmtDocs();
|
||||
assertThat(docs).hasSize(1);
|
||||
OpsDoc stmtsBlocks = docs.get(0);
|
||||
Map<String, String> bindings = stmtsBlocks.getBindings();
|
||||
OpsDoc opsDoc = docs.get(0);
|
||||
Map<String, String> bindings = opsDoc.getBindings();
|
||||
assertThat(bindings).isEqualTo(Map.of(
|
||||
"b1a","Prefix(\"prefix\")",
|
||||
"b1b","Prefix(\"prefix\")",
|
||||
|
||||
@ -37,7 +37,7 @@ public class OpDefTest {
|
||||
@Test
|
||||
public void testLayering() {
|
||||
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
|
||||
assertThat(all).isNotNull();
|
||||
assertThat(all.getStmtDocs()).hasSize(2);
|
||||
OpsDoc doc1 = all.getStmtDocs().get(0);
|
||||
@ -64,7 +64,7 @@ public class OpDefTest {
|
||||
|
||||
@Test
|
||||
public void testStatementRendering() {
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
|
||||
assertThat(all).isNotNull();
|
||||
assertThat(all.getStmtDocs()).hasSize(2);
|
||||
OpsDoc doc1 = all.getStmtDocs().get(0);
|
||||
@ -79,10 +79,10 @@ public class OpDefTest {
|
||||
|
||||
@Test
|
||||
public void testConsumableMapState() {
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
|
||||
List<OpsDoc> docs = all.getStmtDocs();
|
||||
OpsDoc block1 = docs.get(1);
|
||||
List<OpTemplate> stmts = block1.getStmts();
|
||||
List<OpTemplate> stmts = block1.getOpTemplates();
|
||||
OpTemplate stmt0 = stmts.get(0);
|
||||
OpTemplate stmt1 = stmts.get(1);
|
||||
assertThat(stmt0.getParams()).containsAllEntriesOf(Map.of("timeout", 23423, "foobar", "baz"));
|
||||
@ -94,7 +94,7 @@ public class OpDefTest {
|
||||
|
||||
@Test
|
||||
public void testMapOfMaps() {
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of());
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
|
||||
List<OpsDoc> docs = all.getStmtDocs();
|
||||
OpsDoc doc0 = docs.get(0);
|
||||
assertThat(doc0.getName()).isEqualTo("map-of-maps");
|
||||
@ -115,7 +115,7 @@ public class OpDefTest {
|
||||
|
||||
@Test
|
||||
public void testBasicStringStmt() {
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of());
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
|
||||
List<OpsDoc> docs = all.getStmtDocs();
|
||||
OpsDoc doc1 = docs.get(1);
|
||||
assertThat(doc1.getName()).isEqualTo("string-statement");
|
||||
@ -130,7 +130,7 @@ public class OpDefTest {
|
||||
|
||||
@Test
|
||||
public void testListOfNamedMap() {
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of());
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
|
||||
List<OpsDoc> docs = all.getStmtDocs();
|
||||
OpsDoc doc2 = docs.get(2);
|
||||
assertThat(doc2.getName()).isEqualTo("list-of-named-map");
|
||||
|
||||
@ -28,15 +28,15 @@ import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class StmtEscapingTest {
|
||||
public class OpEscapingTest {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(StmtEscapingTest.class);
|
||||
private final static Logger logger = LogManager.getLogger(OpEscapingTest.class);
|
||||
private static List<OpTemplate> defs;
|
||||
|
||||
@BeforeAll
|
||||
public static void testLayering() {
|
||||
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/escaped_stmts.yaml", Map.of());
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/escaped_ops.yaml", Map.of());
|
||||
assertThat(all).isNotNull();
|
||||
assertThat(all.getStmtDocs()).hasSize(1);
|
||||
OpsDoc doc1 = all.getStmtDocs().get(0);
|
||||
@ -24,89 +24,93 @@ import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class StmtVariationTests {
|
||||
public class OpVariationTests {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(StmtVariationTests.class);
|
||||
private final static Logger logger = LogManager.getLogger(OpVariationTests.class);
|
||||
|
||||
@Test
|
||||
public void testListStmtsOnly() {
|
||||
public void testListOpsOnly() {
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList docs = ysl.loadString(
|
||||
"statements:\n" +
|
||||
" - first statement\n" +
|
||||
" - second statement\n"
|
||||
"ops:\n" +
|
||||
" - first op\n" +
|
||||
" - second op\n"
|
||||
);
|
||||
|
||||
assertThat(docs.getStmtsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getStmtsDocs().get(0);
|
||||
assertThat(doc.getRawStmtDefs()).hasSize(2);
|
||||
List<RawOpDef> stmts = doc.getRawStmtDefs();
|
||||
RawOpDef s0 = stmts.get(0);
|
||||
assertThat(docs.getOpsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getOpsDocs().get(0);
|
||||
assertThat(doc.getRawOpDefs()).hasSize(2);
|
||||
List<RawOpDef> ops = doc.getRawOpDefs();
|
||||
RawOpDef s0 = ops.get(0);
|
||||
assertThat(s0.getName()).isEqualTo("stmt1");
|
||||
assertThat(s0.getStmt()).isEqualTo("first statement");
|
||||
RawOpDef s1 = stmts.get(1);
|
||||
assertThat(s0.getStmt()).isEqualTo("first op");
|
||||
RawOpDef s1 = ops.get(1);
|
||||
assertThat(s1.getName()).isEqualTo("stmt2");
|
||||
assertThat(s1.getStmt()).isEqualTo("second statement");
|
||||
assertThat(s1.getStmt()).isEqualTo("second op");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingleEntryMapStmtsOnly() {
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList docs = ysl.loadString(
|
||||
"statements:\n" +
|
||||
" - s1: statement one\n" +
|
||||
" - s2: statement two\n"
|
||||
"ops:\n" +
|
||||
" - s1: op one\n" +
|
||||
" - s2: op two\n"
|
||||
);
|
||||
assertThat(docs.getStmtsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getStmtsDocs().get(0);
|
||||
assertThat(doc.getRawStmtDefs()).hasSize(2);
|
||||
List<RawOpDef> stmts = doc.getRawStmtDefs();
|
||||
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(stmts.get(0).getName()).isEqualTo("s1");
|
||||
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
|
||||
assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(stmts.get(1).getName()).isEqualTo("s2");
|
||||
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
|
||||
assertThat(docs.getOpsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getOpsDocs().get(0);
|
||||
assertThat(doc.getRawOpDefs()).hasSize(2);
|
||||
List<RawOpDef> ops = doc.getRawOpDefs();
|
||||
assertThat(ops.get(0)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(ops.get(0).getName()).isEqualTo("s1");
|
||||
assertThat(ops.get(0).getStmt()).isEqualTo("op one");
|
||||
assertThat(ops.get(1)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(ops.get(1).getName()).isEqualTo("s2");
|
||||
assertThat(ops.get(1).getStmt()).isEqualTo("op two");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMapStmtsOnly() {
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList docs = ysl.loadString(
|
||||
"statements:\n" +
|
||||
" - name: s1\n" +
|
||||
" stmt: statement one\n" +
|
||||
" - name: s2\n" +
|
||||
" stmt: statement two\n"
|
||||
"ops:\n" +
|
||||
" - name: s1\n" +
|
||||
" stmt: op one\n" +
|
||||
" - name: s2\n" +
|
||||
" stmt: op two\n"
|
||||
);
|
||||
assertThat(docs.getStmtsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getStmtsDocs().get(0);
|
||||
assertThat(doc.getRawStmtDefs()).hasSize(2);
|
||||
List<RawOpDef> stmts = doc.getRawStmtDefs();
|
||||
assertThat(docs.getOpsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getOpsDocs().get(0);
|
||||
assertThat(doc.getRawOpDefs()).hasSize(2);
|
||||
List<RawOpDef> stmts = doc.getRawOpDefs();
|
||||
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(stmts.get(0).getName()).isEqualTo("s1");
|
||||
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
|
||||
assertThat(stmts.get(0).getStmt()).isEqualTo("op one");
|
||||
assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(stmts.get(1).getName()).isEqualTo("s2");
|
||||
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
|
||||
assertThat(stmts.get(1).getStmt()).isEqualTo("op two");
|
||||
}
|
||||
|
||||
/**
|
||||
* This test uses the compatible names for the sake of demonstration. Users should know that they can use op and
|
||||
* statement terms interchangeably in op templates.
|
||||
*/
|
||||
@Test
|
||||
public void testMixedForms() {
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList docs = ysl.loadString(
|
||||
"statement:\n" +
|
||||
" - name: s1\n" +
|
||||
" stmt: statement one\n" +
|
||||
" - statement two\n" +
|
||||
" - s3: statement three\n" +
|
||||
" - ST4: statement four\n" +
|
||||
" type: organic\n"
|
||||
" - name: s1\n" +
|
||||
" stmt: statement one\n" +
|
||||
" - statement two\n" +
|
||||
" - s3: statement three\n" +
|
||||
" - ST4: statement four\n" +
|
||||
" type: organic\n"
|
||||
);
|
||||
assertThat(docs.getStmtsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getStmtsDocs().get(0);
|
||||
assertThat(doc.getRawStmtDefs()).hasSize(4);
|
||||
List<RawOpDef> stmts = doc.getRawStmtDefs();
|
||||
assertThat(docs.getOpsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getOpsDocs().get(0);
|
||||
assertThat(doc.getRawOpDefs()).hasSize(4);
|
||||
List<RawOpDef> stmts = doc.getRawOpDefs();
|
||||
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(stmts.get(0).getName()).isEqualTo("s1");
|
||||
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
|
||||
@ -32,9 +32,9 @@ public class RawYamlTemplateLoaderTest {
|
||||
public void testLoadPropertiesBlock() {
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList rawBlockDocs = ysl.loadPath("testdocs/rawblock.yaml");
|
||||
assertThat(rawBlockDocs.getStmtsDocs()).hasSize(1);
|
||||
RawOpsDoc rawBlockDoc = rawBlockDocs.getStmtsDocs().get(0);
|
||||
assertThat(rawBlockDoc.getRawStmtDefs()).hasSize(1);
|
||||
assertThat(rawBlockDocs.getOpsDocs()).hasSize(1);
|
||||
RawOpsDoc rawBlockDoc = rawBlockDocs.getOpsDocs().get(0);
|
||||
assertThat(rawBlockDoc.getRawOpDefs()).hasSize(1);
|
||||
assertThat(rawBlockDoc.getBindings()).hasSize(1);
|
||||
assertThat(rawBlockDoc.getName()).isEqualTo("name");
|
||||
assertThat(rawBlockDoc.getTags()).hasSize(1);
|
||||
@ -44,8 +44,8 @@ public class RawYamlTemplateLoaderTest {
|
||||
@Test
|
||||
public void testLoadFullFormat() {
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_stmts.yaml");
|
||||
List<RawOpsDoc> rawOpsDocs = erthing.getStmtsDocs();
|
||||
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_ops.yaml");
|
||||
List<RawOpsDoc> rawOpsDocs = erthing.getOpsDocs();
|
||||
assertThat(rawOpsDocs).hasSize(2);
|
||||
RawOpsDoc rawOpsDoc = rawOpsDocs.get(0);
|
||||
List<RawOpsBlock> blocks = rawOpsDoc.getBlocks();
|
||||
@ -58,8 +58,8 @@ public class RawYamlTemplateLoaderTest {
|
||||
@Test
|
||||
public void testLoadScenarios() {
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_stmts.yaml");
|
||||
List<RawOpsDoc> rawOpsDocs = erthing.getStmtsDocs();
|
||||
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_ops.yaml");
|
||||
List<RawOpsDoc> rawOpsDocs = erthing.getOpsDocs();
|
||||
assertThat(rawOpsDocs).hasSize(2);
|
||||
RawOpsDoc rawOpsDoc = rawOpsDocs.get(0);
|
||||
List<RawOpsBlock> blocks = rawOpsDoc.getBlocks();
|
||||
|
||||
@ -109,7 +109,7 @@ public class YamlSpecValidator implements STAssemblyValidator {
|
||||
List<Map<String, Object>> expectedList = gson.fromJson(json, type);
|
||||
|
||||
OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
|
||||
List<OpTemplate> stmts = stmtsDocs.getStmts();
|
||||
List<OpTemplate> stmts = stmtsDocs.getOps();
|
||||
List<Map<String, Object>> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList());
|
||||
|
||||
try {
|
||||
@ -143,8 +143,6 @@ public class YamlSpecValidator implements STAssemblyValidator {
|
||||
private void validateYamlWithJson(String desc, String yaml, String json, STNodeReference testset, boolean debug) {
|
||||
System.out.format("%-40s", "- checking yaml->json");
|
||||
|
||||
// StmtsDocList stmts = StatementsLoader.loadString(yaml);
|
||||
|
||||
try {
|
||||
List<Map<String, Object>> docmaps = new RawYamlLoader().loadString(logger, yaml);
|
||||
JsonElement elem = null;
|
||||
|
||||
@ -32,7 +32,7 @@ public class OpDetailOverrideTest {
|
||||
@Test
|
||||
public void testStmtOverrides() {
|
||||
|
||||
OpsDocList doclist = OpsLoader.loadPath("testdocs/stmt_details.yaml", Map.of());
|
||||
OpsDocList doclist = OpsLoader.loadPath("testdocs/op_details.yaml", Map.of());
|
||||
|
||||
assertThat(doclist).isNotNull();
|
||||
|
||||
@ -50,24 +50,24 @@ public class OpDetailOverrideTest {
|
||||
assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","block0--stmt1"));
|
||||
|
||||
OpsBlock doc1block1 = doc1.getBlocks().get(1);
|
||||
List<OpTemplate> stmts = doc1block1.getOps();
|
||||
assertThat(stmts).hasSize(4);
|
||||
List<OpTemplate> ops = doc1block1.getOps();
|
||||
assertThat(ops).hasSize(4);
|
||||
|
||||
s = stmts.get(0);
|
||||
s = ops.get(0);
|
||||
assertThat(s.getName()).isEqualTo("testblock1--stmt1");
|
||||
assertThat(s.getStmt()).contains("astatement1");
|
||||
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--stmt1"));
|
||||
assertThat(s.getBindings()).hasSize(1);
|
||||
assertThat(s.getParams()).hasSize(1);
|
||||
|
||||
s = stmts.get(1);
|
||||
s = ops.get(1);
|
||||
assertThat(s.getName()).isEqualTo("testblock1--s2name");
|
||||
assertThat(s.getStmt()).contains("s2statement data");
|
||||
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s2name"));
|
||||
assertThat(s.getBindings()).hasSize(1);
|
||||
assertThat(s.getParams()).hasSize(1);
|
||||
|
||||
s = stmts.get(2);
|
||||
s = ops.get(2);
|
||||
assertThat(s.getName()).isEqualTo("testblock1--s3");
|
||||
assertThat(s.getStmt()).contains("statement three");
|
||||
assertThat(s.getTags()).containsEntry("tname1", "tval1");
|
||||
@ -75,7 +75,7 @@ public class OpDetailOverrideTest {
|
||||
assertThat(s.getBindings()).hasSize(3);
|
||||
assertThat(s.getParams()).hasSize(2);
|
||||
|
||||
s = stmts.get(3);
|
||||
s = ops.get(3);
|
||||
assertThat(s.getName()).isEqualTo("testblock1--s4");
|
||||
assertThat(s.getStmt()).contains("statement 4");
|
||||
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s4"));
|
||||
|
||||
@ -50,7 +50,7 @@ public class OpsDocListTest {
|
||||
|
||||
@BeforeAll
|
||||
public static void testLoadYaml() {
|
||||
doclist = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
|
||||
doclist = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -76,21 +76,21 @@ public class OpsDocListTest {
|
||||
@Test
|
||||
public void testStmtInheritsBlockData() {
|
||||
OpsDoc doc0 = doclist.getStmtDocs().get(0);
|
||||
List<OpTemplate> stmts1 = doc0.getBlocks().get(0).getOps();
|
||||
assertThat(stmts1).hasSize(2);
|
||||
List<OpTemplate> ops1 = doc0.getBlocks().get(0).getOps();
|
||||
assertThat(ops1).hasSize(2);
|
||||
|
||||
OpsBlock block0 = doc0.getBlocks().get(0);
|
||||
assertThat(block0.getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
|
||||
assertThat(block0.getParams()).containsExactly(MapEntry.entry("param1","value1"));
|
||||
assertThat(block0.getTags()).containsExactly(MapEntry.entry("atagname","atagvalue"));
|
||||
|
||||
assertThat(stmts1.get(0).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
|
||||
assertThat(stmts1.get(0).getParams()).containsExactly(MapEntry.entry("param1","value1"));
|
||||
assertThat(stmts1.get(0).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt1","block","doc1--block0"));
|
||||
assertThat(ops1.get(0).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
|
||||
assertThat(ops1.get(0).getParams()).containsExactly(MapEntry.entry("param1","value1"));
|
||||
assertThat(ops1.get(0).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt1","block","doc1--block0"));
|
||||
|
||||
assertThat(stmts1.get(1).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
|
||||
assertThat(stmts1.get(1).getParams()).containsExactly(MapEntry.entry("param1","value1"));
|
||||
assertThat(stmts1.get(1).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt2","block","doc1--block0"));
|
||||
assertThat(ops1.get(1).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
|
||||
assertThat(ops1.get(1).getParams()).containsExactly(MapEntry.entry("param1","value1"));
|
||||
assertThat(ops1.get(1).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt2","block","doc1--block0"));
|
||||
|
||||
}
|
||||
|
||||
@ -120,15 +120,15 @@ public class OpsDocListTest {
|
||||
@Test
|
||||
public void testStmtsGetter() {
|
||||
OpsDoc doc1 = doclist.getStmtDocs().get(1);
|
||||
List<OpTemplate> stmts = doc1.getStmts();
|
||||
List<OpTemplate> stmts = doc1.getOpTemplates();
|
||||
assertThat(stmts).hasSize(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFilteredStmts() {
|
||||
List<OpTemplate> stmts = doclist.getStmts("");
|
||||
List<OpTemplate> stmts = doclist.getOps("");
|
||||
assertThat(stmts).hasSize(6);
|
||||
stmts = doclist.getStmts("root1:value23");
|
||||
stmts = doclist.getOps("root1:value23");
|
||||
assertThat(stmts).hasSize(2);
|
||||
}
|
||||
|
||||
|
||||
@ -70,8 +70,8 @@ public class ParsedOpTest {
|
||||
ps1: "param-one"
|
||||
""";
|
||||
OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null);
|
||||
assertThat(stmtsDocs.getStmts().size()).isEqualTo(1);
|
||||
OpTemplate opTemplate = stmtsDocs.getStmts().get(0);
|
||||
assertThat(stmtsDocs.getOps().size()).isEqualTo(1);
|
||||
OpTemplate opTemplate = stmtsDocs.getOps().get(0);
|
||||
ParsedOp parsedOp = new ParsedOp(opTemplate, cfg);
|
||||
|
||||
assertThat(parsedOp.getAsFunctionOr("d1","invalid").apply(1L)).isEqualTo("one");
|
||||
|
||||
@ -551,12 +551,12 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
|
||||
|
||||
OpsDocList opsDocList = loadStmtsDocList();
|
||||
|
||||
List<OpTemplate> unfilteredOps = opsDocList.getStmts();
|
||||
List<OpTemplate> filteredOps = opsDocList.getStmts(tagfilter);
|
||||
List<OpTemplate> unfilteredOps = opsDocList.getOps();
|
||||
List<OpTemplate> filteredOps = opsDocList.getOps(tagfilter);
|
||||
|
||||
if (filteredOps.size() == 0) {
|
||||
if (unfilteredOps.size() > 0) { // There were no ops, and it was because they were all filtered out
|
||||
throw new BasicError("There were no active statements with tag filter '"
|
||||
throw new BasicError("There were no active op templates with tag filter '"
|
||||
+ tagfilter + "', since all " + unfilteredOps.size() + " were filtered out.");
|
||||
} else {
|
||||
// There were no ops, and it *wasn't* because they were all filtered out.
|
||||
@ -578,7 +578,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
|
||||
}
|
||||
}
|
||||
if (filteredOps.size() == 0) {
|
||||
throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'");
|
||||
throw new BasicError("There were no active op templates with tag filter '" + tagfilter + "'");
|
||||
}
|
||||
}
|
||||
|
||||
@ -613,8 +613,8 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
|
||||
* taken as the only provided statement.</LI>
|
||||
* <LI>If a 'yaml, or 'workload' parameter is provided, then the statements in that file
|
||||
* are taken with their ratios </LI>
|
||||
* <LI>Any provided tags filter is used to select only the statements which have matching
|
||||
* tags. If no tags are provided, then all the found statements are included.</LI>
|
||||
* <LI>Any provided tags filter is used to select only the op templates which have matching
|
||||
* tags. If no tags are provided, then all the found op templates are included.</LI>
|
||||
* <LI>The ratios and the 'seq' parameter are used to build a sequence of the ready operations,
|
||||
* where the sequence length is the sum of the ratios.</LI>
|
||||
* </OL>
|
||||
|
||||
@ -35,11 +35,11 @@ public class CommandTemplateTest {
|
||||
|
||||
@Test
|
||||
public void testCommandTemplate() {
|
||||
OpsDocList stmtsDocs = OpsLoader.loadString("" +
|
||||
"statements:\n" +
|
||||
OpsDocList opsDocs = OpsLoader.loadString("" +
|
||||
"ops:\n" +
|
||||
" - s1: test1=foo test2=bar",
|
||||
OpTemplateFormat.yaml, Map.of(), null);
|
||||
OpTemplate optpl = stmtsDocs.getStmts().get(0);
|
||||
OpTemplate optpl = opsDocs.getOps().get(0);
|
||||
CommandTemplate ct = new CommandTemplate(optpl);
|
||||
assertThat(ct.isStatic()).isTrue();
|
||||
}
|
||||
@ -48,13 +48,13 @@ public class CommandTemplateTest {
|
||||
public void testCommandTemplateFormat() {
|
||||
Gson gson = new GsonBuilder().setPrettyPrinting().create();
|
||||
OpsDocList stmtsDocs = OpsLoader.loadString("" +
|
||||
"statements:\n" +
|
||||
"ops:\n" +
|
||||
" - s1: test1=foo test2={bar}\n" +
|
||||
" bindings:\n" +
|
||||
" bar: NumberNameToString();\n",
|
||||
OpTemplateFormat.yaml, Map.of(), null
|
||||
);
|
||||
OpTemplate optpl = stmtsDocs.getStmts().get(0);
|
||||
OpTemplate optpl = stmtsDocs.getOps().get(0);
|
||||
CommandTemplate ct = new CommandTemplate(optpl);
|
||||
String format = gson.toJson(ct);
|
||||
logger.debug(format);
|
||||
|
||||
@ -34,12 +34,12 @@ public class NBCLIScenarioParserTemplateVarTest {
|
||||
cmds.forEach(System.out::println);
|
||||
|
||||
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
|
||||
OpTemplate optpl1 = workload1.getStmts().get(0);
|
||||
OpTemplate optpl1 = workload1.getOps().get(0);
|
||||
System.out.println("op from cmd1:"+optpl1);
|
||||
assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n");
|
||||
|
||||
OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArg("workload"),cmds.get(1).getParams());
|
||||
OpTemplate optpl2 = workload2.getStmts().get(0);
|
||||
OpTemplate optpl2 = workload2.getOps().get(0);
|
||||
System.out.println("op from cmd2:"+optpl2);
|
||||
assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n");
|
||||
}
|
||||
@ -51,7 +51,7 @@ public class NBCLIScenarioParserTemplateVarTest {
|
||||
cmds.forEach(System.out::println);
|
||||
|
||||
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
|
||||
OpTemplate optpl1 = workload1.getStmts().get(0);
|
||||
OpTemplate optpl1 = workload1.getOps().get(0);
|
||||
System.out.println("op from cmd1:"+optpl1);
|
||||
assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n");
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
package io.nosqlbench.engine.rest.services;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader;
|
||||
import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
|
||||
import io.nosqlbench.engine.api.scenarios.WorkloadDesc;
|
||||
import io.nosqlbench.engine.rest.transfertypes.WorkspaceItemView;
|
||||
|
||||
@ -576,7 +576,7 @@
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<destFile>${build.directory}/jacoco-data.exec</destFile>
|
||||
<destFile>${project.build.directory}/jacoco-data.exec</destFile>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
@ -586,8 +586,8 @@
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<dataFile>${build.directory}/jacoco-data.exec</dataFile>
|
||||
<outputDirectory>${build.directory}/jacoco-reports</outputDirectory>
|
||||
<dataFile>${project.build.directory}/jacoco-data.exec</dataFile>
|
||||
<outputDirectory>${project.build.directory}/jacoco-reports</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -45,7 +45,7 @@ public class ActivityDef implements NBNamedElement {
|
||||
private final static Logger logger = LogManager.getLogger(ActivityDef.class);
|
||||
// an alias with which to control the activity while it is running
|
||||
private static final String FIELD_ALIAS = "alias";
|
||||
// a file or URL containing the activity: statements, generator bindings, ...
|
||||
// a file or URL containing the activity: op templates, generator bindings, ...
|
||||
private static final String FIELD_ATYPE = "type";
|
||||
// cycles for this activity in either "M" or "N..M" form. "M" form implies "0..M"
|
||||
private static final String FIELD_CYCLES = "cycles";
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -23,7 +23,7 @@ public enum BindType {
|
||||
|
||||
/**
|
||||
* A literal template is one which has no bindings that need to be provided to render a specific statement.
|
||||
* These templates are basically static statements.
|
||||
* These templates are basically static values.
|
||||
* Example: <em>{@code truncate testks.testtable;}</em>
|
||||
*/
|
||||
literal,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user