IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #954 from nosqlbench/nosqlbench-944-jsonnet

Browse Source 
Nosqlbench 944 jsonnet, includes the jsonnet capability and some fixes
This commit is contained in:
Jonathan Shook 2023-01-29 01:07:37 -06:00 committed by GitHub
commit d16decee42
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
43 changed files with 690 additions and 529 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,13 +16,13 @@
package io.nosqlbench.cqlgen.core;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.api.BindingsLibrary;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.cqlgen.api.BindingsLibrary;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import java.io.File;
import java.io.InputStream;
@ -36,13 +36,16 @@ public class CGDefaultCqlBindings implements BindingsLibrary {
public final static String DEFAULT_BINDINGS_FILE = "bindings-cqlgen.yaml";
public CGDefaultCqlBindings() {
String yamlContent = NBIO.all()
Content<?> content = NBIO.all()
.name(DEFAULT_BINDINGS_FILE)
.first()
.map(Content::asString)
.or(() -> loadLocal(DEFAULT_BINDINGS_FILE))
.orElseThrow(() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE + ", from local dir or internally as cqlgen/" + DEFAULT_BINDINGS_FILE));
StmtsDocList stmtsDocs = StatementsLoader.loadString(yamlContent, Map.of());
.or(() -> NBIO.all().prefix(DEFAULT_CFG_DIR).name(DEFAULT_BINDINGS_FILE).first())
.orElseThrow(
() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE +
", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE)
);
OpsDocList stmtsDocs = OpsLoader.loadContent(content, Map.of());
this.bindings = stmtsDocs.getDocBindings();
}

11
adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,9 +17,10 @@
package io.nosqlbench.adapter.diag;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
@ -101,7 +102,7 @@ public class DiagDriverAdapter extends BaseDriverAdapter<DiagOp, DiagSpace> impl
}
@Override
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String, Object> params) {
return StatementsLoader.loadString("ops: 'log:level=INFO'", params).getStmts();
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> params) {
return OpsLoader.loadString("log:level=INFO", OpTemplateFormat.inline, params,null).getStmts();
}
}

9
adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -19,9 +19,10 @@ package io.nosqlbench.adapter.http;
import io.nosqlbench.adapter.http.core.HttpOpMapper;
import io.nosqlbench.adapter.http.core.HttpSpace;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.engine.api.templating.ParsedOp;
import org.apache.logging.log4j.LogManager;
@ -51,7 +52,7 @@ public class HttpOpMapperTest {
}
private static ParsedOp parsedOpFor(String yaml) {
StmtsDocList docs = StatementsLoader.loadString(yaml, Map.of());
OpsDocList docs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
OpTemplate stmtDef = docs.getStmts().get(0);
ParsedOp parsedOp = new ParsedOp(stmtDef, cfg, List.of(adapter.getPreprocessor()));
return parsedOp;

8
adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.adapter.stdout;
import io.nosqlbench.engine.api.activityconfig.yaml.OpData;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
@ -59,8 +59,8 @@ public class StdoutDriverAdapter extends BaseDriverAdapter<StdoutOp, StdoutSpace
}
@Override
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String, Object> cfg) {
Set<String> activeBindingNames = new LinkedHashSet<>(stmtsDocList.getDocBindings().keySet());
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> cfg) {
Set<String> activeBindingNames = new LinkedHashSet<>(opsDocList.getDocBindings().keySet());
if (activeBindingNames.size()==0) {
logger.warn("Unable to synthesize op for driver=" + this.getAdapterName() + " with zero bindings.");
@ -88,7 +88,7 @@ public class StdoutDriverAdapter extends BaseDriverAdapter<StdoutOp, StdoutSpace
return List.of();
}
OpData op = new OpData("synthetic", "synthetic", Map.of(), stmtsDocList.getDocBindings(), cfg,
OpData op = new OpData("synthetic", "synthetic", Map.of(), opsDocList.getDocBindings(), cfg,
Map.of("stmt", genStatementTemplate(filteredBindingNames, cfg)));
return List.of(op);

8
adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientDriverAdapter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -21,7 +21,7 @@ import io.nosqlbench.api.config.standard.ConfigModel;
import io.nosqlbench.api.config.standard.NBConfigModel;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
@ -58,8 +58,8 @@ public class TcpClientDriverAdapter extends BaseDriverAdapter<TcpClientOp, TcpCl
}
@Override
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String,Object> cfg) {
return adap.getSyntheticOpTemplates(stmtsDocList, cfg);
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String,Object> cfg) {
return adap.getSyntheticOpTemplates(opsDocList, cfg);
}
}

8
adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,7 +20,7 @@ import io.nosqlbench.api.config.standard.ConfigModel;
import io.nosqlbench.api.config.standard.NBConfigModel;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
@ -57,7 +57,7 @@ public class TcpServerDriverAdapter extends BaseDriverAdapter<TcpServerOp, TcpSe
}
@Override
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String,Object> cfg) {
return adap.getSyntheticOpTemplates(stmtsDocList, cfg);
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String,Object> cfg) {
return adap.getSyntheticOpTemplates(opsDocList, cfg);
}
}

11
adapters-api/pom.xml
View File

@ -54,6 +54,17 @@
<version>${revision}</version>
</dependency>
<dependency>
<groupId>com.databricks</groupId>
<artifactId>sjsonnet_2.13</artifactId>
<version>0.4.3</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.13.10</version>
</dependency>
</dependencies>

150
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java Normal file
View File

@ -0,0 +1,150 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig;
import com.amazonaws.util.StringInputStream;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import scala.Option;
import sjsonnet.DefaultParseCache;
import sjsonnet.SjsonnetMain;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
public class OpsLoader {
private final static Logger logger = LogManager.getLogger(OpsLoader.class);
public static String[] YAML_EXTENSIONS = new String[]{"yaml", "yml"};
public static OpsDocList loadContent(Content<?> content, Map<String, String> params) {
OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(content.getURI());
return loadString(content.get().toString(), fmt, params, content.getURI());
}
public static OpsDocList loadPath(String path, Map<String, ?> params, String... searchPaths) {
String[] extensions = path.indexOf('.')>-1 ? new String[]{} : YAML_EXTENSIONS;
Content<?> foundPath = NBIO.all().prefix(searchPaths).name(path).extension(extensions).first()
.orElseThrow(() -> new RuntimeException("Unable to load path '" + path + "'"));
OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(foundPath.getURI());
return loadString(foundPath.asString(), fmt, params, foundPath.getURI());
}
public static OpsDocList loadString(final String sourceData, OpTemplateFormat fmt, Map<String, ?> params, URI srcuri) {
logger.trace(() -> "Applying string transformer to data:" + sourceData);
StrInterpolator transformer = new StrInterpolator(params);
String data = transformer.apply(sourceData);
if (srcuri!=null) {
logger.info("workload URI: '" + srcuri + "'");
}
RawOpsLoader loader = new RawOpsLoader(transformer);
RawOpsDocList rawOpsDocList = switch (fmt) {
case jsonnet -> loader.loadString(evaluateJsonnet(srcuri, params));
case yaml, json -> loader.loadString(data);
case inline, stmt -> RawOpsDocList.forSingleStatement(data);
};
// TODO: itemize inline to support ParamParser
OpsDocList layered = new OpsDocList(rawOpsDocList);
transformer.checkpointAccesses().forEach((k, v) -> {
layered.addTemplateVariable(k, v);
if (params.containsKey(k)) {
params.remove(k);
}
});
return layered;
}
private static String evaluateJsonnet(URI uri, Map<String, ?> params) {
List<String> injected = new LinkedList<>(List.of(Path.of(uri).toString()));
params.forEach((k,v) -> {
if (v instanceof CharSequence cs) {
injected.addAll(List.of("--ext-str",k+"="+cs));
}
});
var stdoutBuffer = new ByteArrayOutputStream();
var stderrBuffer = new ByteArrayOutputStream();
var stdoutStream = new PrintStream(stdoutBuffer);
var stderrStream = new PrintStream(stderrBuffer);
StringInputStream inputStream;
try {
inputStream = new StringInputStream("");
} catch (Exception e) {
throw new RuntimeException("Error building input stream for jsonnet:" + e, e);
}
int resultStatus = SjsonnetMain.main0(
injected.toArray(new String[0]),
new DefaultParseCache(),
inputStream,
stdoutStream,
stderrStream,
new os.Path(Path.of(System.getProperty("user.dir"))),
Option.empty(),
Option.empty()
);
String stdoutOutput = stdoutBuffer.toString(StandardCharsets.UTF_8);
String stderrOutput = stderrBuffer.toString(StandardCharsets.UTF_8);
if ("jsonnet".equals(String.valueOf(params.get("dryrun")))) {
logger.info("dryrun=jsonnet, dumping result to stdout and stderr:");
System.out.println(stdoutOutput);
System.err.println(stderrOutput);
if (resultStatus==0 && stderrOutput.isEmpty()) {
logger.info("no errors detected during jsonnet evaluation.");
System.exit(0);
} else {
logger.error("ERRORS detected during jsonnet evaluation:\n" + stderrOutput);
System.exit(2);
}
}
if (!stderrOutput.isEmpty()) {
BasicError error = new BasicError("stderr output from jsonnet preprocessing: " + stderrOutput);
if (resultStatus!=0) {
throw error;
} else {
logger.warn(error.toString(),error);
}
}
logger.info("jsonnet processing read '" + uri +"', rendered " + stdoutOutput.split("\n").length + " lines.");
logger.trace("jsonnet result:\n" + stdoutOutput);
return stdoutOutput;
}
}

8
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmtOp.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedOpTemplate.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Set;
import java.util.function.Function;
public class ParsedStmtOp {
public class ParsedOpTemplate {
private final OpTemplate optpl;
private final ParsedStringTemplate parsed;
@ -37,13 +37,13 @@ public class ParsedStmtOp {
*
* @param optpl An existing statement def as read from the YAML API.
*/
public ParsedStmtOp(OpTemplate optpl) {
public ParsedOpTemplate(OpTemplate optpl) {
this.optpl = optpl;
String transformed = getStmt();
parsed = new ParsedStringTemplate(transformed, optpl.getBindings());
}
public ParsedStmtOp orError() {
public ParsedOpTemplate orError() {
if (hasError()) {
throw new RuntimeException("Unable to parse statement: " + this);
}

99
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java
View File

@ -1,99 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Map;
import java.util.Optional;
public class StatementsLoader {
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
private final static Logger logger = LogManager.getLogger(StatementsLoader.class);
public static StmtsDocList loadString(String yamlContent, Map<String,?> params) {
StrInterpolator transformer = new StrInterpolator(params);
RawStmtsLoader loader = new RawStmtsLoader(transformer);
RawStmtsDocList rawDocList = loader.loadString(logger, yamlContent);
StmtsDocList layered = new StmtsDocList(rawDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
if (params.containsKey(k)) {
params.remove(k);
}
});
return layered;
}
public static StmtsDocList loadStmt(
Logger logger,
String statement,
Map<String,?> params
) {
StrInterpolator transformer = new StrInterpolator(params);
statement = transformer.apply(statement);
RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(statement);
StmtsDocList layered = new StmtsDocList(rawStmtsDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
if (params.containsKey(k)) {
params.remove(k);
}
});
return layered;
}
public static StmtsDocList loadContent(
Logger logger,
Content<?> content,
Map<String,String> params
) {
return loadString(content.get().toString(),params);
}
public static StmtsDocList loadPath(
Logger logger,
String path,
Map<String,?> params,
String... searchPaths) {
RawStmtsDocList list = null;
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(content,params);
}
public static StmtsDocList loadPath(
Logger logger,
String path,
String... searchPaths) {
return loadPath(logger, path, Map.of(), searchPaths);
}
}

22
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StatementsOwner.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpsOwner.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,21 +20,21 @@ import io.nosqlbench.api.errors.BasicError;
import java.util.*;
public class StatementsOwner extends RawStmtFields {
public class OpsOwner extends RawOpFields {
private final static List<String> stmtsFieldNames = List.of("op","ops","operation","statement","statements");
private List<RawStmtDef> rawStmtDefs = new ArrayList<>();
private List<RawOpDef> rawOpDefs = new ArrayList<>();
public StatementsOwner() {
public OpsOwner() {
}
public List<RawStmtDef> getRawStmtDefs() {
return rawStmtDefs;
public List<RawOpDef> getRawStmtDefs() {
return rawOpDefs;
}
public void setRawStmtDefs(List<RawStmtDef> rawStmtDefs) {
this.rawStmtDefs = rawStmtDefs;
public void setRawStmtDefs(List<RawOpDef> rawOpDefs) {
this.rawOpDefs = rawOpDefs;
}
public void setFieldsByReflection(Map<String, Object> propsmap) {
@ -63,14 +63,14 @@ public class StatementsOwner extends RawStmtFields {
public void setStatementsFieldByType(Object object) {
if (object instanceof List) {
List<Object> stmtList = (List<Object>) object;
List<RawStmtDef> defs = new ArrayList<>(stmtList.size());
List<RawOpDef> defs = new ArrayList<>(stmtList.size());
for (int i = 0; i < stmtList.size(); i++) {
String defaultName = "stmt" + (i + 1);
Object o = stmtList.get(i);
if (o instanceof String) {
defs.add(new RawStmtDef(defaultName, (String) o));
defs.add(new RawOpDef(defaultName, (String) o));
} else if (o instanceof Map) {
RawStmtDef def = new RawStmtDef(defaultName, (Map<String, Object>) o);
RawOpDef def = new RawOpDef(defaultName, (Map<String, Object>) o);
defs.add(def);
} else {
throw new RuntimeException("Can not construct stmt def from object type:" + o.getClass());

10
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDef.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpDef.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -24,22 +24,22 @@ import java.util.*;
/**
* See specification for what this should do in UniformWorkloadSpecificationTest
*/
public class RawStmtDef extends RawStmtFields {
public class RawOpDef extends RawOpFields {
private Object op;
private final static List<String> opFieldSynonyms = List.of("stmt", "statement", "op", "operation");
public RawStmtDef() {
public RawOpDef() {
}
public RawStmtDef(String name, String op) {
public RawOpDef(String name, String op) {
setName(name);
this.op = op;
}
@SuppressWarnings("unchecked")
public RawStmtDef(String defaultName, Map<String, Object> map) {
public RawOpDef(String defaultName, Map<String, Object> map) {
setFieldsByReflection(map);
if (this.getName() == null || this.getName().isEmpty()) {
this.setName(defaultName);

10
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtFields.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpFields.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -25,15 +25,15 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
public class RawStmtFields extends Tags {
private final static Logger logger = LogManager.getLogger(RawStmtFields.class);
public class RawOpFields extends Tags {
private final static Logger logger = LogManager.getLogger(RawOpFields.class);
private String name = "";
private String desc = "";
private final Map<String, String> bindings = new LinkedHashMap<>();
private final Map<String, Object> params = new LinkedHashMap<>();
public RawStmtFields() {
public RawOpFields() {
}
public String getDesc() {
@ -80,7 +80,7 @@ public class RawStmtFields extends Tags {
this.params.putAll(config);
}
public void applyBlockParams(RawStmtFields other) {
public void applyBlockParams(RawOpFields other) {
setName(other.getName());
setBindings(other.getBindings());
setTags(other.getTags());

8
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsBlock.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsBlock.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,12 +20,12 @@ import java.util.Map;
/**
* A StmtsDef contains a list of rawStmts, as well as all of the optional
* block parameters that can be assigned to {@link RawStmtFields}, which includes
* block parameters that can be assigned to {@link RawOpFields}, which includes
* a name, config values, data bindings, and filtering tags.
*/
public class RawStmtsBlock extends StatementsOwner {
public class RawOpsBlock extends OpsOwner {
public RawStmtsBlock() {}
public RawOpsBlock() {}
public void setFieldsByReflection(Map<String, Object> propsmap) {
super.setFieldsByReflection(propsmap);

46
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDoc.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -25,26 +25,26 @@ import java.util.Map;
/**
* A statements doc can have both a list of statement blocks and/or a
* list of statements. It can also have all the block parameters
* assignable to {@link RawStmtFields}.
* assignable to {@link RawOpFields}.
* <p>
* The reason for having support both statements or statement blocks
* is merely convenience. If you do not need or want to deal with the
* full blocks format, the extra structure gets in the way.
*/
public class RawStmtsDoc extends StatementsOwner {
public class RawOpsDoc extends OpsOwner {
private RawScenarios scenarios = new RawScenarios();
private final List<RawStmtsBlock> blocks = new ArrayList<>();
private final List<RawOpsBlock> blocks = new ArrayList<>();
private String versionRegex = ".+";
// no-args ctor is required
public RawStmtsDoc() {
public RawOpsDoc() {
}
public static RawStmtsDoc forSingleStatement(String statement) {
RawStmtsDoc rawStmtsDoc = new RawStmtsDoc();
rawStmtsDoc.setStatementsFieldByType(statement);
return rawStmtsDoc;
public static RawOpsDoc forSingleStatement(String statement) {
RawOpsDoc rawOpsDoc = new RawOpsDoc();
rawOpsDoc.setStatementsFieldByType(statement);
return rawOpsDoc;
}
public void setFieldsByReflection(Map<String, Object> properties) {
@ -63,9 +63,9 @@ public class RawStmtsDoc extends StatementsOwner {
for (Object blockData : blockList) {
if (blockData instanceof Map) {
Map<String, Object> blockDataMap = (Map<String, Object>) blockData;
RawStmtsBlock rawStmtsBlock = new RawStmtsBlock();
rawStmtsBlock.setFieldsByReflection(blockDataMap);
blocks.add(rawStmtsBlock);
RawOpsBlock rawOpsBlock = new RawOpsBlock();
rawOpsBlock.setFieldsByReflection(blockDataMap);
blocks.add(rawOpsBlock);
} else {
throw new RuntimeException("Invalid object type for block data: " + blockData.getClass().getCanonicalName());
}
@ -77,10 +77,10 @@ public class RawStmtsDoc extends StatementsOwner {
Object blockData = entry.getValue();
if (blockData instanceof Map) {
Map<String, Object> blockDataMap = (Map<String, Object>) blockData;
RawStmtsBlock rawStmtsBlock = new RawStmtsBlock();
rawStmtsBlock.setName(blockName);
rawStmtsBlock.setFieldsByReflection(blockDataMap);
blocks.add(rawStmtsBlock);
RawOpsBlock rawOpsBlock = new RawOpsBlock();
rawOpsBlock.setName(blockName);
rawOpsBlock.setFieldsByReflection(blockDataMap);
blocks.add(rawOpsBlock);
} else {
throw new RuntimeException("Invalid object type for block data: " + blockData.getClass().getCanonicalName());
}
@ -107,19 +107,19 @@ public class RawStmtsDoc extends StatementsOwner {
*
* @return all logical statement blocks containing statements
*/
public List<RawStmtsBlock> getBlocks() {
List<RawStmtsBlock> stmtBlocks = new ArrayList<>();
public List<RawOpsBlock> getBlocks() {
List<RawOpsBlock> stmtBlocks = new ArrayList<>();
if (!getRawStmtDefs().isEmpty()) {
RawStmtsBlock rawStmtsBlock = new RawStmtsBlock();
rawStmtsBlock.setName("block0");
rawStmtsBlock.setRawStmtDefs(getRawStmtDefs());
stmtBlocks.add(rawStmtsBlock);
RawOpsBlock rawOpsBlock = new RawOpsBlock();
rawOpsBlock.setName("block0");
rawOpsBlock.setRawStmtDefs(getRawStmtDefs());
stmtBlocks.add(rawOpsBlock);
}
stmtBlocks.addAll(this.blocks);
return stmtBlocks;
}
public void setBlocks(List<RawStmtsBlock> blocks) {
public void setBlocks(List<RawOpsBlock> blocks) {
this.blocks.clear();
this.blocks.addAll(blocks);
}

48
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDocList.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import java.util.List;
public class RawOpsDocList {
private final List<RawOpsDoc> rawOpsDocList;
public RawOpsDocList(List<RawOpsDoc> rawOpsDocList) {
this.rawOpsDocList = rawOpsDocList;
}
public static RawOpsDocList forSingleStatement(String statement) {
RawOpsDoc rawOpsDoc = RawOpsDoc.forSingleStatement(statement);
return new RawOpsDocList(List.of(rawOpsDoc));
}
public static RawOpsDocList none() {
return new RawOpsDocList(List.of());
}
public List<RawOpsDoc> getStmtsDocs() {
return rawOpsDocList;
}
public String toString() {
int docs = rawOpsDocList.size();
int blocks = rawOpsDocList.stream().map(RawOpsDoc::getBlocks).mapToInt(List::size).sum();
long optemplates = rawOpsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawStmtDefs().stream()).count();
return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates;
}
}

44
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsLoader.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java
View File

@ -21,6 +21,7 @@ import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
@ -28,18 +29,18 @@ import org.snakeyaml.engine.v2.api.LoadSettings;
import java.util.*;
import java.util.function.Function;
public class RawStmtsLoader {
public class RawOpsLoader {
private final static Logger logger = LogManager.getLogger(RawOpsLoader.class);
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
List<Function<String, String>> stringTransformers = new ArrayList<>();
private final ArrayList<Function<String,String>> transformers = new ArrayList<>();
public RawStmtsLoader(Function<String,String> transformer) {
public RawOpsLoader(Function<String,String> transformer) {
addTransformer(transformer);
}
public RawStmtsLoader() {
public RawOpsLoader() {
addTransformer(new StrInterpolator());
}
@ -47,11 +48,10 @@ public class RawStmtsLoader {
Collections.addAll(this.transformers, newTransformer);
}
public RawStmtsDocList loadString(Logger logger, final String originalData) {
public RawOpsDocList loadString(final String originalData) {
logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
String data = originalData;
try {
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
for (Function<String, String> transformer : transformers) {
data = transformer.apply(data);
}
@ -60,11 +60,10 @@ public class RawStmtsLoader {
throw t;
}
return parseYaml(logger, data);
return parseYaml(data);
}
public RawStmtsDocList loadPath(
Logger logger,
public RawOpsDocList loadPath(
String path,
String... searchPaths) {
@ -72,22 +71,22 @@ public class RawStmtsLoader {
try {
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(logger, data);
return loadString(data);
} catch (Exception e) {
throw new RuntimeException("error while reading file " + path, e);
}
}
private RawStmtsDocList parseYaml(Logger logger, String data) {
public RawOpsDocList parseYaml(String data) {
LoadSettings loadSettings = LoadSettings.builder().build();
Load yaml = new Load(loadSettings);
Iterable<Object> objects = yaml.loadAllFromString(data);
List<RawStmtsDoc> newDocList = new ArrayList<>();
List<RawOpsDoc> newDocList = new ArrayList<>();
for (Object object : objects) {
if (object instanceof Map) {
RawStmtsDoc doc = new RawStmtsDoc();
RawOpsDoc doc = new RawOpsDoc();
Map<String, Object> docfields = (Map<String, Object>) object;
doc.setFieldsByReflection(docfields);
if (docfields.size()>0) {
@ -99,21 +98,8 @@ public class RawStmtsLoader {
throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName());
}
}
RawStmtsDocList rawStmtsDocList = new RawStmtsDocList(newDocList);
return rawStmtsDocList;
}
protected String applyTransforms(Logger logger, String data) {
for (Function<String, String> xform : stringTransformers) {
try {
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + xform);
data = xform.apply(data);
} catch (Exception e) {
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
throw t;
}
}
return data;
RawOpsDocList rawOpsDocList = new RawOpsDocList(newDocList);
return rawOpsDocList;
}
}

48
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java
View File

@ -1,48 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import java.util.List;
public class RawStmtsDocList {
private final List<RawStmtsDoc> rawStmtsDocList;
public RawStmtsDocList(List<RawStmtsDoc> rawStmtsDocList) {
this.rawStmtsDocList = rawStmtsDocList;
}
public static RawStmtsDocList forSingleStatement(String statement) {
RawStmtsDoc rawStmtsDoc = RawStmtsDoc.forSingleStatement(statement);
return new RawStmtsDocList(List.of(rawStmtsDoc));
}
public static RawStmtsDocList none() {
return new RawStmtsDocList(List.of());
}
public List<RawStmtsDoc> getStmtsDocs() {
return rawStmtsDocList;
}
public String toString() {
int docs = rawStmtsDocList.size();
int blocks = rawStmtsDocList.stream().map(RawStmtsDoc::getBlocks).mapToInt(List::size).sum();
long optemplates = rawStmtsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawStmtDefs().stream()).count();
return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates;
}
}

2
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java
View File

@ -76,7 +76,7 @@ public class RawYamlLoader {
private List<Map<String,Object>> parseYaml(Logger logger, String data) {
Load yaml = new Load(LoadSettings.builder().build());
Iterable<Object> objects = yaml.loadAllFromString(data);
List<RawStmtsDoc> newDocList = new ArrayList<>();
List<RawOpsDoc> newDocList = new ArrayList<>();
List<Map<String,Object>> maps = new ArrayList<>();

24
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.MultiMapLookup;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpDef;
import io.nosqlbench.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -27,15 +27,15 @@ import java.util.*;
public class OpDef extends OpTemplate {
private final static Logger logger = LogManager.getLogger(OpDef.class);
private final RawStmtDef rawStmtDef;
private final StmtsBlock block;
private final RawOpDef rawOpDef;
private final OpsBlock block;
private final LinkedHashMap<String, Object> params;
private final LinkedHashMap<String, String> bindings;
private final LinkedHashMap<String, String> tags;
public OpDef(StmtsBlock block, RawStmtDef rawStmtDef) {
public OpDef(OpsBlock block, RawOpDef rawOpDef) {
this.block = block;
this.rawStmtDef = rawStmtDef;
this.rawOpDef = rawOpDef;
this.params = composeParams();
this.bindings = composeBindings();
this.tags = composeTags();
@ -43,12 +43,12 @@ public class OpDef extends OpTemplate {
@Override
public String getName() {
return block.getName() + "--" + rawStmtDef.getName();
return block.getName() + "--" + rawOpDef.getName();
}
@Override
public Optional<Map<String, Object>> getOp() {
Object op = rawStmtDef.getOp();
Object op = rawOpDef.getOp();
if (op == null) {
return Optional.empty();
}
@ -75,7 +75,7 @@ public class OpDef extends OpTemplate {
}
private LinkedHashMap<String, String> composeBindings() {
MultiMapLookup<String> lookup = new MultiMapLookup<>(rawStmtDef.getBindings(), block.getBindings());
MultiMapLookup<String> lookup = new MultiMapLookup<>(rawOpDef.getBindings(), block.getBindings());
return new LinkedHashMap<>(lookup);
}
@ -85,7 +85,7 @@ public class OpDef extends OpTemplate {
}
private LinkedHashMap<String, Object> composeParams() {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpDef.getParams(), block.getParams());
LinkedHashMap<String, Object> params = new LinkedHashMap<>(lookup);
return params;
}
@ -98,7 +98,7 @@ public class OpDef extends OpTemplate {
}
private LinkedHashMap<String, String> composeTags() {
LinkedHashMap<String, String> tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawStmtDef.getTags(), block.getTags()));
LinkedHashMap<String, String> tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawOpDef.getTags(), block.getTags()));
tagsWithName.put("name",getName());
tagsWithName.put("block",block.getName());
return tagsWithName;
@ -111,7 +111,7 @@ public class OpDef extends OpTemplate {
@Override
public String getDesc() {
return rawStmtDef.getDesc();
return rawOpDef.getDesc();
}
}

94
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplateFormat.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig.yaml;
import java.net.URI;
import java.nio.file.Path;
/**
* The formats which are recognized as source data for the workload. Any serialization may be supported
* which can be converted from a character string to an {@link io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList} structure.
*
* Those which are derived from paths may be determined by their filename extension. Others, which are provided from internal
* NoSQLBench sources, may only be invoked explicitly.
*/
public enum OpTemplateFormat {
/**
* The source is in YAML format
*/
yaml("yaml", "yml"),
/**
* The source is in JSON format
*/
json("json", "json5"),
/**
* The source is in jsonnet format, suitable for pre-processing with context data
*/
jsonnet("jsonnet", "jsonnet5"),
/**
* The source is inline, meaning {@link io.nosqlbench.api.config.params.ParamsParser} format.
* This is similar to the {@link #stmt} format except that it is parsed for internal op structure.
* This format is not detectable by source path, and thus can only be used when provided directly
* from the caller.
*/
inline(),
/**
* The source is in single-statement form, meaning that it is known to be the value of the 'stmt'
* field of an op template. This format is not detectable by source path, and thus can only be used when provided directly
* from the caller.
*/
stmt();
private final String[] pathExtensions;
OpTemplateFormat(String... pathExtensions) {
this.pathExtensions = pathExtensions;
}
public static OpTemplateFormat valueOfURI(URI uri) {
var fullName = uri.toString();
String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase();
for (OpTemplateFormat value : values()) {
for (String pathExtension : value.pathExtensions) {
if (pathExtension.equals(extension)) {
return value;
}
}
}
throw new RuntimeException("Unable to determine source format for " + uri);
}
public static OpTemplateFormat valueOfPath(Path path) {
var fullName = path.toString();
String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase();
for (OpTemplateFormat value : values()) {
for (String pathExtension : value.pathExtensions) {
if (pathExtension.equals(extension)) {
return value;
}
}
}
throw new RuntimeException("Unable to determine source format for " + path);
}
}

40
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsBlock.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsBlock.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,30 +17,30 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.MultiMapLookup;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpDef;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsBlock;
import io.nosqlbench.api.engine.util.Tagged;
import org.jetbrains.annotations.NotNull;
import java.util.*;
public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
public class OpsBlock implements Tagged, Iterable<OpTemplate> {
private final RawStmtsBlock rawStmtsBlock;
private final StmtsDoc rawStmtsDoc;
private final RawOpsBlock rawOpsBlock;
private final OpsDoc rawOpsDoc;
private final int blockIdx;
public StmtsBlock(RawStmtsBlock rawStmtsBlock, StmtsDoc rawStmtsDoc, int blockIdx) {
this.rawStmtsBlock = rawStmtsBlock;
this.rawStmtsDoc = rawStmtsDoc;
public OpsBlock(RawOpsBlock rawOpsBlock, OpsDoc rawOpsDoc, int blockIdx) {
this.rawOpsBlock = rawOpsBlock;
this.rawOpsDoc = rawOpsDoc;
this.blockIdx = blockIdx;
}
public List<OpTemplate> getOps() {
List<OpTemplate> rawOpTemplates = new ArrayList<>();
List<RawStmtDef> statements = rawStmtsBlock.getRawStmtDefs();
List<RawOpDef> statements = rawOpsBlock.getRawStmtDefs();
for (int i = 0; i < statements.size(); i++) {
rawOpTemplates.add(
@ -52,11 +52,11 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
public String getName() {
StringBuilder sb = new StringBuilder();
if (!rawStmtsDoc.getName().isEmpty()) {
sb.append(rawStmtsDoc.getName()).append("--");
if (!rawOpsDoc.getName().isEmpty()) {
sb.append(rawOpsDoc.getName()).append("--");
}
if (!rawStmtsBlock.getName().isEmpty()) {
sb.append(rawStmtsBlock.getName());
if (!rawOpsBlock.getName().isEmpty()) {
sb.append(rawOpsBlock.getName());
} else {
sb.append("block").append(blockIdx);
}
@ -64,15 +64,15 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
}
public Map<String, String> getTags() {
return new MultiMapLookup<>(rawStmtsBlock.getTags(), rawStmtsDoc.getTags());
return new MultiMapLookup<>(rawOpsBlock.getTags(), rawOpsDoc.getTags());
}
public Map<String, Object> getParams() {
return new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
return new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
}
public Map<String, String> getParamsAsText() {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
LinkedHashMap<String, String> stringmap = new LinkedHashMap<>();
lookup.forEach((k, v) -> stringmap.put(k, v.toString()));
return stringmap;
@ -81,7 +81,7 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
@SuppressWarnings("unchecked")
public <V> V getParamOrDefault(String name, V defaultValue) {
Objects.requireNonNull(defaultValue);
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
if (!lookup.containsKey(name)) {
return defaultValue;
}
@ -90,14 +90,14 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
}
public <V> V getParam(String name, Class<? extends V> type) {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
Object object = lookup.get(name);
V value = type.cast(object);
return value;
}
public Map<String, String> getBindings() {
return new MultiMapLookup<>(rawStmtsBlock.getBindings(), rawStmtsDoc.getBindings());
return new MultiMapLookup<>(rawOpsBlock.getBindings(), rawOpsDoc.getBindings());
}
@Override

42
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDoc.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDoc.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,8 +16,8 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsBlock;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDoc;
import io.nosqlbench.api.engine.util.Tagged;
import java.util.ArrayList;
@ -30,26 +30,26 @@ import java.util.stream.Collectors;
* StmtsDoc creates a logical view of a statements doc that includes
* all inherited and overridden values for bindings, tags, and params.
*/
public class StmtsDoc implements Tagged, Iterable<StmtsBlock> {
public class OpsDoc implements Tagged, Iterable<OpsBlock> {
private final RawStmtsDoc rawStmtsDoc;
private final RawOpsDoc rawOpsDoc;
public StmtsDoc(RawStmtsDoc rawStmtsDoc) {
this.rawStmtsDoc = rawStmtsDoc;
public OpsDoc(RawOpsDoc rawOpsDoc) {
this.rawOpsDoc = rawOpsDoc;
}
/**
* @return a usable list of blocks, including inherited bindings, params, and tags
* from the parent doc
*/
public List<StmtsBlock> getBlocks() {
List<StmtsBlock> blocks = new ArrayList<>();
public List<OpsBlock> getBlocks() {
List<OpsBlock> blocks = new ArrayList<>();
int blockIdx = 0;
for (RawStmtsBlock rawStmtsBlock : rawStmtsDoc.getBlocks()) {
String compositeName = rawStmtsDoc.getName() +
(rawStmtsBlock.getName().isEmpty() ? "" : "-" + rawStmtsBlock.getName());
StmtsBlock compositeBlock = new StmtsBlock(rawStmtsBlock, this, ++blockIdx);
for (RawOpsBlock rawOpsBlock : rawOpsDoc.getBlocks()) {
String compositeName = rawOpsDoc.getName() +
(rawOpsBlock.getName().isEmpty() ? "" : "-" + rawOpsBlock.getName());
OpsBlock compositeBlock = new OpsBlock(rawOpsBlock, this, ++blockIdx);
blocks.add(compositeBlock);
}
@ -61,28 +61,28 @@ public class StmtsDoc implements Tagged, Iterable<StmtsBlock> {
*/
@Override
public Map<String, String> getTags() {
return rawStmtsDoc.getTags();
return rawOpsDoc.getTags();
}
/**
* @return a usable map of parameters, including those inherited from the parent doc
*/
public Map<String, Object> getParams() {
return rawStmtsDoc.getParams();
return rawOpsDoc.getParams();
}
/**
* @return a usable map of bindings, including those inherited from the parent doc
*/
public Map<String, String> getBindings() {
return rawStmtsDoc.getBindings();
return rawOpsDoc.getBindings();
}
/**
* @return the name of this block
*/
public String getName() {
return rawStmtsDoc.getName();
return rawOpsDoc.getName();
}
/**
@ -95,19 +95,19 @@ public class StmtsDoc implements Tagged, Iterable<StmtsBlock> {
/**
* Allow StmtsDoc to be used in iterable loops.
* @return An iterator of {@link StmtsBlock}
* @return An iterator of {@link OpsBlock}
*/
@Override
public Iterator<StmtsBlock> iterator() {
public Iterator<OpsBlock> iterator() {
return getBlocks().iterator();
}
public Scenarios getScenarios() {
return new Scenarios(rawStmtsDoc.getRawScenarios());
return new Scenarios(rawOpsDoc.getRawScenarios());
}
public String getDescription() {
return rawStmtsDoc.getDesc();
return rawOpsDoc.getDesc();
}
}

42
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java
View File

@ -16,9 +16,9 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDoc;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList;
import io.nosqlbench.engine.api.util.TagFilter;
import io.nosqlbench.api.config.standard.ConfigModel;
import io.nosqlbench.api.config.standard.NBConfigModel;
@ -31,30 +31,30 @@ import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class StmtsDocList implements Iterable<StmtsDoc> {
private final static Logger logger = LogManager.getLogger(StmtsDocList.class);
public class OpsDocList implements Iterable<OpsDoc> {
private final static Logger logger = LogManager.getLogger(OpsDocList.class);
private final RawStmtsDocList rawStmtsDocList;
private final RawOpsDocList rawOpsDocList;
private final Map<String, String> templateVariables = new LinkedHashMap<>();
public StmtsDocList(RawStmtsDocList rawStmtsDocList) {
this.rawStmtsDocList = rawStmtsDocList;
public OpsDocList(RawOpsDocList rawOpsDocList) {
this.rawOpsDocList = rawOpsDocList;
}
public static StmtsDocList none() {
return new StmtsDocList(RawStmtsDocList.none());
public static OpsDocList none() {
return new OpsDocList(RawOpsDocList.none());
}
public List<StmtsDoc> getStmtDocs(String tagFilter) {
public List<OpsDoc> getStmtDocs(String tagFilter) {
TagFilter tf = new TagFilter(tagFilter);
return getStmtDocs().stream()
.filter(tf::matchesTagged)
.collect(Collectors.toList());
}
public List<StmtsDoc> getStmtDocs() {
return rawStmtsDocList.getStmtsDocs().stream()
.map(StmtsDoc::new)
public List<OpsDoc> getStmtDocs() {
return rawOpsDocList.getStmtsDocs().stream()
.map(OpsDoc::new)
.collect(Collectors.toList());
}
@ -81,7 +81,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
@Override
public Iterator<StmtsDoc> iterator() {
public Iterator<OpsDoc> iterator() {
return getStmtDocs().iterator();
}
@ -94,7 +94,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
public Map<String, String> getDocBindings() {
LinkedHashMap<String, String> docBindings = new LinkedHashMap<>();
getStmtDocs().stream()
.map(StmtsDoc::getBindings)
.map(OpsDoc::getBindings)
.forEach(docBindings::putAll);
return docBindings;
}
@ -130,7 +130,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
}
public NBConfigModel getConfigModel() {
ConfigModel cfgmodel = ConfigModel.of(StmtsDocList.class);
ConfigModel cfgmodel = ConfigModel.of(OpsDocList.class);
getTemplateVariables().forEach((k, v) -> {
cfgmodel.add(Param.defaultTo(k, v, "template parameter found in the yaml workload"));
});
@ -144,9 +144,9 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
int blockscount = 0;
int opscount = 0;
for (StmtsDoc stmtDoc : this.getStmtDocs()) {
for (OpsDoc stmtDoc : this.getStmtDocs()) {
docscount++;
for (StmtsBlock block : stmtDoc.getBlocks()) {
for (OpsBlock block : stmtDoc.getBlocks()) {
blockscount++;
for (OpTemplate op : block.getOps()) {
opscount++;
@ -161,12 +161,12 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
}
public static NBConfigModelExpander TEMPLATE_VAR_EXPANDER = workload -> {
StmtsDocList loaded = StatementsLoader.loadPath(logger, (String) workload, "activities");
OpsDocList loaded = OpsLoader.loadPath((String) workload, Map.of(), "activities");
return loaded.getConfigModel();
};
public Pattern getVersionRegex() {
List<RawStmtsDoc> stmtDocs = rawStmtsDocList.getStmtsDocs();
List<RawOpsDoc> stmtDocs = rawOpsDocList.getStmtsDocs();
return Pattern.compile(stmtDocs.size()>0 ? stmtDocs.get(0).getVersionRegex() : ".*");
}
}

8
adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/decorators/SyntheticOpTemplateProvider.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.activityimpl.uniform.decorators;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.templating.DriverAdapterDecorators;
import java.util.List;
@ -38,8 +38,8 @@ public interface SyntheticOpTemplateProvider extends DriverAdapterDecorators {
/**
* If a driver adapter supports creating example op templates from bindings,
* it must implement this method to do so.
* @param stmtsDocList The existing doc structure, which should contain no fully defined op templates, but may contain other elements like bindings
* @param opsDocList The existing doc structure, which should contain no fully defined op templates, but may contain other elements like bindings
* @return A list of op templates, size zero or more
*/
List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String,Object> params);
List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String,Object> params);
}

16
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/StatementsLoaderTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,8 +16,8 @@
package io.nosqlbench.engine.api.activityconfig;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import org.junit.jupiter.api.Test;
import java.util.List;
@ -25,15 +25,15 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class StatementsLoaderTest {
public class OpsLoaderTest {
@Test
public void testTemplateVarSubstitution() {
StmtsDocList stmtsDocs = StatementsLoader.loadPath(null, "activities/template_vars", "src/test/resources");
OpsDocList stmtsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources");
assertThat(stmtsDocs).isNotNull();
List<StmtsDoc> docs = stmtsDocs.getStmtDocs();
List<OpsDoc> docs = stmtsDocs.getStmtDocs();
assertThat(docs).hasSize(1);
StmtsDoc stmtsBlocks = docs.get(0);
OpsDoc stmtsBlocks = docs.get(0);
Map<String, String> bindings = stmtsBlocks.getBindings();
assertThat(bindings).isEqualTo(Map.of(
"b1a","Prefix(\"prefix\")",
@ -46,7 +46,7 @@ public class StatementsLoaderTest {
public void testInvalidYamlProperties() {
Exception caught = null;
try {
StatementsLoader.loadPath(null, "activities/invalid_prop", "src/test/resources");
OpsLoader.loadPath("activities/invalid_prop", Map.of(),"src/test/resources");
} catch (Exception e) {
caught = e;
}

58
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,11 +16,11 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsBlock;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsBlock;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
@ -37,22 +37,22 @@ public class OpDefTest {
@Test
public void testLayering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = all.getStmtDocs().get(0);
OpsDoc doc1 = all.getStmtDocs().get(0);
assertThat(doc1.getName()).isEqualTo("doc1");
assertThat(doc1.getBlocks()).hasSize(1);
StmtsDoc doc2 = all.getStmtDocs().get(1);
OpsDoc doc2 = all.getStmtDocs().get(1);
assertThat(doc2.getBlocks()).hasSize(2);
StmtsBlock block1 = doc1.getBlocks().get(0);
OpsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getBindings()).hasSize(2);
assertThat(block1.getName()).isEqualTo("doc1--block0");
assertThat(block1.getTags()).hasSize(1);
StmtsBlock block21 = doc2.getBlocks().get(0);
StmtsBlock block22 = doc2.getBlocks().get(1);
OpsBlock block21 = doc2.getBlocks().get(0);
OpsBlock block22 = doc2.getBlocks().get(1);
assertThat(block21.getName()).isEqualTo("doc2--block1");
assertThat(block21.getTags()).hasSize(3);
@ -64,11 +64,11 @@ public class OpDefTest {
@Test
public void testStatementRendering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = all.getStmtDocs().get(0);
StmtsBlock block1 = doc1.getBlocks().get(0);
OpsDoc doc1 = all.getStmtDocs().get(0);
OpsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getName()).isEqualTo("doc1--block0");
List<OpTemplate> ops = block1.getOps();
assertThat(ops).hasSize(2);
@ -79,9 +79,9 @@ public class OpDefTest {
@Test
public void testConsumableMapState() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc block1 = docs.get(1);
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc block1 = docs.get(1);
List<OpTemplate> stmts = block1.getStmts();
OpTemplate stmt0 = stmts.get(0);
OpTemplate stmt1 = stmts.get(1);
@ -94,12 +94,12 @@ public class OpDefTest {
@Test
public void testMapOfMaps() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc0 = docs.get(0);
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc0 = docs.get(0);
assertThat(doc0.getName()).isEqualTo("map-of-maps");
assertThat(doc0.getBlocks()).hasSize(1);
StmtsBlock block1 = doc0.getBlocks().get(0);
OpsBlock block1 = doc0.getBlocks().get(0);
assertThat(block1.getName()).isEqualTo("map-of-maps--block0");
assertThat(block1.getOps()).hasSize(2);
OpTemplate op0 = block1.getOps().get(0);
@ -115,12 +115,12 @@ public class OpDefTest {
@Test
public void testBasicStringStmt() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc1 = docs.get(1);
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc1 = docs.get(1);
assertThat(doc1.getName()).isEqualTo("string-statement");
assertThat(doc1.getBlocks()).hasSize(1);
StmtsBlock block1 = doc1.getBlocks().get(0);
OpsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getName()).isEqualTo("string-statement--block0");
assertThat(block1.getOps()).hasSize(1);
OpTemplate op0 = block1.getOps().get(0);
@ -130,12 +130,12 @@ public class OpDefTest {
@Test
public void testListOfNamedMap() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc2 = docs.get(2);
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc2 = docs.get(2);
assertThat(doc2.getName()).isEqualTo("list-of-named-map");
assertThat(doc2.getBlocks()).hasSize(1);
StmtsBlock block1 = doc2.getBlocks().get(0);
OpsBlock block1 = doc2.getBlocks().get(0);
assertThat(block1.getOps()).hasSize(1);
OpTemplate op0 = block1.getOps().get(0);
System.out.println(op0.getParams());

56
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoaderTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlTemplateLoaderTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -25,15 +25,15 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class RawYamlStatementLoaderTest {
private final static Logger logger = LogManager.getLogger(RawYamlStatementLoaderTest.class);
public class RawYamlTemplateLoaderTest {
private final static Logger logger = LogManager.getLogger(RawYamlTemplateLoaderTest.class);
@Test
public void testLoadPropertiesBlock() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList rawBlockDocs = ysl.loadPath(logger, "testdocs/rawblock.yaml");
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList rawBlockDocs = ysl.loadPath("testdocs/rawblock.yaml");
assertThat(rawBlockDocs.getStmtsDocs()).hasSize(1);
RawStmtsDoc rawBlockDoc = rawBlockDocs.getStmtsDocs().get(0);
RawOpsDoc rawBlockDoc = rawBlockDocs.getStmtsDocs().get(0);
assertThat(rawBlockDoc.getRawStmtDefs()).hasSize(1);
assertThat(rawBlockDoc.getBindings()).hasSize(1);
assertThat(rawBlockDoc.getName()).isEqualTo("name");
@ -43,30 +43,30 @@ public class RawYamlStatementLoaderTest {
@Test
public void testLoadFullFormat() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
List<RawStmtsDoc> rawStmtsDocs = erthing.getStmtsDocs();
assertThat(rawStmtsDocs).hasSize(2);
RawStmtsDoc rawStmtsDoc = rawStmtsDocs.get(0);
List<RawStmtsBlock> blocks = rawStmtsDoc.getBlocks();
assertThat(rawStmtsDoc.getName()).isEqualTo("doc1");
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_stmts.yaml");
List<RawOpsDoc> rawOpsDocs = erthing.getStmtsDocs();
assertThat(rawOpsDocs).hasSize(2);
RawOpsDoc rawOpsDoc = rawOpsDocs.get(0);
List<RawOpsBlock> blocks = rawOpsDoc.getBlocks();
assertThat(rawOpsDoc.getName()).isEqualTo("doc1");
assertThat(blocks).hasSize(1);
RawStmtsBlock rawStmtsBlock = blocks.get(0);
assertThat(rawStmtsBlock.getName()).isEqualTo("block0");
RawOpsBlock rawOpsBlock = blocks.get(0);
assertThat(rawOpsBlock.getName()).isEqualTo("block0");
}
@Test
public void testLoadScenarios() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
List<RawStmtsDoc> rawStmtsDocs = erthing.getStmtsDocs();
assertThat(rawStmtsDocs).hasSize(2);
RawStmtsDoc rawStmtsDoc = rawStmtsDocs.get(0);
List<RawStmtsBlock> blocks = rawStmtsDoc.getBlocks();
assertThat(rawStmtsDoc.getDesc()).isEqualTo(
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_stmts.yaml");
List<RawOpsDoc> rawOpsDocs = erthing.getStmtsDocs();
assertThat(rawOpsDocs).hasSize(2);
RawOpsDoc rawOpsDoc = rawOpsDocs.get(0);
List<RawOpsBlock> blocks = rawOpsDoc.getBlocks();
assertThat(rawOpsDoc.getDesc()).isEqualTo(
"a quintessential description - this is superseded by dedicated specification tests and will be removed");
RawScenarios rawScenarios = rawStmtsDoc.getRawScenarios();
RawScenarios rawScenarios = rawOpsDoc.getRawScenarios();
assertThat(rawScenarios.getScenarioNames()).containsExactly("default", "schema-only");
Map<String, String> defaultScenario = rawScenarios.getNamedScenario("default");
assertThat(defaultScenario.keySet())
@ -79,16 +79,16 @@ public class RawYamlStatementLoaderTest {
assertThat(schemaOnlyScenario.values())
.containsExactly("run driver=blah tags=phase:schema");
assertThat(rawStmtsDoc.getName()).isEqualTo("doc1");
assertThat(rawOpsDoc.getName()).isEqualTo("doc1");
assertThat(blocks).hasSize(1);
RawStmtsBlock rawStmtsBlock = blocks.get(0);
assertThat(rawStmtsBlock.getName()).isEqualTo("block0");
RawOpsBlock rawOpsBlock = blocks.get(0);
assertThat(rawOpsBlock.getName()).isEqualTo("block0");
}
@Test
public void testErrorMsg() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/badyamlfile.yaml");
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList erthing = ysl.loadPath("testdocs/badyamlfile.yaml");
}
}

11
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.*;
import org.junit.jupiter.api.BeforeAll;
import org.apache.logging.log4j.Logger;
@ -24,6 +24,7 @@ import org.apache.logging.log4j.LogManager;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
@ -35,15 +36,15 @@ public class StmtEscapingTest {
@BeforeAll
public static void testLayering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/escaped_stmts.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/escaped_stmts.yaml", Map.of());
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(1);
StmtsDoc doc1 = all.getStmtDocs().get(0);
OpsDoc doc1 = all.getStmtDocs().get(0);
// assertThat(doc1.getName()).isEqualTo("doc1");
assertThat(doc1.getBlocks()).hasSize(1);
StmtsBlock block1 = doc1.getBlocks().get(0);
OpsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getBindings()).hasSize(0);
assertThat(block1.getTags()).hasSize(0);
assertThat(block1.getOps()).hasSize(3);

62
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtVariationTests.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -30,72 +30,72 @@ public class StmtVariationTests {
@Test
public void testListStmtsOnly() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList docs = ysl.loadString(logger,
"statements:\n" +
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString(
"statements:\n" +
" - first statement\n" +
" - second statement\n"
);
assertThat(docs.getStmtsDocs()).hasSize(1);
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
RawOpsDoc doc = docs.getStmtsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(2);
List<RawStmtDef> stmts = doc.getRawStmtDefs();
RawStmtDef s0 = stmts.get(0);
List<RawOpDef> stmts = doc.getRawStmtDefs();
RawOpDef s0 = stmts.get(0);
assertThat(s0.getName()).isEqualTo("stmt1");
assertThat(s0.getStmt()).isEqualTo("first statement");
RawStmtDef s1 = stmts.get(1);
RawOpDef s1 = stmts.get(1);
assertThat(s1.getName()).isEqualTo("stmt2");
assertThat(s1.getStmt()).isEqualTo("second statement");
}
@Test
public void testSingleEntryMapStmtsOnly() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList docs = ysl.loadString(logger,
"statements:\n" +
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString(
"statements:\n" +
" - s1: statement one\n" +
" - s2: statement two\n"
);
assertThat(docs.getStmtsDocs()).hasSize(1);
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
RawOpsDoc doc = docs.getStmtsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(2);
List<RawStmtDef> stmts = doc.getRawStmtDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class);
List<RawOpDef> stmts = doc.getRawStmtDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(0).getName()).isEqualTo("s1");
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(1).getName()).isEqualTo("s2");
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
}
@Test
public void testMapStmtsOnly() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList docs = ysl.loadString(logger,
"statements:\n" +
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString(
"statements:\n" +
" - name: s1\n" +
" stmt: statement one\n" +
" - name: s2\n" +
" stmt: statement two\n"
);
assertThat(docs.getStmtsDocs()).hasSize(1);
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
RawOpsDoc doc = docs.getStmtsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(2);
List<RawStmtDef> stmts = doc.getRawStmtDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class);
List<RawOpDef> stmts = doc.getRawStmtDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(0).getName()).isEqualTo("s1");
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(1).getName()).isEqualTo("s2");
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
}
@Test
public void testMixedForms() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList docs = ysl.loadString(logger,
"statement:\n" +
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString(
"statement:\n" +
" - name: s1\n" +
" stmt: statement one\n" +
" - statement two\n" +
@ -104,19 +104,19 @@ public class StmtVariationTests {
" type: organic\n"
);
assertThat(docs.getStmtsDocs()).hasSize(1);
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
RawOpsDoc doc = docs.getStmtsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(4);
List<RawStmtDef> stmts = doc.getRawStmtDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class);
List<RawOpDef> stmts = doc.getRawStmtDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(0).getName()).isEqualTo("s1");
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(1).getName()).isEqualTo("stmt2");
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
assertThat(stmts.get(2)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(2)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(2).getName()).isEqualTo("s3");
assertThat(stmts.get(2).getStmt()).isEqualTo("statement three");
assertThat(stmts.get(3)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(3)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(3).getName()).isEqualTo("ST4");
assertThat(stmts.get(3).getStmt()).isEqualTo("statement four");
assertThat(stmts.get(3).getParams().get("type")).isEqualTo("organic");

14
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java
View File

@ -18,10 +18,11 @@ package io.nosqlbench.engine.api.activityconfig.specifications;
import com.google.gson.*;
import com.google.gson.reflect.TypeToken;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.nb.spectest.api.STAssemblyValidator;
import io.nosqlbench.nb.spectest.core.STNodeAssembly;
import io.nosqlbench.nb.spectest.loaders.STDefaultLoader;
@ -107,7 +108,7 @@ public class YamlSpecValidator implements STAssemblyValidator {
}.getType();
List<Map<String, Object>> expectedList = gson.fromJson(json, type);
StmtsDocList stmtsDocs = StatementsLoader.loadString(yaml, Map.of());
OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
List<OpTemplate> stmts = stmtsDocs.getStmts();
List<Map<String, Object>> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList());
@ -174,10 +175,9 @@ public class YamlSpecValidator implements STAssemblyValidator {
throw new RuntimeException("unknown type in comparator: " + json);
}
} catch (Exception e) {
StringBuilder sb = new StringBuilder();
sb.append("error while verifying model:\n");
sb.append(" path: ").append(testset.getPath().toString()).append("\n");
sb.append(" line: ").append(testset.getLineNumber()).append("\n");
String sb = "error while verifying model:\n" +
" path: " + testset.getPath().toString() + "\n" +
" line: " + testset.getLineNumber() + "\n";
logger.error(sb + ": " + e.getMessage(), e);
throw new RuntimeException(e);

16
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
@ -26,21 +26,21 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class StmtDetailOverrideTest {
private static final Logger logger = LogManager.getLogger(StmtDetailOverrideTest.class);
public class OpDetailOverrideTest {
private static final Logger logger = LogManager.getLogger(OpDetailOverrideTest.class);
@Test
public void testStmtOverrides() {
StmtsDocList doclist = StatementsLoader.loadPath(logger, "testdocs/stmt_details.yaml");
OpsDocList doclist = OpsLoader.loadPath("testdocs/stmt_details.yaml", Map.of());
assertThat(doclist).isNotNull();
assertThat(doclist.getStmtDocs()).hasSize(1);
StmtsDoc doc1 = doclist.getStmtDocs().get(0);
OpsDoc doc1 = doclist.getStmtDocs().get(0);
assertThat(doc1.getBlocks()).hasSize(2);
StmtsBlock doc1block0 = doc1.getBlocks().get(0);
OpsBlock doc1block0 = doc1.getBlocks().get(0);
assertThat(doc1block0.getOps().size()).isEqualTo(1);
OpTemplate s = doc1block0.getOps().get(0);
assertThat(s.getName()).isEqualTo("block0--stmt1");
@ -49,7 +49,7 @@ public class StmtDetailOverrideTest {
assertThat(s.getParams()).hasSize(1);
assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","block0--stmt1"));
StmtsBlock doc1block1 = doc1.getBlocks().get(1);
OpsBlock doc1block1 = doc1.getBlocks().get(1);
List<OpTemplate> stmts = doc1block1.getOps();
assertThat(stmts).hasSize(4);

26
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import org.assertj.core.data.MapEntry;
import org.junit.jupiter.api.BeforeAll;
import org.apache.logging.log4j.Logger;
@ -29,10 +29,10 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class StmtsDocListTest {
public class OpsDocListTest {
private static final Logger logger = LogManager.getLogger(StmtsDocListTest.class);
private static StmtsDocList doclist;
private static final Logger logger = LogManager.getLogger(OpsDocListTest.class);
private static OpsDocList doclist;
private final LinkedHashMap<String, String> doc0bindings = new LinkedHashMap<String, String>() {{
put("b2", "b2d");
@ -50,17 +50,17 @@ public class StmtsDocListTest {
@BeforeAll
public static void testLoadYaml() {
doclist = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
doclist = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
}
@Test
public void testBlocksInheritDocData() {
assertThat(doclist).isNotNull();
assertThat(doclist.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = doclist.getStmtDocs().get(0);
OpsDoc doc1 = doclist.getStmtDocs().get(0);
assertThat(doc1.getBlocks()).hasSize(1);
StmtsBlock doc1block0 = doc1.getBlocks().get(0);
OpsBlock doc1block0 = doc1.getBlocks().get(0);
assertThat(doc1.getTags()).isEqualTo(doc0tags);
assertThat(doc1.getBindings()).isEqualTo(doc0bindings);
@ -75,11 +75,11 @@ public class StmtsDocListTest {
@Test
public void testStmtInheritsBlockData() {
StmtsDoc doc0 = doclist.getStmtDocs().get(0);
OpsDoc doc0 = doclist.getStmtDocs().get(0);
List<OpTemplate> stmts1 = doc0.getBlocks().get(0).getOps();
assertThat(stmts1).hasSize(2);
StmtsBlock block0 = doc0.getBlocks().get(0);
OpsBlock block0 = doc0.getBlocks().get(0);
assertThat(block0.getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
assertThat(block0.getParams()).containsExactly(MapEntry.entry("param1","value1"));
assertThat(block0.getTags()).containsExactly(MapEntry.entry("atagname","atagvalue"));
@ -96,8 +96,8 @@ public class StmtsDocListTest {
@Test
public void testBlockLayersDocData() {
StmtsDoc doc1 = doclist.getStmtDocs().get(1);
StmtsBlock block0 = doc1.getBlocks().get(0);
OpsDoc doc1 = doclist.getStmtDocs().get(1);
OpsBlock block0 = doc1.getBlocks().get(0);
Map<String, String> doc1block0tags = block0.getTags();
Map<String, String> doc1block0params = block0.getParamsAsText();
@ -119,7 +119,7 @@ public class StmtsDocListTest {
@Test
public void testStmtsGetter() {
StmtsDoc doc1 = doclist.getStmtDocs().get(1);
OpsDoc doc1 = doclist.getStmtDocs().get(1);
List<OpTemplate> stmts = doc1.getStmts();
assertThat(stmts).hasSize(4);
}

20
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtOpTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedOpTemplateTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,33 +16,35 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class ParsedStmtOpTest {
private static final Logger logger = LogManager.getLogger(ParsedStmtOpTest.class);
private static StmtsDocList doclist;
public class ParsedOpTemplateTest {
private static final Logger logger = LogManager.getLogger(ParsedOpTemplateTest.class);
private static OpsDocList doclist;
@BeforeAll
public static void testLoadYaml() {
doclist = StatementsLoader.loadPath(logger, "testdocs/bindings.yaml");
doclist = OpsLoader.loadPath("testdocs/bindings.yaml", Map.of());
}
@Test
public void testBasicParser() {
StmtsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0);
OpsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0);
OpTemplate stmtDef0 = block0.getOps().get(0);
ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow();
assertThat(parsed0.getMissing()).containsExactly("delta");
assertThat(parsed0.hasError()).isTrue();
StmtsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1);
OpsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1);
OpTemplate stmtDef1 = block1.getOps().get(0);
ParsedStringTemplate parsed1 = stmtDef1.getParsed().orElseThrow();
assertThat(parsed1.getMissing()).containsExactly();
@ -51,7 +53,7 @@ public class ParsedStmtOpTest {
@Test
public void testMultipleBindingUsage() {
StmtsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2);
OpsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2);
OpTemplate stmtDef0 = block2.getOps().get(0);
ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow();

9
adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,10 +16,11 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpData;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.api.config.standard.ConfigModel;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.api.config.standard.Param;
@ -68,7 +69,7 @@ public class ParsedOpTest {
params:
ps1: "param-one"
""";
StmtsDocList stmtsDocs = StatementsLoader.loadString(opt, cfg.getMap());
OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null);
assertThat(stmtsDocs.getStmts().size()).isEqualTo(1);
OpTemplate opTemplate = stmtsDocs.getStmts().get(0);
ParsedOp parsedOp = new ParsedOp(opTemplate, cfg);

24
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -37,10 +37,10 @@ import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DryRunOpDispenserWrapper;
@ -549,10 +549,10 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
StmtsDocList stmtsDocList = loadStmtsDocList();
OpsDocList opsDocList = loadStmtsDocList();
List<OpTemplate> unfilteredOps = stmtsDocList.getStmts();
List<OpTemplate> filteredOps = stmtsDocList.getStmts(tagfilter);
List<OpTemplate> unfilteredOps = opsDocList.getStmts();
List<OpTemplate> filteredOps = opsDocList.getStmts(tagfilter);
if (filteredOps.size() == 0) {
if (unfilteredOps.size() > 0) { // There were no ops, and it was because they were all filtered out
@ -563,7 +563,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
// In this case, let's try to synthesize the ops as long as at least a default driver was provided
if (defaultDriverAdapter.isPresent() && defaultDriverAdapter.get() instanceof SyntheticOpTemplateProvider sotp) {
filteredOps = sotp.getSyntheticOpTemplates(stmtsDocList, getActivityDef().getParams());
filteredOps = sotp.getSyntheticOpTemplates(opsDocList, getActivityDef().getParams());
Objects.requireNonNull(filteredOps);
if (filteredOps.size() == 0) {
throw new BasicError("Attempted to create synthetic ops from driver '" + defaultDriverAdapter.get().getAdapterName() + "'" +
@ -573,7 +573,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
throw new BasicError("""
No op templates were provided. You must provide one of these activity parameters:
1) workload=some.yaml
2) op='inline template
2) op='inline template'
3) driver=stdout (or any other drive that can synthesize ops)""");
}
}
@ -661,20 +661,20 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
return planner.resolve();
}
protected StmtsDocList loadStmtsDocList() {
protected OpsDocList loadStmtsDocList() {
try {
Optional<String> stmt = activityDef.getParams().getOptionalString("op", "stmt", "statement");
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (stmt.isPresent()) {
workloadSource = "commandline:" + stmt.get();
return StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams());
return OpsLoader.loadString(stmt.get(), OpTemplateFormat.inline, activityDef.getParams(), null);
} else if (op_yaml_loc.isPresent()) {
workloadSource = "yaml:" + op_yaml_loc.get();
return StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities");
return OpsLoader.loadPath(op_yaml_loc.get(), activityDef.getParams(), "activities");
}
return StmtsDocList.none();
return OpsDocList.none();
} catch (Exception e) {
throw new OpConfigError("Error loading op templates: " + e, workloadSource, e);

11
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java
View File

@ -20,9 +20,9 @@ import io.nosqlbench.api.config.standard.*;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.OpDispenser;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
@ -54,11 +54,12 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
public StandardActivity(ActivityDef activityDef) {
super(activityDef);
OpsDocList workload;
Optional<String> yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (yaml_loc.isPresent()) {
Map<String, Object> disposable = new LinkedHashMap<>(activityDef.getParams());
StmtsDocList workload = StatementsLoader.loadPath(logger, yaml_loc.get(), disposable, "activities");
workload = OpsLoader.loadPath(yaml_loc.get(), disposable, "activities");
yamlmodel = workload.getConfigModel();
} else {
yamlmodel = ConfigModel.of(StandardActivity.class).asReadOnly();
@ -163,11 +164,11 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
}
@Override
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String, Object> cfg) {
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> cfg) {
List<OpTemplate> opTemplates = new ArrayList<>();
for (DriverAdapter adapter : adapters.values()) {
if (adapter instanceof SyntheticOpTemplateProvider sotp) {
List<OpTemplate> newTemplates = sotp.getSyntheticOpTemplates(stmtsDocList, cfg);
List<OpTemplate> newTemplates = sotp.getSyntheticOpTemplates(opsDocList, cfg);
opTemplates.addAll(newTemplates);
}
}

8
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -18,8 +18,8 @@ package io.nosqlbench.engine.api.activityimpl.uniform;
import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.api.config.standard.NBConfigModel;
@ -69,7 +69,7 @@ public class StandardActivityType<A extends StandardActivity<?,?>> extends Simpl
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (op_yaml_loc.isPresent()) {
Map<String,Object> disposable = new LinkedHashMap<>(activityDef.getParams());
StmtsDocList workload = StatementsLoader.loadPath(logger, op_yaml_loc.get(), disposable, "activities");
OpsDocList workload = OpsLoader.loadPath(op_yaml_loc.get(), disposable, "activities");
cfgModel=cfgModel.add(workload.getConfigModel());
}
NBConfiguration cfg = cfgModel.apply(activityDef.getParams());

22
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -16,10 +16,10 @@
package io.nosqlbench.engine.api.scenarios;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
@ -53,7 +53,7 @@ public class NBCLIScenarioParser {
.prefix("activities")
.prefix(includes)
.name(workload)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.first();
return found.isPresent();
}
@ -67,7 +67,7 @@ public class NBCLIScenarioParser {
.prefix("activities")
.prefix(includes)
.name(workloadName)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.first();
//
Content<?> workloadContent = found.orElseThrow();
@ -111,10 +111,10 @@ public class NBCLIScenarioParser {
.prefix(SEARCH_IN)
.prefix(includes)
.name(workloadName)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.first().orElseThrow();
// TODO: The yaml needs to be parsed with arguments from each command independently to support template vars
StmtsDocList scenariosYaml = StatementsLoader.loadContent(logger, yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
OpsDocList scenariosYaml = OpsLoader.loadContent(yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
Scenarios scenarios = scenariosYaml.getDocScenarios();
String[] nameparts = scenarioName.split("\\.",2);
@ -314,12 +314,12 @@ public class NBCLIScenarioParser {
}
Content<?> content = NBIO.all().prefix(SEARCH_IN)
.name(referenced).extension(RawStmtsLoader.YAML_EXTENSIONS)
.name(referenced).extension(RawOpsLoader.YAML_EXTENSIONS)
.one();
StmtsDocList stmts = null;
OpsDocList stmts = null;
try {
stmts = StatementsLoader.loadContent(logger, content, Map.of());
stmts = OpsLoader.loadContent(content, Map.of());
if (stmts.getStmtDocs().size() == 0) {
logger.warn("Encountered yaml with no docs in '" + referenced + "'");
continue;
@ -379,7 +379,7 @@ public class NBCLIScenarioParser {
List<Content<?>> activities = searchin
.prefix(includes)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.list();
return filterForScenarios(activities);

15
engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -18,9 +18,10 @@ package io.nosqlbench.engine.api.templating;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
@ -34,10 +35,10 @@ public class CommandTemplateTest {
@Test
public void testCommandTemplate() {
StmtsDocList stmtsDocs = StatementsLoader.loadString("" +
OpsDocList stmtsDocs = OpsLoader.loadString("" +
"statements:\n" +
" - s1: test1=foo test2=bar",
Map.of());
OpTemplateFormat.yaml, Map.of(), null);
OpTemplate optpl = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(optpl);
assertThat(ct.isStatic()).isTrue();
@ -46,12 +47,12 @@ public class CommandTemplateTest {
@Test
public void testCommandTemplateFormat() {
Gson gson = new GsonBuilder().setPrettyPrinting().create();
StmtsDocList stmtsDocs = StatementsLoader.loadString("" +
OpsDocList stmtsDocs = OpsLoader.loadString("" +
"statements:\n" +
" - s1: test1=foo test2={bar}\n" +
" bindings:\n" +
" bar: NumberNameToString();\n",
Map.of()
OpTemplateFormat.yaml, Map.of(), null
);
OpTemplate optpl = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(optpl);

6
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -30,7 +30,7 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogDu
import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogImporterUtility;
import io.nosqlbench.engine.api.activityapi.input.InputType;
import io.nosqlbench.engine.api.activityapi.output.OutputType;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.engine.core.annotation.Annotators;
import io.nosqlbench.engine.core.lifecycle.process.NBCLIErrorHandler;
import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader;
@ -304,7 +304,7 @@ public class NBCLI implements Function<String[], Integer> {
Optional<Content<?>> tocopy = NBIO.classpath()
.prefix("activities")
.prefix(options.wantsIncludes())
.name(resourceToCopy).extension(RawStmtsLoader.YAML_EXTENSIONS).first();
.name(resourceToCopy).extension(RawOpsLoader.YAML_EXTENSIONS).first();
if (tocopy.isEmpty()) {

12
engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,9 +16,9 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import org.junit.jupiter.api.Test;
import java.util.List;
@ -33,12 +33,12 @@ public class NBCLIScenarioParserTemplateVarTest {
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0);
System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n");
StmtsDocList workload2 = StatementsLoader.loadPath(null, cmds.get(1).getArg("workload"),cmds.get(1).getParams());
OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArg("workload"),cmds.get(1).getParams());
OpTemplate optpl2 = workload2.getStmts().get(0);
System.out.println("op from cmd2:"+optpl2);
assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n");
@ -50,7 +50,7 @@ public class NBCLIScenarioParserTemplateVarTest {
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0);
System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n");

8
nbr-examples/pom.xml
View File

@ -57,6 +57,14 @@
<directory>src/test/resources</directory>
<filtering>false</filtering> <!-- exclusion from defaults -->
</testResource>
<testResource>
<directory>src/test/resources</directory>
<filtering>true</filtering>
<includes>
<include>**/log4j2-test.xml</include>
<include>**/log4j2.xml</include>
</includes>
</testResource>
</testResources>
<plugins>