IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-11-23 01:06:47 -06:00
Code Issues Packages Projects Releases Wiki Activity

jsonnet diagnostic improvements

Browse Source
This commit is contained in:
Jonathan Shook 2023-01-26 18:20:19 -06:00
parent dc9684dd98
commit ef4c84ddaa
20 changed files with 190 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java
View File

@ -16,13 +16,13 @@
package io.nosqlbench.cqlgen.core;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.api.BindingsLibrary;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.cqlgen.api.BindingsLibrary;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import java.io.File;
import java.io.InputStream;
@ -36,13 +36,16 @@ public class CGDefaultCqlBindings implements BindingsLibrary {
public final static String DEFAULT_BINDINGS_FILE = "bindings-cqlgen.yaml";
public CGDefaultCqlBindings() {
String yamlContent = NBIO.all()
Content<?> content = NBIO.all()
.name(DEFAULT_BINDINGS_FILE)
.first()
.map(Content::asString)
.or(() -> loadLocal(DEFAULT_BINDINGS_FILE))
.orElseThrow(() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE + ", from local dir or internally as cqlgen/" + DEFAULT_BINDINGS_FILE));
OpsDocList stmtsDocs = OpsLoader.loadString(yamlContent, Map.of());
.or(() -> NBIO.all().prefix(DEFAULT_CFG_DIR).name(DEFAULT_BINDINGS_FILE).first())
.orElseThrow(
() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE +
", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE)
);
OpsDocList stmtsDocs = OpsLoader.loadContent(content, Map.of());
this.bindings = stmtsDocs.getDocBindings();
}

3
adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java
View File

@ -19,6 +19,7 @@ package io.nosqlbench.adapter.diag;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
@ -102,6 +103,6 @@ public class DiagDriverAdapter extends BaseDriverAdapter<DiagOp, DiagSpace> impl
@Override
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> params) {
return OpsLoader.loadString("ops: 'log:level=INFO'", params).getStmts();
return OpsLoader.loadString("ops: 'log:level=INFO'", OpTemplateFormat.inline, params,null).getStmts();
}
}

3
adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java
View File

@ -21,6 +21,7 @@ import io.nosqlbench.adapter.http.core.HttpSpace;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.engine.api.templating.ParsedOp;
@ -51,7 +52,7 @@ public class HttpOpMapperTest {
}
private static ParsedOp parsedOpFor(String yaml) {
OpsDocList docs = OpsLoader.loadString(yaml, Map.of());
OpsDocList docs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
OpTemplate stmtDef = docs.getStmts().get(0);
ParsedOp parsedOp = new ParsedOp(stmtDef, cfg, List.of(adapter.getPreprocessor()));
return parsedOp;

5
adapters-api/pom.xml
View File

@ -54,6 +54,11 @@
<version>${revision}</version>
</dependency>
<dependency>
<groupId>com.databricks</groupId>
<artifactId>sjsonnet_2.13</artifactId>
<version>0.4.3</version>
</dependency>
</dependencies>

109
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java
View File

@ -16,66 +16,115 @@
package io.nosqlbench.engine.api.activityconfig;
import com.amazonaws.util.StringInputStream;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import scala.Option;
import sjsonnet.DefaultParseCache;
import sjsonnet.SjsonnetMain;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
public class OpsLoader {
private final static Logger logger = LogManager.getLogger(OpsLoader.class);
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml", "json", "jsonnet"};
public static String[] YAML_EXTENSIONS = new String[]{"yaml", "yml"};
public static OpsDocList loadContent(Content<?> content, Map<String,String> params) {
return loadString(content.get().toString(),params);
public static OpsDocList loadContent(Content<?> content, Map<String, String> params) {
OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(content.getURI());
return loadString(content.get().toString(), fmt, params, content.getURI());
}
public static OpsDocList loadPath(String path, Map<String,?> params, String... searchPaths) {
RawOpsDocList list = null;
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(content,params);
}
public static OpsDocList loadPath(
String path,
String... searchPaths) {
return loadPath(path, Map.of(), searchPaths);
public static OpsDocList loadPath(String path, Map<String, ?> params, String... searchPaths) {
String[] extensions = path.indexOf('.')>-1 ? new String[]{} : YAML_EXTENSIONS;
Content<?> foundPath = NBIO.all().prefix(searchPaths).name(path).extension(extensions).first()
.orElseThrow(() -> new RuntimeException("Unable to load path '" + path + "'"));
OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(foundPath.getURI());
return loadString(foundPath.asString(), fmt, params, foundPath.getURI());
}
public static OpsDocList loadString(String yamlContent, Map<String,?> params) {
public static OpsDocList loadString(final String sourceData, OpTemplateFormat fmt, Map<String, ?> params, URI srcuri) {
logger.trace(() -> "Applying string transformer to data:" + sourceData);
StrInterpolator transformer = new StrInterpolator(params);
String data = transformer.apply(sourceData);
if (srcuri!=null) {
logger.info("Loading workload template from '" + srcuri + "'");
}
RawOpsLoader loader = new RawOpsLoader(transformer);
RawOpsDocList rawDocList = loader.loadString(yamlContent);
OpsDocList layered = new OpsDocList(rawDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
params.remove(k);
});
return layered;
}
RawOpsDocList rawOpsDocList = switch (fmt) {
case jsonnet -> loader.loadString(evaluateJsonnet(srcuri, params));
case yaml, json -> loader.loadString(data);
case inline, stmt -> RawOpsDocList.forSingleStatement(data);
};
// TODO: itemize inline to support ParamParser
public static OpsDocList loadStmt(String statement, Map<String,?> params) {
StrInterpolator transformer = new StrInterpolator(params);
statement = transformer.apply(statement);
RawOpsDocList rawOpsDocList = RawOpsDocList.forSingleStatement(statement);
OpsDocList layered = new OpsDocList(rawOpsDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
transformer.checkpointAccesses().forEach((k, v) -> {
layered.addTemplateVariable(k, v);
params.remove(k);
});
return layered;
}
private static String evaluateJsonnet(URI uri, Map<String, ?> params) {
List<String> injected = new LinkedList<>(List.of(Path.of(uri).toString()));
params.forEach((k,v) -> {
if (v instanceof CharSequence cs) {
injected.addAll(List.of("--ext-str",k+"="+cs));
}
});
var stdoutBuffer = new ByteArrayOutputStream();
var stderrBuffer = new ByteArrayOutputStream();
var stdoutStream = new PrintStream(stdoutBuffer);
var stderrStream = new PrintStream(stderrBuffer);
StringInputStream inputStream;
try {
inputStream = new StringInputStream("");
} catch (Exception e) {
throw new RuntimeException("Error building input stream for jsonnet:" + e, e);
}
int resultStatus = SjsonnetMain.main0(
injected.toArray(new String[0]),
new DefaultParseCache(),
inputStream,
stdoutStream,
stderrStream,
new os.Path(Path.of(System.getProperty("user.dir"))),
Option.empty(),
Option.empty()
);
String stdoutOutput = stdoutBuffer.toString(StandardCharsets.UTF_8);
String stderrOutput = stderrBuffer.toString(StandardCharsets.UTF_8);
if (!stderrOutput.isEmpty()) {
logger.error("stderr output from jsonnet preprocessing: " + stderrOutput);
}
logger.info("jsonnet processing read '" + uri +"', rendered " + stdoutOutput.split("\n").length + " lines.");
logger.trace("jsonnet result:\n" + stdoutOutput);
return stdoutOutput;
}
}

2
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java
View File

@ -78,7 +78,7 @@ public class RawOpsLoader {
}
private RawOpsDocList parseYaml(String data) {
public RawOpsDocList parseYaml(String data) {
LoadSettings loadSettings = LoadSettings.builder().build();
Load yaml = new Load(loadSettings);
Iterable<Object> objects = yaml.loadAllFromString(data);

58
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplateFormat.java
View File

@ -16,13 +16,46 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import java.net.URI;
import java.nio.file.Path;
/**
* The formats which are recognized as source data for the workload. Any serialization may be supported
* which can be converted from a character string to an {@link io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList} structure.
*
* Those which are derived from paths may be determined by their filename extension. Others, which are provided from internal
* NoSQLBench sources, may only be invoked explicitly.
*/
public enum OpTemplateFormat {
yaml("yaml","yml"),
json("json","json5"),
jsonnet("jsonnet","jsonnet5"),
inline();
/**
* The source is in YAML format
*/
yaml("yaml", "yml"),
/**
* The source is in JSON format
*/
json("json", "json5"),
/**
* The source is in jsonnet format, suitable for pre-processing with context data
*/
jsonnet("jsonnet", "jsonnet5"),
/**
* The source is inline, meaning {@link io.nosqlbench.api.config.params.ParamsParser} format.
* This is similar to the {@link #stmt} format except that it is parsed for internal op structure.
* This format is not detectable by source path, and thus can only be used when provided directly
* from the caller.
*/
inline(),
/**
* The source is in single-statement form, meaning that it is known to be the value of the 'stmt'
* field of an op template. This format is not detectable by source path, and thus can only be used when provided directly
* from the caller.
*/
stmt();
private final String[] pathExtensions;
@ -30,9 +63,24 @@ public enum OpTemplateFormat {
this.pathExtensions = pathExtensions;
}
public static OpTemplateFormat valueOfURI(URI uri) {
var fullName = uri.toString();
String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase();
for (OpTemplateFormat value : values()) {
for (String pathExtension : value.pathExtensions) {
if (pathExtension.equals(extension)) {
return value;
}
}
}
throw new RuntimeException("Unable to determine source format for " + uri);
}
public static OpTemplateFormat valueOfPath(Path path) {
var fullName = path.toString();
String extension = fullName.substring(fullName.lastIndexOf('.')).toLowerCase();
String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase();
for (OpTemplateFormat value : values()) {
for (String pathExtension : value.pathExtensions) {

2
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java
View File

@ -161,7 +161,7 @@ public class OpsDocList implements Iterable<OpsDoc> {
}
public static NBConfigModelExpander TEMPLATE_VAR_EXPANDER = workload -> {
OpsDocList loaded = OpsLoader.loadPath((String) workload, "activities");
OpsDocList loaded = OpsLoader.loadPath((String) workload, Map.of(), "activities");
return loaded.getConfigModel();
};

4
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java
View File

@ -29,7 +29,7 @@ public class OpsLoaderTest {
@Test
public void testTemplateVarSubstitution() {
OpsDocList stmtsDocs = OpsLoader.loadPath("activities/template_vars", "src/test/resources");
OpsDocList stmtsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources");
assertThat(stmtsDocs).isNotNull();
List<OpsDoc> docs = stmtsDocs.getStmtDocs();
assertThat(docs).hasSize(1);
@ -46,7 +46,7 @@ public class OpsLoaderTest {
public void testInvalidYamlProperties() {
Exception caught = null;
try {
OpsLoader.loadPath("activities/invalid_prop", "src/test/resources");
OpsLoader.loadPath("activities/invalid_prop", Map.of(),"src/test/resources");
} catch (Exception e) {
caught = e;
}

12
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java
View File

@ -37,7 +37,7 @@ public class OpDefTest {
@Test
public void testLayering() {
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2);
OpsDoc doc1 = all.getStmtDocs().get(0);
@ -64,7 +64,7 @@ public class OpDefTest {
@Test
public void testStatementRendering() {
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2);
OpsDoc doc1 = all.getStmtDocs().get(0);
@ -79,7 +79,7 @@ public class OpDefTest {
@Test
public void testConsumableMapState() {
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc block1 = docs.get(1);
List<OpTemplate> stmts = block1.getStmts();
@ -94,7 +94,7 @@ public class OpDefTest {
@Test
public void testMapOfMaps() {
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc0 = docs.get(0);
assertThat(doc0.getName()).isEqualTo("map-of-maps");
@ -115,7 +115,7 @@ public class OpDefTest {
@Test
public void testBasicStringStmt() {
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc1 = docs.get(1);
assertThat(doc1.getName()).isEqualTo("string-statement");
@ -130,7 +130,7 @@ public class OpDefTest {
@Test
public void testListOfNamedMap() {
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc2 = docs.get(2);
assertThat(doc2.getName()).isEqualTo("list-of-named-map");

3
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java
View File

@ -24,6 +24,7 @@ import org.apache.logging.log4j.LogManager;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
@ -35,7 +36,7 @@ public class StmtEscapingTest {
@BeforeAll
public static void testLayering() {
OpsDocList all = OpsLoader.loadPath("testdocs/escaped_stmts.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/escaped_stmts.yaml", Map.of());
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(1);
OpsDoc doc1 = all.getStmtDocs().get(0);

3
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java
View File

@ -21,6 +21,7 @@ import com.google.gson.reflect.TypeToken;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.nb.spectest.api.STAssemblyValidator;
import io.nosqlbench.nb.spectest.core.STNodeAssembly;
@ -107,7 +108,7 @@ public class YamlSpecValidator implements STAssemblyValidator {
}.getType();
List<Map<String, Object>> expectedList = gson.fromJson(json, type);
OpsDocList stmtsDocs = OpsLoader.loadString(yaml, Map.of());
OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
List<OpTemplate> stmts = stmtsDocs.getStmts();
List<Map<String, Object>> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList());

2
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java
View File

@ -32,7 +32,7 @@ public class OpDetailOverrideTest {
@Test
public void testStmtOverrides() {
OpsDocList doclist = OpsLoader.loadPath("testdocs/stmt_details.yaml");
OpsDocList doclist = OpsLoader.loadPath("testdocs/stmt_details.yaml", Map.of());
assertThat(doclist).isNotNull();

2
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java
View File

@ -50,7 +50,7 @@ public class OpsDocListTest {
@BeforeAll
public static void testLoadYaml() {
doclist = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
doclist = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of());
}
@Test

4
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedOpTemplateTest.java
View File

@ -23,6 +23,8 @@ import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class ParsedOpTemplateTest {
@ -31,7 +33,7 @@ public class ParsedOpTemplateTest {
@BeforeAll
public static void testLoadYaml() {
doclist = OpsLoader.loadPath("testdocs/bindings.yaml");
doclist = OpsLoader.loadPath("testdocs/bindings.yaml", Map.of());
}
@Test

3
adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java
View File

@ -19,6 +19,7 @@ package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpData;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.api.config.standard.ConfigModel;
import io.nosqlbench.api.config.standard.NBConfiguration;
@ -68,7 +69,7 @@ public class ParsedOpTest {
params:
ps1: "param-one"
""";
OpsDocList stmtsDocs = OpsLoader.loadString(opt, cfg.getMap());
OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null);
assertThat(stmtsDocs.getStmts().size()).isEqualTo(1);
OpTemplate opTemplate = stmtsDocs.getStmts().get(0);
ParsedOp parsedOp = new ParsedOp(opTemplate, cfg);

3
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -39,6 +39,7 @@ import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
@ -667,7 +668,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (stmt.isPresent()) {
workloadSource = "commandline:" + stmt.get();
return OpsLoader.loadStmt(stmt.get(), activityDef.getParams());
return OpsLoader.loadString(stmt.get(), OpTemplateFormat.inline, activityDef.getParams(), null);
} else if (op_yaml_loc.isPresent()) {
workloadSource = "yaml:" + op_yaml_loc.get();
return OpsLoader.loadPath(op_yaml_loc.get(), activityDef.getParams(), "activities");

3
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java
View File

@ -54,11 +54,12 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
public StandardActivity(ActivityDef activityDef) {
super(activityDef);
OpsDocList workload;
Optional<String> yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (yaml_loc.isPresent()) {
Map<String, Object> disposable = new LinkedHashMap<>(activityDef.getParams());
OpsDocList workload = OpsLoader.loadPath(yaml_loc.get(), disposable, "activities");
workload = OpsLoader.loadPath(yaml_loc.get(), disposable, "activities");
yamlmodel = workload.getConfigModel();
} else {
yamlmodel = ConfigModel.of(StandardActivity.class).asReadOnly();

5
engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java
View File

@ -20,6 +20,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -37,7 +38,7 @@ public class CommandTemplateTest {
OpsDocList stmtsDocs = OpsLoader.loadString("" +
"statements:\n" +
" - s1: test1=foo test2=bar",
Map.of());
OpTemplateFormat.yaml, Map.of(), null);
OpTemplate optpl = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(optpl);
assertThat(ct.isStatic()).isTrue();
@ -51,7 +52,7 @@ public class CommandTemplateTest {
" - s1: test1=foo test2={bar}\n" +
" bindings:\n" +
" bar: NumberNameToString();\n",
Map.of()
OpTemplateFormat.yaml, Map.of(), null
);
OpTemplate optpl = stmtsDocs.getStmts().get(0);
CommandTemplate ct = new CommandTemplate(optpl);

8
nbr-examples/pom.xml
View File

@ -57,6 +57,14 @@
<directory>src/test/resources</directory>
<filtering>false</filtering> <!-- exclusion from defaults -->
</testResource>
<testResource>
<directory>src/test/resources</directory>
<filtering>true</filtering>
<includes>
<include>**/log4j2-test.xml</include>
<include>**/log4j2.xml</include>
</includes>
</testResource>
</testResources>
<plugins>