diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java index c6478f093..d933991d4 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java @@ -16,13 +16,13 @@ package io.nosqlbench.cqlgen.core; -import io.nosqlbench.cqlgen.model.CqlColumnBase; -import io.nosqlbench.cqlgen.binders.Binding; -import io.nosqlbench.cqlgen.api.BindingsLibrary; -import io.nosqlbench.engine.api.activityconfig.OpsLoader; -import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.api.content.Content; import io.nosqlbench.api.content.NBIO; +import io.nosqlbench.cqlgen.api.BindingsLibrary; +import io.nosqlbench.cqlgen.binders.Binding; +import io.nosqlbench.cqlgen.model.CqlColumnBase; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import java.io.File; import java.io.InputStream; @@ -36,13 +36,16 @@ public class CGDefaultCqlBindings implements BindingsLibrary { public final static String DEFAULT_BINDINGS_FILE = "bindings-cqlgen.yaml"; public CGDefaultCqlBindings() { - String yamlContent = NBIO.all() + Content content = NBIO.all() .name(DEFAULT_BINDINGS_FILE) .first() - .map(Content::asString) - .or(() -> loadLocal(DEFAULT_BINDINGS_FILE)) - .orElseThrow(() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE + ", from local dir or internally as cqlgen/" + DEFAULT_BINDINGS_FILE)); - OpsDocList stmtsDocs = OpsLoader.loadString(yamlContent, Map.of()); + .or(() -> NBIO.all().prefix(DEFAULT_CFG_DIR).name(DEFAULT_BINDINGS_FILE).first()) + .orElseThrow( + () -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE + + ", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE) + ); + + OpsDocList stmtsDocs = OpsLoader.loadContent(content, Map.of()); this.bindings = stmtsDocs.getDocBindings(); } diff --git a/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java b/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java index e2b732fd6..1e20c84f5 100644 --- a/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java +++ b/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.diag; import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.activityimpl.OpMapper; import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter; @@ -102,6 +103,6 @@ public class DiagDriverAdapter extends BaseDriverAdapter impl @Override public List getSyntheticOpTemplates(OpsDocList opsDocList, Map params) { - return OpsLoader.loadString("ops: 'log:level=INFO'", params).getStmts(); + return OpsLoader.loadString("ops: 'log:level=INFO'", OpTemplateFormat.inline, params,null).getStmts(); } } diff --git a/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java b/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java index 719243ed0..42b9036d5 100644 --- a/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java +++ b/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java @@ -21,6 +21,7 @@ import io.nosqlbench.adapter.http.core.HttpSpace; import io.nosqlbench.api.config.standard.NBConfiguration; import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache; import io.nosqlbench.engine.api.templating.ParsedOp; @@ -51,7 +52,7 @@ public class HttpOpMapperTest { } private static ParsedOp parsedOpFor(String yaml) { - OpsDocList docs = OpsLoader.loadString(yaml, Map.of()); + OpsDocList docs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null); OpTemplate stmtDef = docs.getStmts().get(0); ParsedOp parsedOp = new ParsedOp(stmtDef, cfg, List.of(adapter.getPreprocessor())); return parsedOp; diff --git a/adapters-api/pom.xml b/adapters-api/pom.xml index 0d9c2219a..52bedd0df 100644 --- a/adapters-api/pom.xml +++ b/adapters-api/pom.xml @@ -54,6 +54,11 @@ ${revision} + + com.databricks + sjsonnet_2.13 + 0.4.3 + diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java index 64edc1673..a83c61c10 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java @@ -16,66 +16,115 @@ package io.nosqlbench.engine.api.activityconfig; +import com.amazonaws.util.StringInputStream; import io.nosqlbench.api.content.Content; import io.nosqlbench.api.content.NBIO; -import io.nosqlbench.api.errors.BasicError; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.templating.StrInterpolator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import scala.Option; +import sjsonnet.DefaultParseCache; +import sjsonnet.SjsonnetMain; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.LinkedList; +import java.util.List; import java.util.Map; -import java.util.Optional; public class OpsLoader { private final static Logger logger = LogManager.getLogger(OpsLoader.class); - public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml", "json", "jsonnet"}; + public static String[] YAML_EXTENSIONS = new String[]{"yaml", "yml"}; - public static OpsDocList loadContent(Content content, Map params) { - return loadString(content.get().toString(),params); + public static OpsDocList loadContent(Content content, Map params) { + OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(content.getURI()); + return loadString(content.get().toString(), fmt, params, content.getURI()); } - public static OpsDocList loadPath(String path, Map params, String... searchPaths) { - RawOpsDocList list = null; - Optional> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first(); - String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path)); - return loadString(content,params); - } - public static OpsDocList loadPath( - String path, - String... searchPaths) { - return loadPath(path, Map.of(), searchPaths); + public static OpsDocList loadPath(String path, Map params, String... searchPaths) { + String[] extensions = path.indexOf('.')>-1 ? new String[]{} : YAML_EXTENSIONS; + + Content foundPath = NBIO.all().prefix(searchPaths).name(path).extension(extensions).first() + .orElseThrow(() -> new RuntimeException("Unable to load path '" + path + "'")); + OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(foundPath.getURI()); + return loadString(foundPath.asString(), fmt, params, foundPath.getURI()); } - public static OpsDocList loadString(String yamlContent, Map params) { + public static OpsDocList loadString(final String sourceData, OpTemplateFormat fmt, Map params, URI srcuri) { + logger.trace(() -> "Applying string transformer to data:" + sourceData); StrInterpolator transformer = new StrInterpolator(params); + String data = transformer.apply(sourceData); + if (srcuri!=null) { + logger.info("Loading workload template from '" + srcuri + "'"); + } + RawOpsLoader loader = new RawOpsLoader(transformer); - RawOpsDocList rawDocList = loader.loadString(yamlContent); - OpsDocList layered = new OpsDocList(rawDocList); - transformer.checkpointAccesses().forEach((k,v) -> { - layered.addTemplateVariable(k,v); - params.remove(k); - }); - return layered; - } + RawOpsDocList rawOpsDocList = switch (fmt) { + case jsonnet -> loader.loadString(evaluateJsonnet(srcuri, params)); + case yaml, json -> loader.loadString(data); + case inline, stmt -> RawOpsDocList.forSingleStatement(data); + }; + // TODO: itemize inline to support ParamParser - public static OpsDocList loadStmt(String statement, Map params) { - StrInterpolator transformer = new StrInterpolator(params); - statement = transformer.apply(statement); - RawOpsDocList rawOpsDocList = RawOpsDocList.forSingleStatement(statement); OpsDocList layered = new OpsDocList(rawOpsDocList); - transformer.checkpointAccesses().forEach((k,v) -> { - layered.addTemplateVariable(k,v); + + transformer.checkpointAccesses().forEach((k, v) -> { + layered.addTemplateVariable(k, v); params.remove(k); }); + return layered; } + private static String evaluateJsonnet(URI uri, Map params) { + List injected = new LinkedList<>(List.of(Path.of(uri).toString())); + params.forEach((k,v) -> { + if (v instanceof CharSequence cs) { + injected.addAll(List.of("--ext-str",k+"="+cs)); + } + }); + var stdoutBuffer = new ByteArrayOutputStream(); + var stderrBuffer = new ByteArrayOutputStream(); + var stdoutStream = new PrintStream(stdoutBuffer); + var stderrStream = new PrintStream(stderrBuffer); + StringInputStream inputStream; + try { + inputStream = new StringInputStream(""); + } catch (Exception e) { + throw new RuntimeException("Error building input stream for jsonnet:" + e, e); + } + + int resultStatus = SjsonnetMain.main0( + injected.toArray(new String[0]), + new DefaultParseCache(), + inputStream, + stdoutStream, + stderrStream, + new os.Path(Path.of(System.getProperty("user.dir"))), + Option.empty(), + Option.empty() + ); + + String stdoutOutput = stdoutBuffer.toString(StandardCharsets.UTF_8); + String stderrOutput = stderrBuffer.toString(StandardCharsets.UTF_8); + if (!stderrOutput.isEmpty()) { + logger.error("stderr output from jsonnet preprocessing: " + stderrOutput); + } + logger.info("jsonnet processing read '" + uri +"', rendered " + stdoutOutput.split("\n").length + " lines."); + logger.trace("jsonnet result:\n" + stdoutOutput); + + return stdoutOutput; + } } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java index bb0228164..75e722b21 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java @@ -78,7 +78,7 @@ public class RawOpsLoader { } - private RawOpsDocList parseYaml(String data) { + public RawOpsDocList parseYaml(String data) { LoadSettings loadSettings = LoadSettings.builder().build(); Load yaml = new Load(loadSettings); Iterable objects = yaml.loadAllFromString(data); diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplateFormat.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplateFormat.java index 467fde872..fc3e5e098 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplateFormat.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplateFormat.java @@ -16,13 +16,46 @@ package io.nosqlbench.engine.api.activityconfig.yaml; +import java.net.URI; import java.nio.file.Path; +/** + * The formats which are recognized as source data for the workload. Any serialization may be supported + * which can be converted from a character string to an {@link io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList} structure. + * + * Those which are derived from paths may be determined by their filename extension. Others, which are provided from internal + * NoSQLBench sources, may only be invoked explicitly. + */ public enum OpTemplateFormat { - yaml("yaml","yml"), - json("json","json5"), - jsonnet("jsonnet","jsonnet5"), - inline(); + /** + * The source is in YAML format + */ + yaml("yaml", "yml"), + + /** + * The source is in JSON format + */ + json("json", "json5"), + + /** + * The source is in jsonnet format, suitable for pre-processing with context data + */ + jsonnet("jsonnet", "jsonnet5"), + + /** + * The source is inline, meaning {@link io.nosqlbench.api.config.params.ParamsParser} format. + * This is similar to the {@link #stmt} format except that it is parsed for internal op structure. + * This format is not detectable by source path, and thus can only be used when provided directly + * from the caller. + */ + inline(), + + /** + * The source is in single-statement form, meaning that it is known to be the value of the 'stmt' + * field of an op template. This format is not detectable by source path, and thus can only be used when provided directly + * from the caller. + */ + stmt(); private final String[] pathExtensions; @@ -30,9 +63,24 @@ public enum OpTemplateFormat { this.pathExtensions = pathExtensions; } + public static OpTemplateFormat valueOfURI(URI uri) { + var fullName = uri.toString(); + String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase(); + + for (OpTemplateFormat value : values()) { + for (String pathExtension : value.pathExtensions) { + if (pathExtension.equals(extension)) { + return value; + } + } + } + throw new RuntimeException("Unable to determine source format for " + uri); + + } + public static OpTemplateFormat valueOfPath(Path path) { var fullName = path.toString(); - String extension = fullName.substring(fullName.lastIndexOf('.')).toLowerCase(); + String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase(); for (OpTemplateFormat value : values()) { for (String pathExtension : value.pathExtensions) { diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java index 7777f0c9b..68f79dd79 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java @@ -161,7 +161,7 @@ public class OpsDocList implements Iterable { } public static NBConfigModelExpander TEMPLATE_VAR_EXPANDER = workload -> { - OpsDocList loaded = OpsLoader.loadPath((String) workload, "activities"); + OpsDocList loaded = OpsLoader.loadPath((String) workload, Map.of(), "activities"); return loaded.getConfigModel(); }; diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java index f0b02be9c..53e28d5cb 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java @@ -29,7 +29,7 @@ public class OpsLoaderTest { @Test public void testTemplateVarSubstitution() { - OpsDocList stmtsDocs = OpsLoader.loadPath("activities/template_vars", "src/test/resources"); + OpsDocList stmtsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources"); assertThat(stmtsDocs).isNotNull(); List docs = stmtsDocs.getStmtDocs(); assertThat(docs).hasSize(1); @@ -46,7 +46,7 @@ public class OpsLoaderTest { public void testInvalidYamlProperties() { Exception caught = null; try { - OpsLoader.loadPath("activities/invalid_prop", "src/test/resources"); + OpsLoader.loadPath("activities/invalid_prop", Map.of(),"src/test/resources"); } catch (Exception e) { caught = e; } diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java index c0f8f3aad..b9bc680fc 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java @@ -37,7 +37,7 @@ public class OpDefTest { @Test public void testLayering() { - OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml"); + OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(2); OpsDoc doc1 = all.getStmtDocs().get(0); @@ -64,7 +64,7 @@ public class OpDefTest { @Test public void testStatementRendering() { - OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml"); + OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(2); OpsDoc doc1 = all.getStmtDocs().get(0); @@ -79,7 +79,7 @@ public class OpDefTest { @Test public void testConsumableMapState() { - OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml"); + OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); List docs = all.getStmtDocs(); OpsDoc block1 = docs.get(1); List stmts = block1.getStmts(); @@ -94,7 +94,7 @@ public class OpDefTest { @Test public void testMapOfMaps() { - OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml"); + OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); List docs = all.getStmtDocs(); OpsDoc doc0 = docs.get(0); assertThat(doc0.getName()).isEqualTo("map-of-maps"); @@ -115,7 +115,7 @@ public class OpDefTest { @Test public void testBasicStringStmt() { - OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml"); + OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); List docs = all.getStmtDocs(); OpsDoc doc1 = docs.get(1); assertThat(doc1.getName()).isEqualTo("string-statement"); @@ -130,7 +130,7 @@ public class OpDefTest { @Test public void testListOfNamedMap() { - OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml"); + OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); List docs = all.getStmtDocs(); OpsDoc doc2 = docs.get(2); assertThat(doc2.getName()).isEqualTo("list-of-named-map"); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java index 36218b17e..62235c697 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java @@ -24,6 +24,7 @@ import org.apache.logging.log4j.LogManager; import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -35,7 +36,7 @@ public class StmtEscapingTest { @BeforeAll public static void testLayering() { - OpsDocList all = OpsLoader.loadPath("testdocs/escaped_stmts.yaml"); + OpsDocList all = OpsLoader.loadPath("testdocs/escaped_stmts.yaml", Map.of()); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(1); OpsDoc doc1 = all.getStmtDocs().get(0); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java index 75867cdb2..9bbe834e8 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java @@ -21,6 +21,7 @@ import com.google.gson.reflect.TypeToken; import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.nb.spectest.api.STAssemblyValidator; import io.nosqlbench.nb.spectest.core.STNodeAssembly; @@ -107,7 +108,7 @@ public class YamlSpecValidator implements STAssemblyValidator { }.getType(); List> expectedList = gson.fromJson(json, type); - OpsDocList stmtsDocs = OpsLoader.loadString(yaml, Map.of()); + OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null); List stmts = stmtsDocs.getStmts(); List> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList()); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java index 14535d711..30da12cce 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java @@ -32,7 +32,7 @@ public class OpDetailOverrideTest { @Test public void testStmtOverrides() { - OpsDocList doclist = OpsLoader.loadPath("testdocs/stmt_details.yaml"); + OpsDocList doclist = OpsLoader.loadPath("testdocs/stmt_details.yaml", Map.of()); assertThat(doclist).isNotNull(); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java index 834b41420..436d95f61 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java @@ -50,7 +50,7 @@ public class OpsDocListTest { @BeforeAll public static void testLoadYaml() { - doclist = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml"); + doclist = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); } @Test diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedOpTemplateTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedOpTemplateTest.java index 6c0ecfab6..dc0eb9887 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedOpTemplateTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedOpTemplateTest.java @@ -23,6 +23,8 @@ import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; public class ParsedOpTemplateTest { @@ -31,7 +33,7 @@ public class ParsedOpTemplateTest { @BeforeAll public static void testLoadYaml() { - doclist = OpsLoader.loadPath("testdocs/bindings.yaml"); + doclist = OpsLoader.loadPath("testdocs/bindings.yaml", Map.of()); } @Test diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java index 169847bdb..96838318a 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java @@ -19,6 +19,7 @@ package io.nosqlbench.engine.api.templating; import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpData; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.api.config.standard.ConfigModel; import io.nosqlbench.api.config.standard.NBConfiguration; @@ -68,7 +69,7 @@ public class ParsedOpTest { params: ps1: "param-one" """; - OpsDocList stmtsDocs = OpsLoader.loadString(opt, cfg.getMap()); + OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null); assertThat(stmtsDocs.getStmts().size()).isEqualTo(1); OpTemplate opTemplate = stmtsDocs.getStmts().get(0); ParsedOp parsedOp = new ParsedOp(opTemplate, cfg); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java index dbe3fee1f..5d898ae0e 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java @@ -39,6 +39,7 @@ import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters; import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec; import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; @@ -667,7 +668,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs Optional op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload"); if (stmt.isPresent()) { workloadSource = "commandline:" + stmt.get(); - return OpsLoader.loadStmt(stmt.get(), activityDef.getParams()); + return OpsLoader.loadString(stmt.get(), OpTemplateFormat.inline, activityDef.getParams(), null); } else if (op_yaml_loc.isPresent()) { workloadSource = "yaml:" + op_yaml_loc.get(); return OpsLoader.loadPath(op_yaml_loc.get(), activityDef.getParams(), "activities"); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java index 6f5c41790..f21c8a468 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java @@ -54,11 +54,12 @@ public class StandardActivity extends SimpleActivity implements public StandardActivity(ActivityDef activityDef) { super(activityDef); + OpsDocList workload; Optional yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload"); if (yaml_loc.isPresent()) { Map disposable = new LinkedHashMap<>(activityDef.getParams()); - OpsDocList workload = OpsLoader.loadPath(yaml_loc.get(), disposable, "activities"); + workload = OpsLoader.loadPath(yaml_loc.get(), disposable, "activities"); yamlmodel = workload.getConfigModel(); } else { yamlmodel = ConfigModel.of(StandardActivity.class).asReadOnly(); diff --git a/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java b/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java index 4a29e1581..1644a0d37 100644 --- a/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java +++ b/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java @@ -20,6 +20,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -37,7 +38,7 @@ public class CommandTemplateTest { OpsDocList stmtsDocs = OpsLoader.loadString("" + "statements:\n" + " - s1: test1=foo test2=bar", - Map.of()); + OpTemplateFormat.yaml, Map.of(), null); OpTemplate optpl = stmtsDocs.getStmts().get(0); CommandTemplate ct = new CommandTemplate(optpl); assertThat(ct.isStatic()).isTrue(); @@ -51,7 +52,7 @@ public class CommandTemplateTest { " - s1: test1=foo test2={bar}\n" + " bindings:\n" + " bar: NumberNameToString();\n", - Map.of() + OpTemplateFormat.yaml, Map.of(), null ); OpTemplate optpl = stmtsDocs.getStmts().get(0); CommandTemplate ct = new CommandTemplate(optpl); diff --git a/nbr-examples/pom.xml b/nbr-examples/pom.xml index 2691012d9..44028da45 100644 --- a/nbr-examples/pom.xml +++ b/nbr-examples/pom.xml @@ -57,6 +57,14 @@ src/test/resources false + + src/test/resources + true + + **/log4j2-test.xml + **/log4j2.xml + +