diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java index efae010c3..d933991d4 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,13 @@ package io.nosqlbench.cqlgen.core; -import io.nosqlbench.cqlgen.model.CqlColumnBase; -import io.nosqlbench.cqlgen.binders.Binding; -import io.nosqlbench.cqlgen.api.BindingsLibrary; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; import io.nosqlbench.api.content.Content; import io.nosqlbench.api.content.NBIO; +import io.nosqlbench.cqlgen.api.BindingsLibrary; +import io.nosqlbench.cqlgen.binders.Binding; +import io.nosqlbench.cqlgen.model.CqlColumnBase; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import java.io.File; import java.io.InputStream; @@ -36,13 +36,16 @@ public class CGDefaultCqlBindings implements BindingsLibrary { public final static String DEFAULT_BINDINGS_FILE = "bindings-cqlgen.yaml"; public CGDefaultCqlBindings() { - String yamlContent = NBIO.all() + Content content = NBIO.all() .name(DEFAULT_BINDINGS_FILE) .first() - .map(Content::asString) - .or(() -> loadLocal(DEFAULT_BINDINGS_FILE)) - .orElseThrow(() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE + ", from local dir or internally as cqlgen/" + DEFAULT_BINDINGS_FILE)); - StmtsDocList stmtsDocs = StatementsLoader.loadString(yamlContent, Map.of()); + .or(() -> NBIO.all().prefix(DEFAULT_CFG_DIR).name(DEFAULT_BINDINGS_FILE).first()) + .orElseThrow( + () -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE + + ", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE) + ); + + OpsDocList stmtsDocs = OpsLoader.loadContent(content, Map.of()); this.bindings = stmtsDocs.getDocBindings(); } diff --git a/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java b/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java index 037b6fb05..55065bdb0 100644 --- a/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java +++ b/adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,10 @@ package io.nosqlbench.adapter.diag; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.activityimpl.OpMapper; import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; @@ -101,7 +102,7 @@ public class DiagDriverAdapter extends BaseDriverAdapter impl } @Override - public List getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map params) { - return StatementsLoader.loadString("ops: 'log:level=INFO'", params).getStmts(); + public List getSyntheticOpTemplates(OpsDocList opsDocList, Map params) { + return OpsLoader.loadString("log:level=INFO", OpTemplateFormat.inline, params,null).getStmts(); } } diff --git a/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java b/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java index 76cebcb44..42b9036d5 100644 --- a/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java +++ b/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,9 +19,10 @@ package io.nosqlbench.adapter.http; import io.nosqlbench.adapter.http.core.HttpOpMapper; import io.nosqlbench.adapter.http.core.HttpSpace; import io.nosqlbench.api.config.standard.NBConfiguration; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache; import io.nosqlbench.engine.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; @@ -51,7 +52,7 @@ public class HttpOpMapperTest { } private static ParsedOp parsedOpFor(String yaml) { - StmtsDocList docs = StatementsLoader.loadString(yaml, Map.of()); + OpsDocList docs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null); OpTemplate stmtDef = docs.getStmts().get(0); ParsedOp parsedOp = new ParsedOp(stmtDef, cfg, List.of(adapter.getPreprocessor())); return parsedOp; diff --git a/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java b/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java index f7fa16f5f..1bf1156d5 100644 --- a/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java +++ b/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java @@ -18,7 +18,7 @@ package io.nosqlbench.adapter.stdout; import io.nosqlbench.engine.api.activityconfig.yaml.OpData; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.activityimpl.OpMapper; import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; @@ -59,8 +59,8 @@ public class StdoutDriverAdapter extends BaseDriverAdapter getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map cfg) { - Set activeBindingNames = new LinkedHashSet<>(stmtsDocList.getDocBindings().keySet()); + public List getSyntheticOpTemplates(OpsDocList opsDocList, Map cfg) { + Set activeBindingNames = new LinkedHashSet<>(opsDocList.getDocBindings().keySet()); if (activeBindingNames.size()==0) { logger.warn("Unable to synthesize op for driver=" + this.getAdapterName() + " with zero bindings."); @@ -88,7 +88,7 @@ public class StdoutDriverAdapter extends BaseDriverAdapter getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map cfg) { - return adap.getSyntheticOpTemplates(stmtsDocList, cfg); + public List getSyntheticOpTemplates(OpsDocList opsDocList, Map cfg) { + return adap.getSyntheticOpTemplates(opsDocList, cfg); } } diff --git a/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapter.java b/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapter.java index 941088854..832b3eb45 100644 --- a/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapter.java +++ b/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ import io.nosqlbench.api.config.standard.ConfigModel; import io.nosqlbench.api.config.standard.NBConfigModel; import io.nosqlbench.api.config.standard.NBConfiguration; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.activityimpl.OpMapper; import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; @@ -57,7 +57,7 @@ public class TcpServerDriverAdapter extends BaseDriverAdapter getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map cfg) { - return adap.getSyntheticOpTemplates(stmtsDocList, cfg); + public List getSyntheticOpTemplates(OpsDocList opsDocList, Map cfg) { + return adap.getSyntheticOpTemplates(opsDocList, cfg); } } diff --git a/adapters-api/pom.xml b/adapters-api/pom.xml index 9e5902988..c4f5f21a8 100644 --- a/adapters-api/pom.xml +++ b/adapters-api/pom.xml @@ -54,6 +54,17 @@ ${revision} + + com.databricks + sjsonnet_2.13 + 0.4.3 + + + + org.scala-lang + scala-library + 2.13.10 + diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java new file mode 100644 index 000000000..cb9732d14 --- /dev/null +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.api.activityconfig; + +import com.amazonaws.util.StringInputStream; +import io.nosqlbench.api.content.Content; +import io.nosqlbench.api.content.NBIO; +import io.nosqlbench.api.errors.BasicError; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; +import io.nosqlbench.engine.api.templating.StrInterpolator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import scala.Option; +import sjsonnet.DefaultParseCache; +import sjsonnet.SjsonnetMain; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +public class OpsLoader { + + private final static Logger logger = LogManager.getLogger(OpsLoader.class); + + public static String[] YAML_EXTENSIONS = new String[]{"yaml", "yml"}; + + public static OpsDocList loadContent(Content content, Map params) { + OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(content.getURI()); + return loadString(content.get().toString(), fmt, params, content.getURI()); + } + + public static OpsDocList loadPath(String path, Map params, String... searchPaths) { + String[] extensions = path.indexOf('.')>-1 ? new String[]{} : YAML_EXTENSIONS; + + Content foundPath = NBIO.all().prefix(searchPaths).name(path).extension(extensions).first() + .orElseThrow(() -> new RuntimeException("Unable to load path '" + path + "'")); + OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(foundPath.getURI()); + return loadString(foundPath.asString(), fmt, params, foundPath.getURI()); + } + + public static OpsDocList loadString(final String sourceData, OpTemplateFormat fmt, Map params, URI srcuri) { + + logger.trace(() -> "Applying string transformer to data:" + sourceData); + StrInterpolator transformer = new StrInterpolator(params); + String data = transformer.apply(sourceData); + if (srcuri!=null) { + logger.info("workload URI: '" + srcuri + "'"); + } + + RawOpsLoader loader = new RawOpsLoader(transformer); + RawOpsDocList rawOpsDocList = switch (fmt) { + case jsonnet -> loader.loadString(evaluateJsonnet(srcuri, params)); + case yaml, json -> loader.loadString(data); + case inline, stmt -> RawOpsDocList.forSingleStatement(data); + }; + // TODO: itemize inline to support ParamParser + + OpsDocList layered = new OpsDocList(rawOpsDocList); + + transformer.checkpointAccesses().forEach((k, v) -> { + layered.addTemplateVariable(k, v); + if (params.containsKey(k)) { + params.remove(k); + } + }); + + return layered; + } + + private static String evaluateJsonnet(URI uri, Map params) { + List injected = new LinkedList<>(List.of(Path.of(uri).toString())); + params.forEach((k,v) -> { + if (v instanceof CharSequence cs) { + injected.addAll(List.of("--ext-str",k+"="+cs)); + } + }); + + var stdoutBuffer = new ByteArrayOutputStream(); + var stderrBuffer = new ByteArrayOutputStream(); + var stdoutStream = new PrintStream(stdoutBuffer); + var stderrStream = new PrintStream(stderrBuffer); + StringInputStream inputStream; + try { + inputStream = new StringInputStream(""); + } catch (Exception e) { + throw new RuntimeException("Error building input stream for jsonnet:" + e, e); + } + + int resultStatus = SjsonnetMain.main0( + injected.toArray(new String[0]), + new DefaultParseCache(), + inputStream, + stdoutStream, + stderrStream, + new os.Path(Path.of(System.getProperty("user.dir"))), + Option.empty(), + Option.empty() + ); + + String stdoutOutput = stdoutBuffer.toString(StandardCharsets.UTF_8); + String stderrOutput = stderrBuffer.toString(StandardCharsets.UTF_8); + if ("jsonnet".equals(String.valueOf(params.get("dryrun")))) { + logger.info("dryrun=jsonnet, dumping result to stdout and stderr:"); + System.out.println(stdoutOutput); + System.err.println(stderrOutput); + if (resultStatus==0 && stderrOutput.isEmpty()) { + logger.info("no errors detected during jsonnet evaluation."); + System.exit(0); + } else { + logger.error("ERRORS detected during jsonnet evaluation:\n" + stderrOutput); + System.exit(2); + } + } + if (!stderrOutput.isEmpty()) { + BasicError error = new BasicError("stderr output from jsonnet preprocessing: " + stderrOutput); + if (resultStatus!=0) { + throw error; + } else { + logger.warn(error.toString(),error); + } + } + logger.info("jsonnet processing read '" + uri +"', rendered " + stdoutOutput.split("\n").length + " lines."); + logger.trace("jsonnet result:\n" + stdoutOutput); + + return stdoutOutput; + } + +} diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmtOp.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedOpTemplate.java similarity index 96% rename from adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmtOp.java rename to adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedOpTemplate.java index 919fda214..302f7466d 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmtOp.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedOpTemplate.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; -public class ParsedStmtOp { +public class ParsedOpTemplate { private final OpTemplate optpl; private final ParsedStringTemplate parsed; @@ -37,13 +37,13 @@ public class ParsedStmtOp { * * @param optpl An existing statement def as read from the YAML API. */ - public ParsedStmtOp(OpTemplate optpl) { + public ParsedOpTemplate(OpTemplate optpl) { this.optpl = optpl; String transformed = getStmt(); parsed = new ParsedStringTemplate(transformed, optpl.getBindings()); } - public ParsedStmtOp orError() { + public ParsedOpTemplate orError() { if (hasError()) { throw new RuntimeException("Unable to parse statement: " + this); } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java deleted file mode 100644 index 47bf6735c..000000000 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.engine.api.activityconfig; - -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; -import io.nosqlbench.engine.api.templating.StrInterpolator; -import io.nosqlbench.api.content.Content; -import io.nosqlbench.api.content.NBIO; -import io.nosqlbench.api.errors.BasicError; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.Map; -import java.util.Optional; - -public class StatementsLoader { - - public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"}; - - private final static Logger logger = LogManager.getLogger(StatementsLoader.class); - - public static StmtsDocList loadString(String yamlContent, Map params) { - - StrInterpolator transformer = new StrInterpolator(params); - RawStmtsLoader loader = new RawStmtsLoader(transformer); - RawStmtsDocList rawDocList = loader.loadString(logger, yamlContent); - StmtsDocList layered = new StmtsDocList(rawDocList); - transformer.checkpointAccesses().forEach((k,v) -> { - layered.addTemplateVariable(k,v); - if (params.containsKey(k)) { - params.remove(k); - } - }); - return layered; - } - - public static StmtsDocList loadStmt( - Logger logger, - String statement, - Map params - ) { - StrInterpolator transformer = new StrInterpolator(params); - statement = transformer.apply(statement); - RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(statement); - StmtsDocList layered = new StmtsDocList(rawStmtsDocList); - transformer.checkpointAccesses().forEach((k,v) -> { - layered.addTemplateVariable(k,v); - if (params.containsKey(k)) { - params.remove(k); - } - }); - return layered; - } - - public static StmtsDocList loadContent( - Logger logger, - Content content, - Map params - ) { - return loadString(content.get().toString(),params); - } - - public static StmtsDocList loadPath( - Logger logger, - String path, - Map params, - String... searchPaths) { - - RawStmtsDocList list = null; - Optional> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first(); - String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path)); - return loadString(content,params); - } - - - public static StmtsDocList loadPath( - Logger logger, - String path, - String... searchPaths) { - return loadPath(logger, path, Map.of(), searchPaths); - } - -} diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StatementsOwner.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpsOwner.java similarity index 87% rename from adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StatementsOwner.java rename to adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpsOwner.java index 6279ae120..db817449f 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StatementsOwner.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpsOwner.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,21 +20,21 @@ import io.nosqlbench.api.errors.BasicError; import java.util.*; -public class StatementsOwner extends RawStmtFields { +public class OpsOwner extends RawOpFields { private final static List stmtsFieldNames = List.of("op","ops","operation","statement","statements"); - private List rawStmtDefs = new ArrayList<>(); + private List rawOpDefs = new ArrayList<>(); - public StatementsOwner() { + public OpsOwner() { } - public List getRawStmtDefs() { - return rawStmtDefs; + public List getRawStmtDefs() { + return rawOpDefs; } - public void setRawStmtDefs(List rawStmtDefs) { - this.rawStmtDefs = rawStmtDefs; + public void setRawStmtDefs(List rawOpDefs) { + this.rawOpDefs = rawOpDefs; } public void setFieldsByReflection(Map propsmap) { @@ -63,14 +63,14 @@ public class StatementsOwner extends RawStmtFields { public void setStatementsFieldByType(Object object) { if (object instanceof List) { List stmtList = (List) object; - List defs = new ArrayList<>(stmtList.size()); + List defs = new ArrayList<>(stmtList.size()); for (int i = 0; i < stmtList.size(); i++) { String defaultName = "stmt" + (i + 1); Object o = stmtList.get(i); if (o instanceof String) { - defs.add(new RawStmtDef(defaultName, (String) o)); + defs.add(new RawOpDef(defaultName, (String) o)); } else if (o instanceof Map) { - RawStmtDef def = new RawStmtDef(defaultName, (Map) o); + RawOpDef def = new RawOpDef(defaultName, (Map) o); defs.add(def); } else { throw new RuntimeException("Can not construct stmt def from object type:" + o.getClass()); diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDef.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpDef.java similarity index 95% rename from adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDef.java rename to adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpDef.java index 456e2821d..07fa007f2 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDef.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpDef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,22 +24,22 @@ import java.util.*; /** * See specification for what this should do in UniformWorkloadSpecificationTest */ -public class RawStmtDef extends RawStmtFields { +public class RawOpDef extends RawOpFields { private Object op; private final static List opFieldSynonyms = List.of("stmt", "statement", "op", "operation"); - public RawStmtDef() { + public RawOpDef() { } - public RawStmtDef(String name, String op) { + public RawOpDef(String name, String op) { setName(name); this.op = op; } @SuppressWarnings("unchecked") - public RawStmtDef(String defaultName, Map map) { + public RawOpDef(String defaultName, Map map) { setFieldsByReflection(map); if (this.getName() == null || this.getName().isEmpty()) { this.setName(defaultName); diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtFields.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpFields.java similarity index 96% rename from adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtFields.java rename to adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpFields.java index b5e6c752f..3491dce09 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtFields.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpFields.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,15 +25,15 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; -public class RawStmtFields extends Tags { - private final static Logger logger = LogManager.getLogger(RawStmtFields.class); +public class RawOpFields extends Tags { + private final static Logger logger = LogManager.getLogger(RawOpFields.class); private String name = ""; private String desc = ""; private final Map bindings = new LinkedHashMap<>(); private final Map params = new LinkedHashMap<>(); - public RawStmtFields() { + public RawOpFields() { } public String getDesc() { @@ -80,7 +80,7 @@ public class RawStmtFields extends Tags { this.params.putAll(config); } - public void applyBlockParams(RawStmtFields other) { + public void applyBlockParams(RawOpFields other) { setName(other.getName()); setBindings(other.getBindings()); setTags(other.getTags()); diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsBlock.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsBlock.java similarity index 82% rename from adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsBlock.java rename to adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsBlock.java index c61d72bdb..5384202b7 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsBlock.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,12 +20,12 @@ import java.util.Map; /** * A StmtsDef contains a list of rawStmts, as well as all of the optional - * block parameters that can be assigned to {@link RawStmtFields}, which includes + * block parameters that can be assigned to {@link RawOpFields}, which includes * a name, config values, data bindings, and filtering tags. */ -public class RawStmtsBlock extends StatementsOwner { +public class RawOpsBlock extends OpsOwner { - public RawStmtsBlock() {} + public RawOpsBlock() {} public void setFieldsByReflection(Map propsmap) { super.setFieldsByReflection(propsmap); diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDoc.java similarity index 77% rename from adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java rename to adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDoc.java index 1fc89d63b..f0c235a31 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDoc.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,26 +25,26 @@ import java.util.Map; /** * A statements doc can have both a list of statement blocks and/or a * list of statements. It can also have all the block parameters - * assignable to {@link RawStmtFields}. + * assignable to {@link RawOpFields}. *

* The reason for having support both statements or statement blocks * is merely convenience. If you do not need or want to deal with the * full blocks format, the extra structure gets in the way. */ -public class RawStmtsDoc extends StatementsOwner { +public class RawOpsDoc extends OpsOwner { private RawScenarios scenarios = new RawScenarios(); - private final List blocks = new ArrayList<>(); + private final List blocks = new ArrayList<>(); private String versionRegex = ".+"; // no-args ctor is required - public RawStmtsDoc() { + public RawOpsDoc() { } - public static RawStmtsDoc forSingleStatement(String statement) { - RawStmtsDoc rawStmtsDoc = new RawStmtsDoc(); - rawStmtsDoc.setStatementsFieldByType(statement); - return rawStmtsDoc; + public static RawOpsDoc forSingleStatement(String statement) { + RawOpsDoc rawOpsDoc = new RawOpsDoc(); + rawOpsDoc.setStatementsFieldByType(statement); + return rawOpsDoc; } public void setFieldsByReflection(Map properties) { @@ -63,9 +63,9 @@ public class RawStmtsDoc extends StatementsOwner { for (Object blockData : blockList) { if (blockData instanceof Map) { Map blockDataMap = (Map) blockData; - RawStmtsBlock rawStmtsBlock = new RawStmtsBlock(); - rawStmtsBlock.setFieldsByReflection(blockDataMap); - blocks.add(rawStmtsBlock); + RawOpsBlock rawOpsBlock = new RawOpsBlock(); + rawOpsBlock.setFieldsByReflection(blockDataMap); + blocks.add(rawOpsBlock); } else { throw new RuntimeException("Invalid object type for block data: " + blockData.getClass().getCanonicalName()); } @@ -77,10 +77,10 @@ public class RawStmtsDoc extends StatementsOwner { Object blockData = entry.getValue(); if (blockData instanceof Map) { Map blockDataMap = (Map) blockData; - RawStmtsBlock rawStmtsBlock = new RawStmtsBlock(); - rawStmtsBlock.setName(blockName); - rawStmtsBlock.setFieldsByReflection(blockDataMap); - blocks.add(rawStmtsBlock); + RawOpsBlock rawOpsBlock = new RawOpsBlock(); + rawOpsBlock.setName(blockName); + rawOpsBlock.setFieldsByReflection(blockDataMap); + blocks.add(rawOpsBlock); } else { throw new RuntimeException("Invalid object type for block data: " + blockData.getClass().getCanonicalName()); } @@ -107,19 +107,19 @@ public class RawStmtsDoc extends StatementsOwner { * * @return all logical statement blocks containing statements */ - public List getBlocks() { - List stmtBlocks = new ArrayList<>(); + public List getBlocks() { + List stmtBlocks = new ArrayList<>(); if (!getRawStmtDefs().isEmpty()) { - RawStmtsBlock rawStmtsBlock = new RawStmtsBlock(); - rawStmtsBlock.setName("block0"); - rawStmtsBlock.setRawStmtDefs(getRawStmtDefs()); - stmtBlocks.add(rawStmtsBlock); + RawOpsBlock rawOpsBlock = new RawOpsBlock(); + rawOpsBlock.setName("block0"); + rawOpsBlock.setRawStmtDefs(getRawStmtDefs()); + stmtBlocks.add(rawOpsBlock); } stmtBlocks.addAll(this.blocks); return stmtBlocks; } - public void setBlocks(List blocks) { + public void setBlocks(List blocks) { this.blocks.clear(); this.blocks.addAll(blocks); } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDocList.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDocList.java new file mode 100644 index 000000000..0446f23c3 --- /dev/null +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDocList.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.api.activityconfig.rawyaml; + +import java.util.List; + +public class RawOpsDocList { + + private final List rawOpsDocList; + + public RawOpsDocList(List rawOpsDocList) { + this.rawOpsDocList = rawOpsDocList; + } + + public static RawOpsDocList forSingleStatement(String statement) { + RawOpsDoc rawOpsDoc = RawOpsDoc.forSingleStatement(statement); + return new RawOpsDocList(List.of(rawOpsDoc)); + } + + public static RawOpsDocList none() { + return new RawOpsDocList(List.of()); + } + + public List getStmtsDocs() { + return rawOpsDocList; + } + + public String toString() { + int docs = rawOpsDocList.size(); + int blocks = rawOpsDocList.stream().map(RawOpsDoc::getBlocks).mapToInt(List::size).sum(); + long optemplates = rawOpsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawStmtDefs().stream()).count(); + return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates; + } +} diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsLoader.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java similarity index 69% rename from adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsLoader.java rename to adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java index d37b982ca..75e722b21 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsLoader.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java @@ -21,6 +21,7 @@ import io.nosqlbench.api.content.NBIO; import io.nosqlbench.api.errors.BasicError; import io.nosqlbench.api.errors.OpConfigError; import io.nosqlbench.engine.api.templating.StrInterpolator; +import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.snakeyaml.engine.v2.api.Load; import org.snakeyaml.engine.v2.api.LoadSettings; @@ -28,18 +29,18 @@ import org.snakeyaml.engine.v2.api.LoadSettings; import java.util.*; import java.util.function.Function; -public class RawStmtsLoader { +public class RawOpsLoader { + private final static Logger logger = LogManager.getLogger(RawOpsLoader.class); public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"}; - List> stringTransformers = new ArrayList<>(); private final ArrayList> transformers = new ArrayList<>(); - public RawStmtsLoader(Function transformer) { + public RawOpsLoader(Function transformer) { addTransformer(transformer); } - public RawStmtsLoader() { + public RawOpsLoader() { addTransformer(new StrInterpolator()); } @@ -47,11 +48,10 @@ public class RawStmtsLoader { Collections.addAll(this.transformers, newTransformer); } - public RawStmtsDocList loadString(Logger logger, final String originalData) { + public RawOpsDocList loadString(final String originalData) { + logger.trace(() -> "Applying string transformer to yaml data:" + originalData); String data = originalData; - try { - if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + originalData); for (Function transformer : transformers) { data = transformer.apply(data); } @@ -60,11 +60,10 @@ public class RawStmtsLoader { throw t; } - return parseYaml(logger, data); + return parseYaml(data); } - public RawStmtsDocList loadPath( - Logger logger, + public RawOpsDocList loadPath( String path, String... searchPaths) { @@ -72,22 +71,22 @@ public class RawStmtsLoader { try { Optional> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first(); data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path)); - return loadString(logger, data); + return loadString(data); } catch (Exception e) { throw new RuntimeException("error while reading file " + path, e); } } - private RawStmtsDocList parseYaml(Logger logger, String data) { + public RawOpsDocList parseYaml(String data) { LoadSettings loadSettings = LoadSettings.builder().build(); Load yaml = new Load(loadSettings); Iterable objects = yaml.loadAllFromString(data); - List newDocList = new ArrayList<>(); + List newDocList = new ArrayList<>(); for (Object object : objects) { if (object instanceof Map) { - RawStmtsDoc doc = new RawStmtsDoc(); + RawOpsDoc doc = new RawOpsDoc(); Map docfields = (Map) object; doc.setFieldsByReflection(docfields); if (docfields.size()>0) { @@ -99,21 +98,8 @@ public class RawStmtsLoader { throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName()); } } - RawStmtsDocList rawStmtsDocList = new RawStmtsDocList(newDocList); - return rawStmtsDocList; - } - - protected String applyTransforms(Logger logger, String data) { - for (Function xform : stringTransformers) { - try { - if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + xform); - data = xform.apply(data); - } catch (Exception e) { - RuntimeException t = new OpConfigError("Error applying string transforms to input", e); - throw t; - } - } - return data; + RawOpsDocList rawOpsDocList = new RawOpsDocList(newDocList); + return rawOpsDocList; } } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java deleted file mode 100644 index a87ae402f..000000000 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2022-2023 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.engine.api.activityconfig.rawyaml; - -import java.util.List; - -public class RawStmtsDocList { - - private final List rawStmtsDocList; - - public RawStmtsDocList(List rawStmtsDocList) { - this.rawStmtsDocList = rawStmtsDocList; - } - - public static RawStmtsDocList forSingleStatement(String statement) { - RawStmtsDoc rawStmtsDoc = RawStmtsDoc.forSingleStatement(statement); - return new RawStmtsDocList(List.of(rawStmtsDoc)); - } - - public static RawStmtsDocList none() { - return new RawStmtsDocList(List.of()); - } - - public List getStmtsDocs() { - return rawStmtsDocList; - } - - public String toString() { - int docs = rawStmtsDocList.size(); - int blocks = rawStmtsDocList.stream().map(RawStmtsDoc::getBlocks).mapToInt(List::size).sum(); - long optemplates = rawStmtsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawStmtDefs().stream()).count(); - return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates; - } -} diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java index dac136806..5358c6171 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java @@ -76,7 +76,7 @@ public class RawYamlLoader { private List> parseYaml(Logger logger, String data) { Load yaml = new Load(LoadSettings.builder().build()); Iterable objects = yaml.loadAllFromString(data); - List newDocList = new ArrayList<>(); + List newDocList = new ArrayList<>(); List> maps = new ArrayList<>(); diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java index 2109e9fad..59e445460 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package io.nosqlbench.engine.api.activityconfig.yaml; import io.nosqlbench.engine.api.activityconfig.MultiMapLookup; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpDef; import io.nosqlbench.api.errors.BasicError; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -27,15 +27,15 @@ import java.util.*; public class OpDef extends OpTemplate { private final static Logger logger = LogManager.getLogger(OpDef.class); - private final RawStmtDef rawStmtDef; - private final StmtsBlock block; + private final RawOpDef rawOpDef; + private final OpsBlock block; private final LinkedHashMap params; private final LinkedHashMap bindings; private final LinkedHashMap tags; - public OpDef(StmtsBlock block, RawStmtDef rawStmtDef) { + public OpDef(OpsBlock block, RawOpDef rawOpDef) { this.block = block; - this.rawStmtDef = rawStmtDef; + this.rawOpDef = rawOpDef; this.params = composeParams(); this.bindings = composeBindings(); this.tags = composeTags(); @@ -43,12 +43,12 @@ public class OpDef extends OpTemplate { @Override public String getName() { - return block.getName() + "--" + rawStmtDef.getName(); + return block.getName() + "--" + rawOpDef.getName(); } @Override public Optional> getOp() { - Object op = rawStmtDef.getOp(); + Object op = rawOpDef.getOp(); if (op == null) { return Optional.empty(); } @@ -75,7 +75,7 @@ public class OpDef extends OpTemplate { } private LinkedHashMap composeBindings() { - MultiMapLookup lookup = new MultiMapLookup<>(rawStmtDef.getBindings(), block.getBindings()); + MultiMapLookup lookup = new MultiMapLookup<>(rawOpDef.getBindings(), block.getBindings()); return new LinkedHashMap<>(lookup); } @@ -85,7 +85,7 @@ public class OpDef extends OpTemplate { } private LinkedHashMap composeParams() { - MultiMapLookup lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams()); + MultiMapLookup lookup = new MultiMapLookup<>(rawOpDef.getParams(), block.getParams()); LinkedHashMap params = new LinkedHashMap<>(lookup); return params; } @@ -98,7 +98,7 @@ public class OpDef extends OpTemplate { } private LinkedHashMap composeTags() { - LinkedHashMap tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawStmtDef.getTags(), block.getTags())); + LinkedHashMap tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawOpDef.getTags(), block.getTags())); tagsWithName.put("name",getName()); tagsWithName.put("block",block.getName()); return tagsWithName; @@ -111,7 +111,7 @@ public class OpDef extends OpTemplate { @Override public String getDesc() { - return rawStmtDef.getDesc(); + return rawOpDef.getDesc(); } } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplateFormat.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplateFormat.java new file mode 100644 index 000000000..fc3e5e098 --- /dev/null +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplateFormat.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.api.activityconfig.yaml; + +import java.net.URI; +import java.nio.file.Path; + +/** + * The formats which are recognized as source data for the workload. Any serialization may be supported + * which can be converted from a character string to an {@link io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList} structure. + * + * Those which are derived from paths may be determined by their filename extension. Others, which are provided from internal + * NoSQLBench sources, may only be invoked explicitly. + */ +public enum OpTemplateFormat { + /** + * The source is in YAML format + */ + yaml("yaml", "yml"), + + /** + * The source is in JSON format + */ + json("json", "json5"), + + /** + * The source is in jsonnet format, suitable for pre-processing with context data + */ + jsonnet("jsonnet", "jsonnet5"), + + /** + * The source is inline, meaning {@link io.nosqlbench.api.config.params.ParamsParser} format. + * This is similar to the {@link #stmt} format except that it is parsed for internal op structure. + * This format is not detectable by source path, and thus can only be used when provided directly + * from the caller. + */ + inline(), + + /** + * The source is in single-statement form, meaning that it is known to be the value of the 'stmt' + * field of an op template. This format is not detectable by source path, and thus can only be used when provided directly + * from the caller. + */ + stmt(); + + private final String[] pathExtensions; + + OpTemplateFormat(String... pathExtensions) { + this.pathExtensions = pathExtensions; + } + + public static OpTemplateFormat valueOfURI(URI uri) { + var fullName = uri.toString(); + String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase(); + + for (OpTemplateFormat value : values()) { + for (String pathExtension : value.pathExtensions) { + if (pathExtension.equals(extension)) { + return value; + } + } + } + throw new RuntimeException("Unable to determine source format for " + uri); + + } + + public static OpTemplateFormat valueOfPath(Path path) { + var fullName = path.toString(); + String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase(); + + for (OpTemplateFormat value : values()) { + for (String pathExtension : value.pathExtensions) { + if (pathExtension.equals(extension)) { + return value; + } + } + } + throw new RuntimeException("Unable to determine source format for " + path); + } +} diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsBlock.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsBlock.java similarity index 68% rename from adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsBlock.java rename to adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsBlock.java index 231b51322..1f748760f 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsBlock.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,30 +17,30 @@ package io.nosqlbench.engine.api.activityconfig.yaml; import io.nosqlbench.engine.api.activityconfig.MultiMapLookup; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpDef; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsBlock; import io.nosqlbench.api.engine.util.Tagged; import org.jetbrains.annotations.NotNull; import java.util.*; -public class StmtsBlock implements Tagged, Iterable { +public class OpsBlock implements Tagged, Iterable { - private final RawStmtsBlock rawStmtsBlock; - private final StmtsDoc rawStmtsDoc; + private final RawOpsBlock rawOpsBlock; + private final OpsDoc rawOpsDoc; private final int blockIdx; - public StmtsBlock(RawStmtsBlock rawStmtsBlock, StmtsDoc rawStmtsDoc, int blockIdx) { - this.rawStmtsBlock = rawStmtsBlock; - this.rawStmtsDoc = rawStmtsDoc; + public OpsBlock(RawOpsBlock rawOpsBlock, OpsDoc rawOpsDoc, int blockIdx) { + this.rawOpsBlock = rawOpsBlock; + this.rawOpsDoc = rawOpsDoc; this.blockIdx = blockIdx; } public List getOps() { List rawOpTemplates = new ArrayList<>(); - List statements = rawStmtsBlock.getRawStmtDefs(); + List statements = rawOpsBlock.getRawStmtDefs(); for (int i = 0; i < statements.size(); i++) { rawOpTemplates.add( @@ -52,11 +52,11 @@ public class StmtsBlock implements Tagged, Iterable { public String getName() { StringBuilder sb = new StringBuilder(); - if (!rawStmtsDoc.getName().isEmpty()) { - sb.append(rawStmtsDoc.getName()).append("--"); + if (!rawOpsDoc.getName().isEmpty()) { + sb.append(rawOpsDoc.getName()).append("--"); } - if (!rawStmtsBlock.getName().isEmpty()) { - sb.append(rawStmtsBlock.getName()); + if (!rawOpsBlock.getName().isEmpty()) { + sb.append(rawOpsBlock.getName()); } else { sb.append("block").append(blockIdx); } @@ -64,15 +64,15 @@ public class StmtsBlock implements Tagged, Iterable { } public Map getTags() { - return new MultiMapLookup<>(rawStmtsBlock.getTags(), rawStmtsDoc.getTags()); + return new MultiMapLookup<>(rawOpsBlock.getTags(), rawOpsDoc.getTags()); } public Map getParams() { - return new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams()); + return new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams()); } public Map getParamsAsText() { - MultiMapLookup lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams()); + MultiMapLookup lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams()); LinkedHashMap stringmap = new LinkedHashMap<>(); lookup.forEach((k, v) -> stringmap.put(k, v.toString())); return stringmap; @@ -81,7 +81,7 @@ public class StmtsBlock implements Tagged, Iterable { @SuppressWarnings("unchecked") public V getParamOrDefault(String name, V defaultValue) { Objects.requireNonNull(defaultValue); - MultiMapLookup lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams()); + MultiMapLookup lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams()); if (!lookup.containsKey(name)) { return defaultValue; } @@ -90,14 +90,14 @@ public class StmtsBlock implements Tagged, Iterable { } public V getParam(String name, Class type) { - MultiMapLookup lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams()); + MultiMapLookup lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams()); Object object = lookup.get(name); V value = type.cast(object); return value; } public Map getBindings() { - return new MultiMapLookup<>(rawStmtsBlock.getBindings(), rawStmtsDoc.getBindings()); + return new MultiMapLookup<>(rawOpsBlock.getBindings(), rawOpsDoc.getBindings()); } @Override diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDoc.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDoc.java similarity index 67% rename from adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDoc.java rename to adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDoc.java index 2450d5b0e..a09e735a5 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDoc.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDoc.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,8 @@ package io.nosqlbench.engine.api.activityconfig.yaml; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsBlock; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDoc; import io.nosqlbench.api.engine.util.Tagged; import java.util.ArrayList; @@ -30,26 +30,26 @@ import java.util.stream.Collectors; * StmtsDoc creates a logical view of a statements doc that includes * all inherited and overridden values for bindings, tags, and params. */ -public class StmtsDoc implements Tagged, Iterable { +public class OpsDoc implements Tagged, Iterable { - private final RawStmtsDoc rawStmtsDoc; + private final RawOpsDoc rawOpsDoc; - public StmtsDoc(RawStmtsDoc rawStmtsDoc) { - this.rawStmtsDoc = rawStmtsDoc; + public OpsDoc(RawOpsDoc rawOpsDoc) { + this.rawOpsDoc = rawOpsDoc; } /** * @return a usable list of blocks, including inherited bindings, params, and tags * from the parent doc */ - public List getBlocks() { - List blocks = new ArrayList<>(); + public List getBlocks() { + List blocks = new ArrayList<>(); int blockIdx = 0; - for (RawStmtsBlock rawStmtsBlock : rawStmtsDoc.getBlocks()) { - String compositeName = rawStmtsDoc.getName() + - (rawStmtsBlock.getName().isEmpty() ? "" : "-" + rawStmtsBlock.getName()); - StmtsBlock compositeBlock = new StmtsBlock(rawStmtsBlock, this, ++blockIdx); + for (RawOpsBlock rawOpsBlock : rawOpsDoc.getBlocks()) { + String compositeName = rawOpsDoc.getName() + + (rawOpsBlock.getName().isEmpty() ? "" : "-" + rawOpsBlock.getName()); + OpsBlock compositeBlock = new OpsBlock(rawOpsBlock, this, ++blockIdx); blocks.add(compositeBlock); } @@ -61,28 +61,28 @@ public class StmtsDoc implements Tagged, Iterable { */ @Override public Map getTags() { - return rawStmtsDoc.getTags(); + return rawOpsDoc.getTags(); } /** * @return a usable map of parameters, including those inherited from the parent doc */ public Map getParams() { - return rawStmtsDoc.getParams(); + return rawOpsDoc.getParams(); } /** * @return a usable map of bindings, including those inherited from the parent doc */ public Map getBindings() { - return rawStmtsDoc.getBindings(); + return rawOpsDoc.getBindings(); } /** * @return the name of this block */ public String getName() { - return rawStmtsDoc.getName(); + return rawOpsDoc.getName(); } /** @@ -95,19 +95,19 @@ public class StmtsDoc implements Tagged, Iterable { /** * Allow StmtsDoc to be used in iterable loops. - * @return An iterator of {@link StmtsBlock} + * @return An iterator of {@link OpsBlock} */ @Override - public Iterator iterator() { + public Iterator iterator() { return getBlocks().iterator(); } public Scenarios getScenarios() { - return new Scenarios(rawStmtsDoc.getRawScenarios()); + return new Scenarios(rawOpsDoc.getRawScenarios()); } public String getDescription() { - return rawStmtsDoc.getDesc(); + return rawOpsDoc.getDesc(); } } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java similarity index 79% rename from adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java rename to adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java index 800438219..68f79dd79 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java @@ -16,9 +16,9 @@ package io.nosqlbench.engine.api.activityconfig.yaml; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDoc; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList; import io.nosqlbench.engine.api.util.TagFilter; import io.nosqlbench.api.config.standard.ConfigModel; import io.nosqlbench.api.config.standard.NBConfigModel; @@ -31,30 +31,30 @@ import java.util.*; import java.util.regex.Pattern; import java.util.stream.Collectors; -public class StmtsDocList implements Iterable { - private final static Logger logger = LogManager.getLogger(StmtsDocList.class); +public class OpsDocList implements Iterable { + private final static Logger logger = LogManager.getLogger(OpsDocList.class); - private final RawStmtsDocList rawStmtsDocList; + private final RawOpsDocList rawOpsDocList; private final Map templateVariables = new LinkedHashMap<>(); - public StmtsDocList(RawStmtsDocList rawStmtsDocList) { - this.rawStmtsDocList = rawStmtsDocList; + public OpsDocList(RawOpsDocList rawOpsDocList) { + this.rawOpsDocList = rawOpsDocList; } - public static StmtsDocList none() { - return new StmtsDocList(RawStmtsDocList.none()); + public static OpsDocList none() { + return new OpsDocList(RawOpsDocList.none()); } - public List getStmtDocs(String tagFilter) { + public List getStmtDocs(String tagFilter) { TagFilter tf = new TagFilter(tagFilter); return getStmtDocs().stream() .filter(tf::matchesTagged) .collect(Collectors.toList()); } - public List getStmtDocs() { - return rawStmtsDocList.getStmtsDocs().stream() - .map(StmtsDoc::new) + public List getStmtDocs() { + return rawOpsDocList.getStmtsDocs().stream() + .map(OpsDoc::new) .collect(Collectors.toList()); } @@ -81,7 +81,7 @@ public class StmtsDocList implements Iterable { @Override - public Iterator iterator() { + public Iterator iterator() { return getStmtDocs().iterator(); } @@ -94,7 +94,7 @@ public class StmtsDocList implements Iterable { public Map getDocBindings() { LinkedHashMap docBindings = new LinkedHashMap<>(); getStmtDocs().stream() - .map(StmtsDoc::getBindings) + .map(OpsDoc::getBindings) .forEach(docBindings::putAll); return docBindings; } @@ -130,7 +130,7 @@ public class StmtsDocList implements Iterable { } public NBConfigModel getConfigModel() { - ConfigModel cfgmodel = ConfigModel.of(StmtsDocList.class); + ConfigModel cfgmodel = ConfigModel.of(OpsDocList.class); getTemplateVariables().forEach((k, v) -> { cfgmodel.add(Param.defaultTo(k, v, "template parameter found in the yaml workload")); }); @@ -144,9 +144,9 @@ public class StmtsDocList implements Iterable { int blockscount = 0; int opscount = 0; - for (StmtsDoc stmtDoc : this.getStmtDocs()) { + for (OpsDoc stmtDoc : this.getStmtDocs()) { docscount++; - for (StmtsBlock block : stmtDoc.getBlocks()) { + for (OpsBlock block : stmtDoc.getBlocks()) { blockscount++; for (OpTemplate op : block.getOps()) { opscount++; @@ -161,12 +161,12 @@ public class StmtsDocList implements Iterable { } public static NBConfigModelExpander TEMPLATE_VAR_EXPANDER = workload -> { - StmtsDocList loaded = StatementsLoader.loadPath(logger, (String) workload, "activities"); + OpsDocList loaded = OpsLoader.loadPath((String) workload, Map.of(), "activities"); return loaded.getConfigModel(); }; public Pattern getVersionRegex() { - List stmtDocs = rawStmtsDocList.getStmtsDocs(); + List stmtDocs = rawOpsDocList.getStmtsDocs(); return Pattern.compile(stmtDocs.size()>0 ? stmtDocs.get(0).getVersionRegex() : ".*"); } } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/decorators/SyntheticOpTemplateProvider.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/decorators/SyntheticOpTemplateProvider.java index a795db098..95f5299f6 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/decorators/SyntheticOpTemplateProvider.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/decorators/SyntheticOpTemplateProvider.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package io.nosqlbench.engine.api.activityimpl.uniform.decorators; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.templating.DriverAdapterDecorators; import java.util.List; @@ -38,8 +38,8 @@ public interface SyntheticOpTemplateProvider extends DriverAdapterDecorators { /** * If a driver adapter supports creating example op templates from bindings, * it must implement this method to do so. - * @param stmtsDocList The existing doc structure, which should contain no fully defined op templates, but may contain other elements like bindings + * @param opsDocList The existing doc structure, which should contain no fully defined op templates, but may contain other elements like bindings * @return A list of op templates, size zero or more */ - List getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map params); + List getSyntheticOpTemplates(OpsDocList opsDocList, Map params); } diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/StatementsLoaderTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java similarity index 73% rename from adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/StatementsLoaderTest.java rename to adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java index 4f203815f..53e28d5cb 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/StatementsLoaderTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,8 @@ package io.nosqlbench.engine.api.activityconfig; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDoc; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import org.junit.jupiter.api.Test; import java.util.List; @@ -25,15 +25,15 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -public class StatementsLoaderTest { +public class OpsLoaderTest { @Test public void testTemplateVarSubstitution() { - StmtsDocList stmtsDocs = StatementsLoader.loadPath(null, "activities/template_vars", "src/test/resources"); + OpsDocList stmtsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources"); assertThat(stmtsDocs).isNotNull(); - List docs = stmtsDocs.getStmtDocs(); + List docs = stmtsDocs.getStmtDocs(); assertThat(docs).hasSize(1); - StmtsDoc stmtsBlocks = docs.get(0); + OpsDoc stmtsBlocks = docs.get(0); Map bindings = stmtsBlocks.getBindings(); assertThat(bindings).isEqualTo(Map.of( "b1a","Prefix(\"prefix\")", @@ -46,7 +46,7 @@ public class StatementsLoaderTest { public void testInvalidYamlProperties() { Exception caught = null; try { - StatementsLoader.loadPath(null, "activities/invalid_prop", "src/test/resources"); + OpsLoader.loadPath("activities/invalid_prop", Map.of(),"src/test/resources"); } catch (Exception e) { caught = e; } diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java index b6414e4b3..b9bc680fc 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,11 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsBlock; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsBlock; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDoc; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Test; @@ -37,22 +37,22 @@ public class OpDefTest { @Test public void testLayering() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); + OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(2); - StmtsDoc doc1 = all.getStmtDocs().get(0); + OpsDoc doc1 = all.getStmtDocs().get(0); assertThat(doc1.getName()).isEqualTo("doc1"); assertThat(doc1.getBlocks()).hasSize(1); - StmtsDoc doc2 = all.getStmtDocs().get(1); + OpsDoc doc2 = all.getStmtDocs().get(1); assertThat(doc2.getBlocks()).hasSize(2); - StmtsBlock block1 = doc1.getBlocks().get(0); + OpsBlock block1 = doc1.getBlocks().get(0); assertThat(block1.getBindings()).hasSize(2); assertThat(block1.getName()).isEqualTo("doc1--block0"); assertThat(block1.getTags()).hasSize(1); - StmtsBlock block21 = doc2.getBlocks().get(0); - StmtsBlock block22 = doc2.getBlocks().get(1); + OpsBlock block21 = doc2.getBlocks().get(0); + OpsBlock block22 = doc2.getBlocks().get(1); assertThat(block21.getName()).isEqualTo("doc2--block1"); assertThat(block21.getTags()).hasSize(3); @@ -64,11 +64,11 @@ public class OpDefTest { @Test public void testStatementRendering() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); + OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(2); - StmtsDoc doc1 = all.getStmtDocs().get(0); - StmtsBlock block1 = doc1.getBlocks().get(0); + OpsDoc doc1 = all.getStmtDocs().get(0); + OpsBlock block1 = doc1.getBlocks().get(0); assertThat(block1.getName()).isEqualTo("doc1--block0"); List ops = block1.getOps(); assertThat(ops).hasSize(2); @@ -79,9 +79,9 @@ public class OpDefTest { @Test public void testConsumableMapState() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); - List docs = all.getStmtDocs(); - StmtsDoc block1 = docs.get(1); + OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); + List docs = all.getStmtDocs(); + OpsDoc block1 = docs.get(1); List stmts = block1.getStmts(); OpTemplate stmt0 = stmts.get(0); OpTemplate stmt1 = stmts.get(1); @@ -94,12 +94,12 @@ public class OpDefTest { @Test public void testMapOfMaps() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml"); - List docs = all.getStmtDocs(); - StmtsDoc doc0 = docs.get(0); + OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); + List docs = all.getStmtDocs(); + OpsDoc doc0 = docs.get(0); assertThat(doc0.getName()).isEqualTo("map-of-maps"); assertThat(doc0.getBlocks()).hasSize(1); - StmtsBlock block1 = doc0.getBlocks().get(0); + OpsBlock block1 = doc0.getBlocks().get(0); assertThat(block1.getName()).isEqualTo("map-of-maps--block0"); assertThat(block1.getOps()).hasSize(2); OpTemplate op0 = block1.getOps().get(0); @@ -115,12 +115,12 @@ public class OpDefTest { @Test public void testBasicStringStmt() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml"); - List docs = all.getStmtDocs(); - StmtsDoc doc1 = docs.get(1); + OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); + List docs = all.getStmtDocs(); + OpsDoc doc1 = docs.get(1); assertThat(doc1.getName()).isEqualTo("string-statement"); assertThat(doc1.getBlocks()).hasSize(1); - StmtsBlock block1 = doc1.getBlocks().get(0); + OpsBlock block1 = doc1.getBlocks().get(0); assertThat(block1.getName()).isEqualTo("string-statement--block0"); assertThat(block1.getOps()).hasSize(1); OpTemplate op0 = block1.getOps().get(0); @@ -130,12 +130,12 @@ public class OpDefTest { @Test public void testListOfNamedMap() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml"); - List docs = all.getStmtDocs(); - StmtsDoc doc2 = docs.get(2); + OpsDocList all = OpsLoader.loadPath("testdocs/statement_variants.yaml", Map.of()); + List docs = all.getStmtDocs(); + OpsDoc doc2 = docs.get(2); assertThat(doc2.getName()).isEqualTo("list-of-named-map"); assertThat(doc2.getBlocks()).hasSize(1); - StmtsBlock block1 = doc2.getBlocks().get(0); + OpsBlock block1 = doc2.getBlocks().get(0); assertThat(block1.getOps()).hasSize(1); OpTemplate op0 = block1.getOps().get(0); System.out.println(op0.getParams()); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoaderTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlTemplateLoaderTest.java similarity index 58% rename from adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoaderTest.java rename to adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlTemplateLoaderTest.java index 7b4c30998..04ae15c9b 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoaderTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlTemplateLoaderTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,15 +25,15 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -public class RawYamlStatementLoaderTest { - private final static Logger logger = LogManager.getLogger(RawYamlStatementLoaderTest.class); +public class RawYamlTemplateLoaderTest { + private final static Logger logger = LogManager.getLogger(RawYamlTemplateLoaderTest.class); @Test public void testLoadPropertiesBlock() { - RawStmtsLoader ysl = new RawStmtsLoader(); - RawStmtsDocList rawBlockDocs = ysl.loadPath(logger, "testdocs/rawblock.yaml"); + RawOpsLoader ysl = new RawOpsLoader(); + RawOpsDocList rawBlockDocs = ysl.loadPath("testdocs/rawblock.yaml"); assertThat(rawBlockDocs.getStmtsDocs()).hasSize(1); - RawStmtsDoc rawBlockDoc = rawBlockDocs.getStmtsDocs().get(0); + RawOpsDoc rawBlockDoc = rawBlockDocs.getStmtsDocs().get(0); assertThat(rawBlockDoc.getRawStmtDefs()).hasSize(1); assertThat(rawBlockDoc.getBindings()).hasSize(1); assertThat(rawBlockDoc.getName()).isEqualTo("name"); @@ -43,30 +43,30 @@ public class RawYamlStatementLoaderTest { @Test public void testLoadFullFormat() { - RawStmtsLoader ysl = new RawStmtsLoader(); - RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); - List rawStmtsDocs = erthing.getStmtsDocs(); - assertThat(rawStmtsDocs).hasSize(2); - RawStmtsDoc rawStmtsDoc = rawStmtsDocs.get(0); - List blocks = rawStmtsDoc.getBlocks(); - assertThat(rawStmtsDoc.getName()).isEqualTo("doc1"); + RawOpsLoader ysl = new RawOpsLoader(); + RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_stmts.yaml"); + List rawOpsDocs = erthing.getStmtsDocs(); + assertThat(rawOpsDocs).hasSize(2); + RawOpsDoc rawOpsDoc = rawOpsDocs.get(0); + List blocks = rawOpsDoc.getBlocks(); + assertThat(rawOpsDoc.getName()).isEqualTo("doc1"); assertThat(blocks).hasSize(1); - RawStmtsBlock rawStmtsBlock = blocks.get(0); - assertThat(rawStmtsBlock.getName()).isEqualTo("block0"); + RawOpsBlock rawOpsBlock = blocks.get(0); + assertThat(rawOpsBlock.getName()).isEqualTo("block0"); } @Test public void testLoadScenarios() { - RawStmtsLoader ysl = new RawStmtsLoader(); - RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); - List rawStmtsDocs = erthing.getStmtsDocs(); - assertThat(rawStmtsDocs).hasSize(2); - RawStmtsDoc rawStmtsDoc = rawStmtsDocs.get(0); - List blocks = rawStmtsDoc.getBlocks(); - assertThat(rawStmtsDoc.getDesc()).isEqualTo( + RawOpsLoader ysl = new RawOpsLoader(); + RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_stmts.yaml"); + List rawOpsDocs = erthing.getStmtsDocs(); + assertThat(rawOpsDocs).hasSize(2); + RawOpsDoc rawOpsDoc = rawOpsDocs.get(0); + List blocks = rawOpsDoc.getBlocks(); + assertThat(rawOpsDoc.getDesc()).isEqualTo( "a quintessential description - this is superseded by dedicated specification tests and will be removed"); - RawScenarios rawScenarios = rawStmtsDoc.getRawScenarios(); + RawScenarios rawScenarios = rawOpsDoc.getRawScenarios(); assertThat(rawScenarios.getScenarioNames()).containsExactly("default", "schema-only"); Map defaultScenario = rawScenarios.getNamedScenario("default"); assertThat(defaultScenario.keySet()) @@ -79,16 +79,16 @@ public class RawYamlStatementLoaderTest { assertThat(schemaOnlyScenario.values()) .containsExactly("run driver=blah tags=phase:schema"); - assertThat(rawStmtsDoc.getName()).isEqualTo("doc1"); + assertThat(rawOpsDoc.getName()).isEqualTo("doc1"); assertThat(blocks).hasSize(1); - RawStmtsBlock rawStmtsBlock = blocks.get(0); - assertThat(rawStmtsBlock.getName()).isEqualTo("block0"); + RawOpsBlock rawOpsBlock = blocks.get(0); + assertThat(rawOpsBlock.getName()).isEqualTo("block0"); } @Test public void testErrorMsg() { - RawStmtsLoader ysl = new RawStmtsLoader(); - RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/badyamlfile.yaml"); + RawOpsLoader ysl = new RawOpsLoader(); + RawOpsDocList erthing = ysl.loadPath("testdocs/badyamlfile.yaml"); } } diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java index e4e5d6242..62235c697 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.*; import org.junit.jupiter.api.BeforeAll; import org.apache.logging.log4j.Logger; @@ -24,6 +24,7 @@ import org.apache.logging.log4j.LogManager; import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -35,15 +36,15 @@ public class StmtEscapingTest { @BeforeAll public static void testLayering() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/escaped_stmts.yaml"); + OpsDocList all = OpsLoader.loadPath("testdocs/escaped_stmts.yaml", Map.of()); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(1); - StmtsDoc doc1 = all.getStmtDocs().get(0); + OpsDoc doc1 = all.getStmtDocs().get(0); // assertThat(doc1.getName()).isEqualTo("doc1"); assertThat(doc1.getBlocks()).hasSize(1); - StmtsBlock block1 = doc1.getBlocks().get(0); + OpsBlock block1 = doc1.getBlocks().get(0); assertThat(block1.getBindings()).hasSize(0); assertThat(block1.getTags()).hasSize(0); assertThat(block1.getOps()).hasSize(3); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtVariationTests.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtVariationTests.java index d70df6c12..1cc2b679f 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtVariationTests.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtVariationTests.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,72 +30,72 @@ public class StmtVariationTests { @Test public void testListStmtsOnly() { - RawStmtsLoader ysl = new RawStmtsLoader(); - RawStmtsDocList docs = ysl.loadString(logger, - "statements:\n" + + RawOpsLoader ysl = new RawOpsLoader(); + RawOpsDocList docs = ysl.loadString( + "statements:\n" + " - first statement\n" + " - second statement\n" ); assertThat(docs.getStmtsDocs()).hasSize(1); - RawStmtsDoc doc = docs.getStmtsDocs().get(0); + RawOpsDoc doc = docs.getStmtsDocs().get(0); assertThat(doc.getRawStmtDefs()).hasSize(2); - List stmts = doc.getRawStmtDefs(); - RawStmtDef s0 = stmts.get(0); + List stmts = doc.getRawStmtDefs(); + RawOpDef s0 = stmts.get(0); assertThat(s0.getName()).isEqualTo("stmt1"); assertThat(s0.getStmt()).isEqualTo("first statement"); - RawStmtDef s1 = stmts.get(1); + RawOpDef s1 = stmts.get(1); assertThat(s1.getName()).isEqualTo("stmt2"); assertThat(s1.getStmt()).isEqualTo("second statement"); } @Test public void testSingleEntryMapStmtsOnly() { - RawStmtsLoader ysl = new RawStmtsLoader(); - RawStmtsDocList docs = ysl.loadString(logger, - "statements:\n" + + RawOpsLoader ysl = new RawOpsLoader(); + RawOpsDocList docs = ysl.loadString( + "statements:\n" + " - s1: statement one\n" + " - s2: statement two\n" ); assertThat(docs.getStmtsDocs()).hasSize(1); - RawStmtsDoc doc = docs.getStmtsDocs().get(0); + RawOpsDoc doc = docs.getStmtsDocs().get(0); assertThat(doc.getRawStmtDefs()).hasSize(2); - List stmts = doc.getRawStmtDefs(); - assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class); + List stmts = doc.getRawStmtDefs(); + assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(0).getName()).isEqualTo("s1"); assertThat(stmts.get(0).getStmt()).isEqualTo("statement one"); - assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class); + assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(1).getName()).isEqualTo("s2"); assertThat(stmts.get(1).getStmt()).isEqualTo("statement two"); } @Test public void testMapStmtsOnly() { - RawStmtsLoader ysl = new RawStmtsLoader(); - RawStmtsDocList docs = ysl.loadString(logger, - "statements:\n" + + RawOpsLoader ysl = new RawOpsLoader(); + RawOpsDocList docs = ysl.loadString( + "statements:\n" + " - name: s1\n" + " stmt: statement one\n" + " - name: s2\n" + " stmt: statement two\n" ); assertThat(docs.getStmtsDocs()).hasSize(1); - RawStmtsDoc doc = docs.getStmtsDocs().get(0); + RawOpsDoc doc = docs.getStmtsDocs().get(0); assertThat(doc.getRawStmtDefs()).hasSize(2); - List stmts = doc.getRawStmtDefs(); - assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class); + List stmts = doc.getRawStmtDefs(); + assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(0).getName()).isEqualTo("s1"); assertThat(stmts.get(0).getStmt()).isEqualTo("statement one"); - assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class); + assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(1).getName()).isEqualTo("s2"); assertThat(stmts.get(1).getStmt()).isEqualTo("statement two"); } @Test public void testMixedForms() { - RawStmtsLoader ysl = new RawStmtsLoader(); - RawStmtsDocList docs = ysl.loadString(logger, - "statement:\n" + + RawOpsLoader ysl = new RawOpsLoader(); + RawOpsDocList docs = ysl.loadString( + "statement:\n" + " - name: s1\n" + " stmt: statement one\n" + " - statement two\n" + @@ -104,19 +104,19 @@ public class StmtVariationTests { " type: organic\n" ); assertThat(docs.getStmtsDocs()).hasSize(1); - RawStmtsDoc doc = docs.getStmtsDocs().get(0); + RawOpsDoc doc = docs.getStmtsDocs().get(0); assertThat(doc.getRawStmtDefs()).hasSize(4); - List stmts = doc.getRawStmtDefs(); - assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class); + List stmts = doc.getRawStmtDefs(); + assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(0).getName()).isEqualTo("s1"); assertThat(stmts.get(0).getStmt()).isEqualTo("statement one"); - assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class); + assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(1).getName()).isEqualTo("stmt2"); assertThat(stmts.get(1).getStmt()).isEqualTo("statement two"); - assertThat(stmts.get(2)).isOfAnyClassIn(RawStmtDef.class); + assertThat(stmts.get(2)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(2).getName()).isEqualTo("s3"); assertThat(stmts.get(2).getStmt()).isEqualTo("statement three"); - assertThat(stmts.get(3)).isOfAnyClassIn(RawStmtDef.class); + assertThat(stmts.get(3)).isOfAnyClassIn(RawOpDef.class); assertThat(stmts.get(3).getName()).isEqualTo("ST4"); assertThat(stmts.get(3).getStmt()).isEqualTo("statement four"); assertThat(stmts.get(3).getParams().get("type")).isEqualTo("organic"); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java index 86e307e44..9bbe834e8 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java @@ -18,10 +18,11 @@ package io.nosqlbench.engine.api.activityconfig.specifications; import com.google.gson.*; import com.google.gson.reflect.TypeToken; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.nb.spectest.api.STAssemblyValidator; import io.nosqlbench.nb.spectest.core.STNodeAssembly; import io.nosqlbench.nb.spectest.loaders.STDefaultLoader; @@ -107,7 +108,7 @@ public class YamlSpecValidator implements STAssemblyValidator { }.getType(); List> expectedList = gson.fromJson(json, type); - StmtsDocList stmtsDocs = StatementsLoader.loadString(yaml, Map.of()); + OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null); List stmts = stmtsDocs.getStmts(); List> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList()); @@ -174,10 +175,9 @@ public class YamlSpecValidator implements STAssemblyValidator { throw new RuntimeException("unknown type in comparator: " + json); } } catch (Exception e) { - StringBuilder sb = new StringBuilder(); - sb.append("error while verifying model:\n"); - sb.append(" path: ").append(testset.getPath().toString()).append("\n"); - sb.append(" line: ").append(testset.getLineNumber()).append("\n"); + String sb = "error while verifying model:\n" + + " path: " + testset.getPath().toString() + "\n" + + " line: " + testset.getLineNumber() + "\n"; logger.error(sb + ": " + e.getMessage(), e); throw new RuntimeException(e); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java similarity index 86% rename from adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java rename to adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java index ba164cd8d..30da12cce 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ package io.nosqlbench.engine.api.activityconfig.yaml; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Test; @@ -26,21 +26,21 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -public class StmtDetailOverrideTest { - private static final Logger logger = LogManager.getLogger(StmtDetailOverrideTest.class); +public class OpDetailOverrideTest { + private static final Logger logger = LogManager.getLogger(OpDetailOverrideTest.class); @Test public void testStmtOverrides() { - StmtsDocList doclist = StatementsLoader.loadPath(logger, "testdocs/stmt_details.yaml"); + OpsDocList doclist = OpsLoader.loadPath("testdocs/stmt_details.yaml", Map.of()); assertThat(doclist).isNotNull(); assertThat(doclist.getStmtDocs()).hasSize(1); - StmtsDoc doc1 = doclist.getStmtDocs().get(0); + OpsDoc doc1 = doclist.getStmtDocs().get(0); assertThat(doc1.getBlocks()).hasSize(2); - StmtsBlock doc1block0 = doc1.getBlocks().get(0); + OpsBlock doc1block0 = doc1.getBlocks().get(0); assertThat(doc1block0.getOps().size()).isEqualTo(1); OpTemplate s = doc1block0.getOps().get(0); assertThat(s.getName()).isEqualTo("block0--stmt1"); @@ -49,7 +49,7 @@ public class StmtDetailOverrideTest { assertThat(s.getParams()).hasSize(1); assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","block0--stmt1")); - StmtsBlock doc1block1 = doc1.getBlocks().get(1); + OpsBlock doc1block1 = doc1.getBlocks().get(1); List stmts = doc1block1.getOps(); assertThat(stmts).hasSize(4); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java similarity index 86% rename from adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java rename to adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java index cd5976299..436d95f61 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ package io.nosqlbench.engine.api.activityconfig.yaml; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import org.assertj.core.data.MapEntry; import org.junit.jupiter.api.BeforeAll; import org.apache.logging.log4j.Logger; @@ -29,10 +29,10 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -public class StmtsDocListTest { +public class OpsDocListTest { - private static final Logger logger = LogManager.getLogger(StmtsDocListTest.class); - private static StmtsDocList doclist; + private static final Logger logger = LogManager.getLogger(OpsDocListTest.class); + private static OpsDocList doclist; private final LinkedHashMap doc0bindings = new LinkedHashMap() {{ put("b2", "b2d"); @@ -50,17 +50,17 @@ public class StmtsDocListTest { @BeforeAll public static void testLoadYaml() { - doclist = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); + doclist = OpsLoader.loadPath("testdocs/docs_blocks_stmts.yaml", Map.of()); } @Test public void testBlocksInheritDocData() { assertThat(doclist).isNotNull(); assertThat(doclist.getStmtDocs()).hasSize(2); - StmtsDoc doc1 = doclist.getStmtDocs().get(0); + OpsDoc doc1 = doclist.getStmtDocs().get(0); assertThat(doc1.getBlocks()).hasSize(1); - StmtsBlock doc1block0 = doc1.getBlocks().get(0); + OpsBlock doc1block0 = doc1.getBlocks().get(0); assertThat(doc1.getTags()).isEqualTo(doc0tags); assertThat(doc1.getBindings()).isEqualTo(doc0bindings); @@ -75,11 +75,11 @@ public class StmtsDocListTest { @Test public void testStmtInheritsBlockData() { - StmtsDoc doc0 = doclist.getStmtDocs().get(0); + OpsDoc doc0 = doclist.getStmtDocs().get(0); List stmts1 = doc0.getBlocks().get(0).getOps(); assertThat(stmts1).hasSize(2); - StmtsBlock block0 = doc0.getBlocks().get(0); + OpsBlock block0 = doc0.getBlocks().get(0); assertThat(block0.getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d")); assertThat(block0.getParams()).containsExactly(MapEntry.entry("param1","value1")); assertThat(block0.getTags()).containsExactly(MapEntry.entry("atagname","atagvalue")); @@ -96,8 +96,8 @@ public class StmtsDocListTest { @Test public void testBlockLayersDocData() { - StmtsDoc doc1 = doclist.getStmtDocs().get(1); - StmtsBlock block0 = doc1.getBlocks().get(0); + OpsDoc doc1 = doclist.getStmtDocs().get(1); + OpsBlock block0 = doc1.getBlocks().get(0); Map doc1block0tags = block0.getTags(); Map doc1block0params = block0.getParamsAsText(); @@ -119,7 +119,7 @@ public class StmtsDocListTest { @Test public void testStmtsGetter() { - StmtsDoc doc1 = doclist.getStmtDocs().get(1); + OpsDoc doc1 = doclist.getStmtDocs().get(1); List stmts = doc1.getStmts(); assertThat(stmts).hasSize(4); } diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtOpTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedOpTemplateTest.java similarity index 80% rename from adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtOpTest.java rename to adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedOpTemplateTest.java index cfee08f47..dc0eb9887 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtOpTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedOpTemplateTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,33 +16,35 @@ package io.nosqlbench.engine.api.activityconfig.yaml; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; -public class ParsedStmtOpTest { - private static final Logger logger = LogManager.getLogger(ParsedStmtOpTest.class); - private static StmtsDocList doclist; +public class ParsedOpTemplateTest { + private static final Logger logger = LogManager.getLogger(ParsedOpTemplateTest.class); + private static OpsDocList doclist; @BeforeAll public static void testLoadYaml() { - doclist = StatementsLoader.loadPath(logger, "testdocs/bindings.yaml"); + doclist = OpsLoader.loadPath("testdocs/bindings.yaml", Map.of()); } @Test public void testBasicParser() { - StmtsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0); + OpsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0); OpTemplate stmtDef0 = block0.getOps().get(0); ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow(); assertThat(parsed0.getMissing()).containsExactly("delta"); assertThat(parsed0.hasError()).isTrue(); - StmtsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1); + OpsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1); OpTemplate stmtDef1 = block1.getOps().get(0); ParsedStringTemplate parsed1 = stmtDef1.getParsed().orElseThrow(); assertThat(parsed1.getMissing()).containsExactly(); @@ -51,7 +53,7 @@ public class ParsedStmtOpTest { @Test public void testMultipleBindingUsage() { - StmtsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2); + OpsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2); OpTemplate stmtDef0 = block2.getOps().get(0); ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow(); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java index 520b39485..96838318a 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,11 @@ package io.nosqlbench.engine.api.templating; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpData; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.api.config.standard.ConfigModel; import io.nosqlbench.api.config.standard.NBConfiguration; import io.nosqlbench.api.config.standard.Param; @@ -68,7 +69,7 @@ public class ParsedOpTest { params: ps1: "param-one" """; - StmtsDocList stmtsDocs = StatementsLoader.loadString(opt, cfg.getMap()); + OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null); assertThat(stmtsDocs.getStmts().size()).isEqualTo(1); OpTemplate opTemplate = stmtsDocs.getStmts().get(0); ParsedOp parsedOp = new ParsedOp(opTemplate, cfg); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java index 935e2a952..9c38f5fab 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java @@ -37,10 +37,10 @@ import io.nosqlbench.engine.api.activityapi.planning.SequencerType; import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter; import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters; import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.activityimpl.uniform.DryRunOpDispenserWrapper; @@ -549,10 +549,10 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs String tagfilter = activityDef.getParams().getOptionalString("tags").orElse(""); - StmtsDocList stmtsDocList = loadStmtsDocList(); + OpsDocList opsDocList = loadStmtsDocList(); - List unfilteredOps = stmtsDocList.getStmts(); - List filteredOps = stmtsDocList.getStmts(tagfilter); + List unfilteredOps = opsDocList.getStmts(); + List filteredOps = opsDocList.getStmts(tagfilter); if (filteredOps.size() == 0) { if (unfilteredOps.size() > 0) { // There were no ops, and it was because they were all filtered out @@ -563,7 +563,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs // In this case, let's try to synthesize the ops as long as at least a default driver was provided if (defaultDriverAdapter.isPresent() && defaultDriverAdapter.get() instanceof SyntheticOpTemplateProvider sotp) { - filteredOps = sotp.getSyntheticOpTemplates(stmtsDocList, getActivityDef().getParams()); + filteredOps = sotp.getSyntheticOpTemplates(opsDocList, getActivityDef().getParams()); Objects.requireNonNull(filteredOps); if (filteredOps.size() == 0) { throw new BasicError("Attempted to create synthetic ops from driver '" + defaultDriverAdapter.get().getAdapterName() + "'" + @@ -573,7 +573,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs throw new BasicError(""" No op templates were provided. You must provide one of these activity parameters: 1) workload=some.yaml - 2) op='inline template + 2) op='inline template' 3) driver=stdout (or any other drive that can synthesize ops)"""); } } @@ -661,20 +661,20 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs return planner.resolve(); } - protected StmtsDocList loadStmtsDocList() { + protected OpsDocList loadStmtsDocList() { try { Optional stmt = activityDef.getParams().getOptionalString("op", "stmt", "statement"); Optional op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload"); if (stmt.isPresent()) { workloadSource = "commandline:" + stmt.get(); - return StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams()); + return OpsLoader.loadString(stmt.get(), OpTemplateFormat.inline, activityDef.getParams(), null); } else if (op_yaml_loc.isPresent()) { workloadSource = "yaml:" + op_yaml_loc.get(); - return StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities"); + return OpsLoader.loadPath(op_yaml_loc.get(), activityDef.getParams(), "activities"); } - return StmtsDocList.none(); + return OpsDocList.none(); } catch (Exception e) { throw new OpConfigError("Error loading op templates: " + e, workloadSource, e); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java index 0b12cfa02..f21c8a468 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java @@ -20,9 +20,9 @@ import io.nosqlbench.api.config.standard.*; import io.nosqlbench.api.engine.activityimpl.ActivityDef; import io.nosqlbench.api.errors.OpConfigError; import io.nosqlbench.engine.api.activityapi.planning.OpSequence; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.activityimpl.OpDispenser; import io.nosqlbench.engine.api.activityimpl.OpMapper; import io.nosqlbench.engine.api.activityimpl.SimpleActivity; @@ -54,11 +54,12 @@ public class StandardActivity extends SimpleActivity implements public StandardActivity(ActivityDef activityDef) { super(activityDef); + OpsDocList workload; Optional yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload"); if (yaml_loc.isPresent()) { Map disposable = new LinkedHashMap<>(activityDef.getParams()); - StmtsDocList workload = StatementsLoader.loadPath(logger, yaml_loc.get(), disposable, "activities"); + workload = OpsLoader.loadPath(yaml_loc.get(), disposable, "activities"); yamlmodel = workload.getConfigModel(); } else { yamlmodel = ConfigModel.of(StandardActivity.class).asReadOnly(); @@ -163,11 +164,11 @@ public class StandardActivity extends SimpleActivity implements } @Override - public List getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map cfg) { + public List getSyntheticOpTemplates(OpsDocList opsDocList, Map cfg) { List opTemplates = new ArrayList<>(); for (DriverAdapter adapter : adapters.values()) { if (adapter instanceof SyntheticOpTemplateProvider sotp) { - List newTemplates = sotp.getSyntheticOpTemplates(stmtsDocList, cfg); + List newTemplates = sotp.getSyntheticOpTemplates(opsDocList, cfg); opTemplates.addAll(newTemplates); } } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java index 55370c728..c319852e6 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,8 +18,8 @@ package io.nosqlbench.engine.api.activityimpl.uniform; import io.nosqlbench.engine.api.activityapi.core.ActionDispenser; import io.nosqlbench.engine.api.activityapi.core.ActivityType; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.api.engine.activityimpl.ActivityDef; import io.nosqlbench.engine.api.activityimpl.SimpleActivity; import io.nosqlbench.api.config.standard.NBConfigModel; @@ -69,7 +69,7 @@ public class StandardActivityType> extends Simpl Optional op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload"); if (op_yaml_loc.isPresent()) { Map disposable = new LinkedHashMap<>(activityDef.getParams()); - StmtsDocList workload = StatementsLoader.loadPath(logger, op_yaml_loc.get(), disposable, "activities"); + OpsDocList workload = OpsLoader.loadPath(op_yaml_loc.get(), disposable, "activities"); cfgModel=cfgModel.add(workload.getConfigModel()); } NBConfiguration cfg = cfgModel.apply(activityDef.getParams()); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java b/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java index bac785305..73f372582 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java @@ -16,10 +16,10 @@ package io.nosqlbench.engine.api.scenarios; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; import io.nosqlbench.engine.api.templating.StrInterpolator; import io.nosqlbench.api.content.Content; import io.nosqlbench.api.content.NBIO; @@ -53,7 +53,7 @@ public class NBCLIScenarioParser { .prefix("activities") .prefix(includes) .name(workload) - .extension(RawStmtsLoader.YAML_EXTENSIONS) + .extension(RawOpsLoader.YAML_EXTENSIONS) .first(); return found.isPresent(); } @@ -67,7 +67,7 @@ public class NBCLIScenarioParser { .prefix("activities") .prefix(includes) .name(workloadName) - .extension(RawStmtsLoader.YAML_EXTENSIONS) + .extension(RawOpsLoader.YAML_EXTENSIONS) .first(); // Content workloadContent = found.orElseThrow(); @@ -111,10 +111,10 @@ public class NBCLIScenarioParser { .prefix(SEARCH_IN) .prefix(includes) .name(workloadName) - .extension(RawStmtsLoader.YAML_EXTENSIONS) + .extension(RawOpsLoader.YAML_EXTENSIONS) .first().orElseThrow(); // TODO: The yaml needs to be parsed with arguments from each command independently to support template vars - StmtsDocList scenariosYaml = StatementsLoader.loadContent(logger, yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams)); + OpsDocList scenariosYaml = OpsLoader.loadContent(yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams)); Scenarios scenarios = scenariosYaml.getDocScenarios(); String[] nameparts = scenarioName.split("\\.",2); @@ -314,12 +314,12 @@ public class NBCLIScenarioParser { } Content content = NBIO.all().prefix(SEARCH_IN) - .name(referenced).extension(RawStmtsLoader.YAML_EXTENSIONS) + .name(referenced).extension(RawOpsLoader.YAML_EXTENSIONS) .one(); - StmtsDocList stmts = null; + OpsDocList stmts = null; try { - stmts = StatementsLoader.loadContent(logger, content, Map.of()); + stmts = OpsLoader.loadContent(content, Map.of()); if (stmts.getStmtDocs().size() == 0) { logger.warn("Encountered yaml with no docs in '" + referenced + "'"); continue; @@ -379,7 +379,7 @@ public class NBCLIScenarioParser { List> activities = searchin .prefix(includes) - .extension(RawStmtsLoader.YAML_EXTENSIONS) + .extension(RawOpsLoader.YAML_EXTENSIONS) .list(); return filterForScenarios(activities); diff --git a/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java b/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java index 1041452b9..1644a0d37 100644 --- a/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java +++ b/engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,9 +18,10 @@ package io.nosqlbench.engine.api.templating; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Test; @@ -34,10 +35,10 @@ public class CommandTemplateTest { @Test public void testCommandTemplate() { - StmtsDocList stmtsDocs = StatementsLoader.loadString("" + + OpsDocList stmtsDocs = OpsLoader.loadString("" + "statements:\n" + " - s1: test1=foo test2=bar", - Map.of()); + OpTemplateFormat.yaml, Map.of(), null); OpTemplate optpl = stmtsDocs.getStmts().get(0); CommandTemplate ct = new CommandTemplate(optpl); assertThat(ct.isStatic()).isTrue(); @@ -46,12 +47,12 @@ public class CommandTemplateTest { @Test public void testCommandTemplateFormat() { Gson gson = new GsonBuilder().setPrettyPrinting().create(); - StmtsDocList stmtsDocs = StatementsLoader.loadString("" + + OpsDocList stmtsDocs = OpsLoader.loadString("" + "statements:\n" + " - s1: test1=foo test2={bar}\n" + " bindings:\n" + " bar: NumberNameToString();\n", - Map.of() + OpTemplateFormat.yaml, Map.of(), null ); OpTemplate optpl = stmtsDocs.getStmts().get(0); CommandTemplate ct = new CommandTemplate(optpl); diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index ef0878db3..53030962d 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogDu import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogImporterUtility; import io.nosqlbench.engine.api.activityapi.input.InputType; import io.nosqlbench.engine.api.activityapi.output.OutputType; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader; import io.nosqlbench.engine.core.annotation.Annotators; import io.nosqlbench.engine.core.lifecycle.process.NBCLIErrorHandler; import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader; @@ -304,7 +304,7 @@ public class NBCLI implements Function { Optional> tocopy = NBIO.classpath() .prefix("activities") .prefix(options.wantsIncludes()) - .name(resourceToCopy).extension(RawStmtsLoader.YAML_EXTENSIONS).first(); + .name(resourceToCopy).extension(RawOpsLoader.YAML_EXTENSIONS).first(); if (tocopy.isEmpty()) { diff --git a/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java index 98bb4204a..9b89c8bef 100644 --- a/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java +++ b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,9 @@ package io.nosqlbench.engine.cli; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; import org.junit.jupiter.api.Test; import java.util.List; @@ -33,12 +33,12 @@ public class NBCLIScenarioParserTemplateVarTest { List cmds = opts.getCommands(); cmds.forEach(System.out::println); - StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams()); + OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams()); OpTemplate optpl1 = workload1.getStmts().get(0); System.out.println("op from cmd1:"+optpl1); assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n"); - StmtsDocList workload2 = StatementsLoader.loadPath(null, cmds.get(1).getArg("workload"),cmds.get(1).getParams()); + OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArg("workload"),cmds.get(1).getParams()); OpTemplate optpl2 = workload2.getStmts().get(0); System.out.println("op from cmd2:"+optpl2); assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n"); @@ -50,7 +50,7 @@ public class NBCLIScenarioParserTemplateVarTest { List cmds = opts.getCommands(); cmds.forEach(System.out::println); - StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams()); + OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams()); OpTemplate optpl1 = workload1.getStmts().get(0); System.out.println("op from cmd1:"+optpl1); assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n"); diff --git a/nbr-examples/pom.xml b/nbr-examples/pom.xml index 5e66c79a8..820262ef2 100644 --- a/nbr-examples/pom.xml +++ b/nbr-examples/pom.xml @@ -57,6 +57,14 @@ src/test/resources false + + src/test/resources + true + + **/log4j2-test.xml + **/log4j2.xml + +