simplify methods further with fewer logger parameters

This commit is contained in:
Jonathan Shook 2023-01-24 18:09:15 -06:00
parent 23ac085037
commit 9b745dce31
13 changed files with 52 additions and 67 deletions

View File

@ -16,13 +16,13 @@
package io.nosqlbench.engine.api.activityconfig; package io.nosqlbench.engine.api.activityconfig;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator; import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
@ -31,10 +31,26 @@ import java.util.Optional;
public class StatementsLoader { public class StatementsLoader {
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
private final static Logger logger = LogManager.getLogger(StatementsLoader.class); private final static Logger logger = LogManager.getLogger(StatementsLoader.class);
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml", "json", "jsonnet"};
public static StmtsDocList loadContent(Content<?> content, Map<String,String> params) {
return loadString(content.get().toString(),params);
}
public static StmtsDocList loadPath(String path, Map<String,?> params, String... searchPaths) {
RawStmtsDocList list = null;
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(content,params);
}
public static StmtsDocList loadPath(
String path,
String... searchPaths) {
return loadPath(path, Map.of(), searchPaths);
}
public static StmtsDocList loadString(String yamlContent, Map<String,?> params) { public static StmtsDocList loadString(String yamlContent, Map<String,?> params) {
StrInterpolator transformer = new StrInterpolator(params); StrInterpolator transformer = new StrInterpolator(params);
@ -48,11 +64,7 @@ public class StatementsLoader {
return layered; return layered;
} }
public static StmtsDocList loadStmt( public static StmtsDocList loadStmt(String statement, Map<String,?> params) {
Logger logger,
String statement,
Map<String,?> params
) {
StrInterpolator transformer = new StrInterpolator(params); StrInterpolator transformer = new StrInterpolator(params);
statement = transformer.apply(statement); statement = transformer.apply(statement);
RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(statement); RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(statement);
@ -64,32 +76,6 @@ public class StatementsLoader {
return layered; return layered;
} }
public static StmtsDocList loadContent(
Logger logger,
Content<?> content,
Map<String,String> params
) {
return loadString(content.get().toString(),params);
}
public static StmtsDocList loadPath(
Logger logger,
String path,
Map<String,?> params,
String... searchPaths) {
RawStmtsDocList list = null;
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(content,params);
}
public static StmtsDocList loadPath(
Logger logger,
String path,
String... searchPaths) {
return loadPath(logger, path, Map.of(), searchPaths);
}
} }

View File

@ -161,7 +161,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
} }
public static NBConfigModelExpander TEMPLATE_VAR_EXPANDER = workload -> { public static NBConfigModelExpander TEMPLATE_VAR_EXPANDER = workload -> {
StmtsDocList loaded = StatementsLoader.loadPath(logger, (String) workload, "activities"); StmtsDocList loaded = StatementsLoader.loadPath((String) workload, "activities");
return loaded.getConfigModel(); return loaded.getConfigModel();
}; };

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -29,7 +29,7 @@ public class StatementsLoaderTest {
@Test @Test
public void testTemplateVarSubstitution() { public void testTemplateVarSubstitution() {
StmtsDocList stmtsDocs = StatementsLoader.loadPath(null, "activities/template_vars", "src/test/resources"); StmtsDocList stmtsDocs = StatementsLoader.loadPath("activities/template_vars", "src/test/resources");
assertThat(stmtsDocs).isNotNull(); assertThat(stmtsDocs).isNotNull();
List<StmtsDoc> docs = stmtsDocs.getStmtDocs(); List<StmtsDoc> docs = stmtsDocs.getStmtDocs();
assertThat(docs).hasSize(1); assertThat(docs).hasSize(1);
@ -46,7 +46,7 @@ public class StatementsLoaderTest {
public void testInvalidYamlProperties() { public void testInvalidYamlProperties() {
Exception caught = null; Exception caught = null;
try { try {
StatementsLoader.loadPath(null, "activities/invalid_prop", "src/test/resources"); StatementsLoader.loadPath("activities/invalid_prop", "src/test/resources");
} catch (Exception e) { } catch (Exception e) {
caught = e; caught = e;
} }

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -37,7 +37,7 @@ public class OpDefTest {
@Test @Test
public void testLayering() { public void testLayering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); StmtsDocList all = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
assertThat(all).isNotNull(); assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2); assertThat(all.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = all.getStmtDocs().get(0); StmtsDoc doc1 = all.getStmtDocs().get(0);
@ -64,7 +64,7 @@ public class OpDefTest {
@Test @Test
public void testStatementRendering() { public void testStatementRendering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); StmtsDocList all = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
assertThat(all).isNotNull(); assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2); assertThat(all.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = all.getStmtDocs().get(0); StmtsDoc doc1 = all.getStmtDocs().get(0);
@ -79,7 +79,7 @@ public class OpDefTest {
@Test @Test
public void testConsumableMapState() { public void testConsumableMapState() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); StmtsDocList all = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
List<StmtsDoc> docs = all.getStmtDocs(); List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc block1 = docs.get(1); StmtsDoc block1 = docs.get(1);
List<OpTemplate> stmts = block1.getStmts(); List<OpTemplate> stmts = block1.getStmts();
@ -94,7 +94,7 @@ public class OpDefTest {
@Test @Test
public void testMapOfMaps() { public void testMapOfMaps() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml"); StmtsDocList all = StatementsLoader.loadPath("testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs(); List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc0 = docs.get(0); StmtsDoc doc0 = docs.get(0);
assertThat(doc0.getName()).isEqualTo("map-of-maps"); assertThat(doc0.getName()).isEqualTo("map-of-maps");
@ -115,7 +115,7 @@ public class OpDefTest {
@Test @Test
public void testBasicStringStmt() { public void testBasicStringStmt() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml"); StmtsDocList all = StatementsLoader.loadPath("testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs(); List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc1 = docs.get(1); StmtsDoc doc1 = docs.get(1);
assertThat(doc1.getName()).isEqualTo("string-statement"); assertThat(doc1.getName()).isEqualTo("string-statement");
@ -130,7 +130,7 @@ public class OpDefTest {
@Test @Test
public void testListOfNamedMap() { public void testListOfNamedMap() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml"); StmtsDocList all = StatementsLoader.loadPath("testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs(); List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc2 = docs.get(2); StmtsDoc doc2 = docs.get(2);
assertThat(doc2.getName()).isEqualTo("list-of-named-map"); assertThat(doc2.getName()).isEqualTo("list-of-named-map");

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -35,7 +35,7 @@ public class StmtEscapingTest {
@BeforeAll @BeforeAll
public static void testLayering() { public static void testLayering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/escaped_stmts.yaml"); StmtsDocList all = StatementsLoader.loadPath("testdocs/escaped_stmts.yaml");
assertThat(all).isNotNull(); assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(1); assertThat(all.getStmtDocs()).hasSize(1);
StmtsDoc doc1 = all.getStmtDocs().get(0); StmtsDoc doc1 = all.getStmtDocs().get(0);

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -31,7 +31,7 @@ public class ParsedStmtOpTest {
@BeforeAll @BeforeAll
public static void testLoadYaml() { public static void testLoadYaml() {
doclist = StatementsLoader.loadPath(logger, "testdocs/bindings.yaml"); doclist = StatementsLoader.loadPath("testdocs/bindings.yaml");
} }
@Test @Test

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -32,7 +32,7 @@ public class StmtDetailOverrideTest {
@Test @Test
public void testStmtOverrides() { public void testStmtOverrides() {
StmtsDocList doclist = StatementsLoader.loadPath(logger, "testdocs/stmt_details.yaml"); StmtsDocList doclist = StatementsLoader.loadPath("testdocs/stmt_details.yaml");
assertThat(doclist).isNotNull(); assertThat(doclist).isNotNull();

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -50,7 +50,7 @@ public class StmtsDocListTest {
@BeforeAll @BeforeAll
public static void testLoadYaml() { public static void testLoadYaml() {
doclist = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); doclist = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
} }
@Test @Test

View File

@ -38,7 +38,6 @@ import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters; import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec; import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader; import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally; import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
@ -668,10 +667,10 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload"); Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (stmt.isPresent()) { if (stmt.isPresent()) {
workloadSource = "commandline:" + stmt.get(); workloadSource = "commandline:" + stmt.get();
return StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams()); return StatementsLoader.loadStmt(stmt.get(), activityDef.getParams());
} else if (op_yaml_loc.isPresent()) { } else if (op_yaml_loc.isPresent()) {
workloadSource = "yaml:" + op_yaml_loc.get(); workloadSource = "yaml:" + op_yaml_loc.get();
return StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities"); return StatementsLoader.loadPath(op_yaml_loc.get(), activityDef.getParams(), "activities");
} }
return StmtsDocList.none(); return StmtsDocList.none();

View File

@ -58,7 +58,7 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
Optional<String> yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload"); Optional<String> yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (yaml_loc.isPresent()) { if (yaml_loc.isPresent()) {
Map<String, Object> disposable = new LinkedHashMap<>(activityDef.getParams()); Map<String, Object> disposable = new LinkedHashMap<>(activityDef.getParams());
StmtsDocList workload = StatementsLoader.loadPath(logger, yaml_loc.get(), disposable, "activities"); StmtsDocList workload = StatementsLoader.loadPath(yaml_loc.get(), disposable, "activities");
yamlmodel = workload.getConfigModel(); yamlmodel = workload.getConfigModel();
} else { } else {
yamlmodel = ConfigModel.of(StandardActivity.class).asReadOnly(); yamlmodel = ConfigModel.of(StandardActivity.class).asReadOnly();

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -69,7 +69,7 @@ public class StandardActivityType<A extends StandardActivity<?,?>> extends Simpl
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload"); Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (op_yaml_loc.isPresent()) { if (op_yaml_loc.isPresent()) {
Map<String,Object> disposable = new LinkedHashMap<>(activityDef.getParams()); Map<String,Object> disposable = new LinkedHashMap<>(activityDef.getParams());
StmtsDocList workload = StatementsLoader.loadPath(logger, op_yaml_loc.get(), disposable, "activities"); StmtsDocList workload = StatementsLoader.loadPath(op_yaml_loc.get(), disposable, "activities");
cfgModel=cfgModel.add(workload.getConfigModel()); cfgModel=cfgModel.add(workload.getConfigModel());
} }
NBConfiguration cfg = cfgModel.apply(activityDef.getParams()); NBConfiguration cfg = cfgModel.apply(activityDef.getParams());

View File

@ -114,7 +114,7 @@ public class NBCLIScenarioParser {
.extension(RawStmtsLoader.YAML_EXTENSIONS) .extension(RawStmtsLoader.YAML_EXTENSIONS)
.first().orElseThrow(); .first().orElseThrow();
// TODO: The yaml needs to be parsed with arguments from each command independently to support template vars // TODO: The yaml needs to be parsed with arguments from each command independently to support template vars
StmtsDocList scenariosYaml = StatementsLoader.loadContent(logger, yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams)); StmtsDocList scenariosYaml = StatementsLoader.loadContent(yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
Scenarios scenarios = scenariosYaml.getDocScenarios(); Scenarios scenarios = scenariosYaml.getDocScenarios();
String[] nameparts = scenarioName.split("\\.",2); String[] nameparts = scenarioName.split("\\.",2);
@ -319,7 +319,7 @@ public class NBCLIScenarioParser {
StmtsDocList stmts = null; StmtsDocList stmts = null;
try { try {
stmts = StatementsLoader.loadContent(logger, content, Map.of()); stmts = StatementsLoader.loadContent(content, Map.of());
if (stmts.getStmtDocs().size() == 0) { if (stmts.getStmtDocs().size() == 0) {
logger.warn("Encountered yaml with no docs in '" + referenced + "'"); logger.warn("Encountered yaml with no docs in '" + referenced + "'");
continue; continue;

View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2022 nosqlbench * Copyright (c) 2022-2023 nosqlbench
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -33,12 +33,12 @@ public class NBCLIScenarioParserTemplateVarTest {
List<Cmd> cmds = opts.getCommands(); List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println); cmds.forEach(System.out::println);
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams()); StmtsDocList workload1 = StatementsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0); OpTemplate optpl1 = workload1.getStmts().get(0);
System.out.println("op from cmd1:"+optpl1); System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n"); assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n");
StmtsDocList workload2 = StatementsLoader.loadPath(null, cmds.get(1).getArg("workload"),cmds.get(1).getParams()); StmtsDocList workload2 = StatementsLoader.loadPath(cmds.get(1).getArg("workload"),cmds.get(1).getParams());
OpTemplate optpl2 = workload2.getStmts().get(0); OpTemplate optpl2 = workload2.getStmts().get(0);
System.out.println("op from cmd2:"+optpl2); System.out.println("op from cmd2:"+optpl2);
assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n"); assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n");
@ -50,7 +50,7 @@ public class NBCLIScenarioParserTemplateVarTest {
List<Cmd> cmds = opts.getCommands(); List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println); cmds.forEach(System.out::println);
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams()); StmtsDocList workload1 = StatementsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0); OpTemplate optpl1 = workload1.getStmts().get(0);
System.out.println("op from cmd1:"+optpl1); System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n"); assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n");