IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-20 11:38:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

simplify methods further with fewer logger parameters

Browse Source
This commit is contained in:
Jonathan Shook 2023-01-24 18:09:15 -06:00
parent 23ac085037
commit 9b745dce31
13 changed files with 52 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java
View File

@ -16,13 +16,13 @@
package io.nosqlbench.engine.api.activityconfig;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -31,10 +31,26 @@ import java.util.Optional;
public class StatementsLoader {
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
private final static Logger logger = LogManager.getLogger(StatementsLoader.class);
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml", "json", "jsonnet"};
public static StmtsDocList loadContent(Content<?> content, Map<String,String> params) {
return loadString(content.get().toString(),params);
}
public static StmtsDocList loadPath(String path, Map<String,?> params, String... searchPaths) {
RawStmtsDocList list = null;
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(content,params);
}
public static StmtsDocList loadPath(
String path,
String... searchPaths) {
return loadPath(path, Map.of(), searchPaths);
}
public static StmtsDocList loadString(String yamlContent, Map<String,?> params) {
StrInterpolator transformer = new StrInterpolator(params);
@ -48,11 +64,7 @@ public class StatementsLoader {
return layered;
}
public static StmtsDocList loadStmt(
Logger logger,
String statement,
Map<String,?> params
) {
public static StmtsDocList loadStmt(String statement, Map<String,?> params) {
StrInterpolator transformer = new StrInterpolator(params);
statement = transformer.apply(statement);
RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(statement);
@ -64,32 +76,6 @@ public class StatementsLoader {
return layered;
}
public static StmtsDocList loadContent(
Logger logger,
Content<?> content,
Map<String,String> params
) {
return loadString(content.get().toString(),params);
}
public static StmtsDocList loadPath(
Logger logger,
String path,
Map<String,?> params,
String... searchPaths) {
RawStmtsDocList list = null;
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(content,params);
}
public static StmtsDocList loadPath(
Logger logger,
String path,
String... searchPaths) {
return loadPath(logger, path, Map.of(), searchPaths);
}
}

2
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java
View File

@ -161,7 +161,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
}
public static NBConfigModelExpander TEMPLATE_VAR_EXPANDER = workload -> {
StmtsDocList loaded = StatementsLoader.loadPath(logger, (String) workload, "activities");
StmtsDocList loaded = StatementsLoader.loadPath((String) workload, "activities");
return loaded.getConfigModel();
};

6
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/StatementsLoaderTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -29,7 +29,7 @@ public class StatementsLoaderTest {
@Test
public void testTemplateVarSubstitution() {
StmtsDocList stmtsDocs = StatementsLoader.loadPath(null, "activities/template_vars", "src/test/resources");
StmtsDocList stmtsDocs = StatementsLoader.loadPath("activities/template_vars", "src/test/resources");
assertThat(stmtsDocs).isNotNull();
List<StmtsDoc> docs = stmtsDocs.getStmtDocs();
assertThat(docs).hasSize(1);
@ -46,7 +46,7 @@ public class StatementsLoaderTest {
public void testInvalidYamlProperties() {
Exception caught = null;
try {
StatementsLoader.loadPath(null, "activities/invalid_prop", "src/test/resources");
StatementsLoader.loadPath("activities/invalid_prop", "src/test/resources");
} catch (Exception e) {
caught = e;
}

14
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -37,7 +37,7 @@ public class OpDefTest {
@Test
public void testLayering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
StmtsDocList all = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = all.getStmtDocs().get(0);
@ -64,7 +64,7 @@ public class OpDefTest {
@Test
public void testStatementRendering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
StmtsDocList all = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = all.getStmtDocs().get(0);
@ -79,7 +79,7 @@ public class OpDefTest {
@Test
public void testConsumableMapState() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
StmtsDocList all = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc block1 = docs.get(1);
List<OpTemplate> stmts = block1.getStmts();
@ -94,7 +94,7 @@ public class OpDefTest {
@Test
public void testMapOfMaps() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
StmtsDocList all = StatementsLoader.loadPath("testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc0 = docs.get(0);
assertThat(doc0.getName()).isEqualTo("map-of-maps");
@ -115,7 +115,7 @@ public class OpDefTest {
@Test
public void testBasicStringStmt() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
StmtsDocList all = StatementsLoader.loadPath("testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc1 = docs.get(1);
assertThat(doc1.getName()).isEqualTo("string-statement");
@ -130,7 +130,7 @@ public class OpDefTest {
@Test
public void testListOfNamedMap() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
StmtsDocList all = StatementsLoader.loadPath("testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc2 = docs.get(2);
assertThat(doc2.getName()).isEqualTo("list-of-named-map");

4
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -35,7 +35,7 @@ public class StmtEscapingTest {
@BeforeAll
public static void testLayering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/escaped_stmts.yaml");
StmtsDocList all = StatementsLoader.loadPath("testdocs/escaped_stmts.yaml");
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(1);
StmtsDoc doc1 = all.getStmtDocs().get(0);

4
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtOpTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -31,7 +31,7 @@ public class ParsedStmtOpTest {
@BeforeAll
public static void testLoadYaml() {
doclist = StatementsLoader.loadPath(logger, "testdocs/bindings.yaml");
doclist = StatementsLoader.loadPath("testdocs/bindings.yaml");
}
@Test

4
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -32,7 +32,7 @@ public class StmtDetailOverrideTest {
@Test
public void testStmtOverrides() {
StmtsDocList doclist = StatementsLoader.loadPath(logger, "testdocs/stmt_details.yaml");
StmtsDocList doclist = StatementsLoader.loadPath("testdocs/stmt_details.yaml");
assertThat(doclist).isNotNull();

4
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -50,7 +50,7 @@ public class StmtsDocListTest {
@BeforeAll
public static void testLoadYaml() {
doclist = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
doclist = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml");
}
@Test

5
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -38,7 +38,6 @@ import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
@ -668,10 +667,10 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (stmt.isPresent()) {
workloadSource = "commandline:" + stmt.get();
return StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams());
return StatementsLoader.loadStmt(stmt.get(), activityDef.getParams());
} else if (op_yaml_loc.isPresent()) {
workloadSource = "yaml:" + op_yaml_loc.get();
return StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities");
return StatementsLoader.loadPath(op_yaml_loc.get(), activityDef.getParams(), "activities");
}
return StmtsDocList.none();

2
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java
View File

@ -58,7 +58,7 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
Optional<String> yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (yaml_loc.isPresent()) {
Map<String, Object> disposable = new LinkedHashMap<>(activityDef.getParams());
StmtsDocList workload = StatementsLoader.loadPath(logger, yaml_loc.get(), disposable, "activities");
StmtsDocList workload = StatementsLoader.loadPath(yaml_loc.get(), disposable, "activities");
yamlmodel = workload.getConfigModel();
} else {
yamlmodel = ConfigModel.of(StandardActivity.class).asReadOnly();

4
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -69,7 +69,7 @@ public class StandardActivityType<A extends StandardActivity<?,?>> extends Simpl
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (op_yaml_loc.isPresent()) {
Map<String,Object> disposable = new LinkedHashMap<>(activityDef.getParams());
StmtsDocList workload = StatementsLoader.loadPath(logger, op_yaml_loc.get(), disposable, "activities");
StmtsDocList workload = StatementsLoader.loadPath(op_yaml_loc.get(), disposable, "activities");
cfgModel=cfgModel.add(workload.getConfigModel());
}
NBConfiguration cfg = cfgModel.apply(activityDef.getParams());

4
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -114,7 +114,7 @@ public class NBCLIScenarioParser {
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.first().orElseThrow();
// TODO: The yaml needs to be parsed with arguments from each command independently to support template vars
StmtsDocList scenariosYaml = StatementsLoader.loadContent(logger, yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
StmtsDocList scenariosYaml = StatementsLoader.loadContent(yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
Scenarios scenarios = scenariosYaml.getDocScenarios();
String[] nameparts = scenarioName.split("\\.",2);
@ -319,7 +319,7 @@ public class NBCLIScenarioParser {
StmtsDocList stmts = null;
try {
stmts = StatementsLoader.loadContent(logger, content, Map.of());
stmts = StatementsLoader.loadContent(content, Map.of());
if (stmts.getStmtDocs().size() == 0) {
logger.warn("Encountered yaml with no docs in '" + referenced + "'");
continue;

8
engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -33,12 +33,12 @@ public class NBCLIScenarioParserTemplateVarTest {
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams());
StmtsDocList workload1 = StatementsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0);
System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n");
StmtsDocList workload2 = StatementsLoader.loadPath(null, cmds.get(1).getArg("workload"),cmds.get(1).getParams());
StmtsDocList workload2 = StatementsLoader.loadPath(cmds.get(1).getArg("workload"),cmds.get(1).getParams());
OpTemplate optpl2 = workload2.getStmts().get(0);
System.out.println("op from cmd2:"+optpl2);
assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n");
@ -50,7 +50,7 @@ public class NBCLIScenarioParserTemplateVarTest {
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams());
StmtsDocList workload1 = StatementsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0);
System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n");