From 9b745dce31214f7bc55d679405d8fd315d0f01e1 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 24 Jan 2023 18:09:15 -0600 Subject: [PATCH] simplify methods further with fewer logger parameters --- .../api/activityconfig/StatementsLoader.java | 58 +++++++------------ .../api/activityconfig/yaml/StmtsDocList.java | 2 +- .../activityconfig/StatementsLoaderTest.java | 6 +- .../api/activityconfig/rawyaml/OpDefTest.java | 14 ++--- .../rawyaml/StmtEscapingTest.java | 4 +- .../activityconfig/yaml/ParsedStmtOpTest.java | 4 +- .../yaml/StmtDetailOverrideTest.java | 4 +- .../activityconfig/yaml/StmtsDocListTest.java | 4 +- .../api/activityimpl/SimpleActivity.java | 5 +- .../uniform/StandardActivity.java | 2 +- .../uniform/StandardActivityType.java | 4 +- .../api/scenarios/NBCLIScenarioParser.java | 4 +- .../NBCLIScenarioParserTemplateVarTest.java | 8 +-- 13 files changed, 52 insertions(+), 67 deletions(-) diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java index 7e3352ef2..0e7e1b16d 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java @@ -16,13 +16,13 @@ package io.nosqlbench.engine.api.activityconfig; +import io.nosqlbench.api.content.Content; +import io.nosqlbench.api.content.NBIO; +import io.nosqlbench.api.errors.BasicError; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; import io.nosqlbench.engine.api.templating.StrInterpolator; -import io.nosqlbench.api.content.Content; -import io.nosqlbench.api.content.NBIO; -import io.nosqlbench.api.errors.BasicError; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -31,10 +31,26 @@ import java.util.Optional; public class StatementsLoader { - public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"}; - private final static Logger logger = LogManager.getLogger(StatementsLoader.class); + public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml", "json", "jsonnet"}; + + public static StmtsDocList loadContent(Content content, Map params) { + return loadString(content.get().toString(),params); + } + + public static StmtsDocList loadPath(String path, Map params, String... searchPaths) { + RawStmtsDocList list = null; + Optional> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first(); + String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path)); + return loadString(content,params); + } + public static StmtsDocList loadPath( + String path, + String... searchPaths) { + return loadPath(path, Map.of(), searchPaths); + } + public static StmtsDocList loadString(String yamlContent, Map params) { StrInterpolator transformer = new StrInterpolator(params); @@ -48,11 +64,7 @@ public class StatementsLoader { return layered; } - public static StmtsDocList loadStmt( - Logger logger, - String statement, - Map params - ) { + public static StmtsDocList loadStmt(String statement, Map params) { StrInterpolator transformer = new StrInterpolator(params); statement = transformer.apply(statement); RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(statement); @@ -64,32 +76,6 @@ public class StatementsLoader { return layered; } - public static StmtsDocList loadContent( - Logger logger, - Content content, - Map params - ) { - return loadString(content.get().toString(),params); - } - public static StmtsDocList loadPath( - Logger logger, - String path, - Map params, - String... searchPaths) { - - RawStmtsDocList list = null; - Optional> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first(); - String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path)); - return loadString(content,params); - } - - - public static StmtsDocList loadPath( - Logger logger, - String path, - String... searchPaths) { - return loadPath(logger, path, Map.of(), searchPaths); - } } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java index 800438219..b1a480455 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java @@ -161,7 +161,7 @@ public class StmtsDocList implements Iterable { } public static NBConfigModelExpander TEMPLATE_VAR_EXPANDER = workload -> { - StmtsDocList loaded = StatementsLoader.loadPath(logger, (String) workload, "activities"); + StmtsDocList loaded = StatementsLoader.loadPath((String) workload, "activities"); return loaded.getConfigModel(); }; diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/StatementsLoaderTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/StatementsLoaderTest.java index 4f203815f..0c92cbd0b 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/StatementsLoaderTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/StatementsLoaderTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ public class StatementsLoaderTest { @Test public void testTemplateVarSubstitution() { - StmtsDocList stmtsDocs = StatementsLoader.loadPath(null, "activities/template_vars", "src/test/resources"); + StmtsDocList stmtsDocs = StatementsLoader.loadPath("activities/template_vars", "src/test/resources"); assertThat(stmtsDocs).isNotNull(); List docs = stmtsDocs.getStmtDocs(); assertThat(docs).hasSize(1); @@ -46,7 +46,7 @@ public class StatementsLoaderTest { public void testInvalidYamlProperties() { Exception caught = null; try { - StatementsLoader.loadPath(null, "activities/invalid_prop", "src/test/resources"); + StatementsLoader.loadPath("activities/invalid_prop", "src/test/resources"); } catch (Exception e) { caught = e; } diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java index b6414e4b3..199fcd4a8 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ public class OpDefTest { @Test public void testLayering() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); + StmtsDocList all = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml"); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(2); StmtsDoc doc1 = all.getStmtDocs().get(0); @@ -64,7 +64,7 @@ public class OpDefTest { @Test public void testStatementRendering() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); + StmtsDocList all = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml"); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(2); StmtsDoc doc1 = all.getStmtDocs().get(0); @@ -79,7 +79,7 @@ public class OpDefTest { @Test public void testConsumableMapState() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); + StmtsDocList all = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml"); List docs = all.getStmtDocs(); StmtsDoc block1 = docs.get(1); List stmts = block1.getStmts(); @@ -94,7 +94,7 @@ public class OpDefTest { @Test public void testMapOfMaps() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml"); + StmtsDocList all = StatementsLoader.loadPath("testdocs/statement_variants.yaml"); List docs = all.getStmtDocs(); StmtsDoc doc0 = docs.get(0); assertThat(doc0.getName()).isEqualTo("map-of-maps"); @@ -115,7 +115,7 @@ public class OpDefTest { @Test public void testBasicStringStmt() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml"); + StmtsDocList all = StatementsLoader.loadPath("testdocs/statement_variants.yaml"); List docs = all.getStmtDocs(); StmtsDoc doc1 = docs.get(1); assertThat(doc1.getName()).isEqualTo("string-statement"); @@ -130,7 +130,7 @@ public class OpDefTest { @Test public void testListOfNamedMap() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml"); + StmtsDocList all = StatementsLoader.loadPath("testdocs/statement_variants.yaml"); List docs = all.getStmtDocs(); StmtsDoc doc2 = docs.get(2); assertThat(doc2.getName()).isEqualTo("list-of-named-map"); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java index e4e5d6242..ad64a92ec 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ public class StmtEscapingTest { @BeforeAll public static void testLayering() { - StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/escaped_stmts.yaml"); + StmtsDocList all = StatementsLoader.loadPath("testdocs/escaped_stmts.yaml"); assertThat(all).isNotNull(); assertThat(all.getStmtDocs()).hasSize(1); StmtsDoc doc1 = all.getStmtDocs().get(0); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtOpTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtOpTest.java index cfee08f47..4bef95bc2 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtOpTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtOpTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ public class ParsedStmtOpTest { @BeforeAll public static void testLoadYaml() { - doclist = StatementsLoader.loadPath(logger, "testdocs/bindings.yaml"); + doclist = StatementsLoader.loadPath("testdocs/bindings.yaml"); } @Test diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java index ba164cd8d..687205a8e 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,7 +32,7 @@ public class StmtDetailOverrideTest { @Test public void testStmtOverrides() { - StmtsDocList doclist = StatementsLoader.loadPath(logger, "testdocs/stmt_details.yaml"); + StmtsDocList doclist = StatementsLoader.loadPath("testdocs/stmt_details.yaml"); assertThat(doclist).isNotNull(); diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java index cd5976299..ce6a50154 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,7 +50,7 @@ public class StmtsDocListTest { @BeforeAll public static void testLoadYaml() { - doclist = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml"); + doclist = StatementsLoader.loadPath("testdocs/docs_blocks_stmts.yaml"); } @Test diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java index 935e2a952..25f1a5cf0 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java @@ -38,7 +38,6 @@ import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter; import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters; import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec; import io.nosqlbench.engine.api.activityconfig.StatementsLoader; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally; @@ -668,10 +667,10 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs Optional op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload"); if (stmt.isPresent()) { workloadSource = "commandline:" + stmt.get(); - return StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams()); + return StatementsLoader.loadStmt(stmt.get(), activityDef.getParams()); } else if (op_yaml_loc.isPresent()) { workloadSource = "yaml:" + op_yaml_loc.get(); - return StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities"); + return StatementsLoader.loadPath(op_yaml_loc.get(), activityDef.getParams(), "activities"); } return StmtsDocList.none(); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java index 0b12cfa02..816c42781 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java @@ -58,7 +58,7 @@ public class StandardActivity extends SimpleActivity implements Optional yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload"); if (yaml_loc.isPresent()) { Map disposable = new LinkedHashMap<>(activityDef.getParams()); - StmtsDocList workload = StatementsLoader.loadPath(logger, yaml_loc.get(), disposable, "activities"); + StmtsDocList workload = StatementsLoader.loadPath(yaml_loc.get(), disposable, "activities"); yamlmodel = workload.getConfigModel(); } else { yamlmodel = ConfigModel.of(StandardActivity.class).asReadOnly(); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java index 55370c728..3367bfadf 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -69,7 +69,7 @@ public class StandardActivityType> extends Simpl Optional op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload"); if (op_yaml_loc.isPresent()) { Map disposable = new LinkedHashMap<>(activityDef.getParams()); - StmtsDocList workload = StatementsLoader.loadPath(logger, op_yaml_loc.get(), disposable, "activities"); + StmtsDocList workload = StatementsLoader.loadPath(op_yaml_loc.get(), disposable, "activities"); cfgModel=cfgModel.add(workload.getConfigModel()); } NBConfiguration cfg = cfgModel.apply(activityDef.getParams()); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java b/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java index bac785305..54eeffe55 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java @@ -114,7 +114,7 @@ public class NBCLIScenarioParser { .extension(RawStmtsLoader.YAML_EXTENSIONS) .first().orElseThrow(); // TODO: The yaml needs to be parsed with arguments from each command independently to support template vars - StmtsDocList scenariosYaml = StatementsLoader.loadContent(logger, yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams)); + StmtsDocList scenariosYaml = StatementsLoader.loadContent(yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams)); Scenarios scenarios = scenariosYaml.getDocScenarios(); String[] nameparts = scenarioName.split("\\.",2); @@ -319,7 +319,7 @@ public class NBCLIScenarioParser { StmtsDocList stmts = null; try { - stmts = StatementsLoader.loadContent(logger, content, Map.of()); + stmts = StatementsLoader.loadContent(content, Map.of()); if (stmts.getStmtDocs().size() == 0) { logger.warn("Encountered yaml with no docs in '" + referenced + "'"); continue; diff --git a/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java index 98bb4204a..214c0764a 100644 --- a/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java +++ b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,12 +33,12 @@ public class NBCLIScenarioParserTemplateVarTest { List cmds = opts.getCommands(); cmds.forEach(System.out::println); - StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams()); + StmtsDocList workload1 = StatementsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams()); OpTemplate optpl1 = workload1.getStmts().get(0); System.out.println("op from cmd1:"+optpl1); assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n"); - StmtsDocList workload2 = StatementsLoader.loadPath(null, cmds.get(1).getArg("workload"),cmds.get(1).getParams()); + StmtsDocList workload2 = StatementsLoader.loadPath(cmds.get(1).getArg("workload"),cmds.get(1).getParams()); OpTemplate optpl2 = workload2.getStmts().get(0); System.out.println("op from cmd2:"+optpl2); assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n"); @@ -50,7 +50,7 @@ public class NBCLIScenarioParserTemplateVarTest { List cmds = opts.getCommands(); cmds.forEach(System.out::println); - StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams()); + StmtsDocList workload1 = StatementsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams()); OpTemplate optpl1 = workload1.getStmts().get(0); System.out.println("op from cmd1:"+optpl1); assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n");