From 1784f5dcd0c02d9035a7b670ef0ffa0d7fef91fe Mon Sep 17 00:00:00 2001 From: Dave Fisher Date: Wed, 18 Dec 2024 12:01:25 -0800 Subject: [PATCH] Add Property File Support plus Related Workload Improvements (#2115) * Use Property Files for Environment Vars and as AtFiles * Keep a private property overlay over system properties * Add properties from file and use in workload templates * Allow access to properties and env vars is enabled * Reorder to ease reading the code * Insert property, yaml, and json files * Make exceptions meaningful * Catch all errors on error insert file type * Some RawOpsLoader Refactoring and other Small Fixes --- .../api/activityconfig/OpsLoader.java | 8 +- .../activityconfig/rawyaml/RawOpsLoader.java | 56 ++++++--- .../activityconfig/rawyaml/RawYamlLoader.java | 106 ------------------ .../api/templating/StrInterpolator.java | 91 +++++++++++++-- .../specifications/YamlSpecValidator.java | 4 +- .../nb/api/advisor/NBAdvisorPoint.java | 2 +- .../nb/api/config/standard/ConfigModel.java | 4 +- .../nb/api/system/NBEnvironment.java | 28 ++++- .../nosqlbench/engine/cli/NBCLIOptions.java | 30 +++++ .../engine/cli/atfiles/NBAtFile.java | 44 ++++---- .../api/activityimpl/SimpleActivity.java | 5 +- .../uniform/StandardActivity.java | 6 +- 12 files changed, 214 insertions(+), 170 deletions(-) delete mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawYamlLoader.java diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java index a4cb5ce8c..5996aabe1 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java @@ -57,6 +57,7 @@ public class OpsLoader { return loadString(content.get().toString(), fmt, params, content.getURI()); } + // used in OpsDocList (at least) public static OpsDocList loadPath(String path, Map params, String... searchPaths) { String[] extensions = path.indexOf('.') > -1 ? new String[]{} : YAML_EXTENSIONS; ResolverChain chain = new ResolverChain(path); @@ -69,18 +70,17 @@ public class OpsLoader { public static OpsDocList loadString( final String sourceData, OpTemplateFormat fmt, Map params, URI srcuri) { - logger.trace(() -> "Applying string transformer to data:" + sourceData); if (srcuri != null) { logger.info("workload URI: '" + srcuri + "'"); } StrInterpolator transformer = new StrInterpolator(params); - String data = transformer.apply(sourceData); + //String data = transformer.apply(sourceData); RawOpsLoader loader = new RawOpsLoader(transformer); RawOpsDocList rawOpsDocList = switch (fmt) { case jsonnet -> loader.loadString(evaluateJsonnet(srcuri, params)); - case yaml, json -> loader.loadString(data); - case inline, stmt -> RawOpsDocList.forSingleStatement(data); + case yaml, json -> loader.loadString(sourceData); + case inline, stmt -> RawOpsDocList.forSingleStatement(transformer.apply(sourceData)); }; // TODO: itemize inline to support ParamParser diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpsLoader.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpsLoader.java index 502e9091e..977cf7c23 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpsLoader.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpsLoader.java @@ -64,24 +64,9 @@ public class RawOpsLoader { Collections.addAll(this.transformers, newTransformer); } - public RawOpsDocList loadString(final String originalData) { - logger.trace(() -> "Applying string transformer to yaml data:" + originalData); - String data = originalData; - try { - for (Function transformer : transformers) { - data = transformer.apply(data); - } - } catch (Exception e) { - RuntimeException t = new OpConfigError("Error applying string transforms to input", e); - throw t; - } - - return parseYaml(data); - } - public RawOpsDocList loadPath( - String path, - String... searchPaths) { + String path, + String... searchPaths) { String data = null; try { @@ -94,6 +79,29 @@ public class RawOpsLoader { } + public RawOpsDocList loadString(final String originalData) { + String data = applyTransforms(originalData); + return parseYaml(data); + } + + public List> loadStringMap(final String originalData) { + String data = applyTransforms(originalData); + List> maps = parseYamlMap(data); + return maps; + } + + protected String applyTransforms(String originalData) { + String data = originalData; + for (Function transformer : transformers) { + try { + data = transformer.apply(data); + } catch (Exception e) { + throw new OpConfigError("Error applying string transforms to input", e); + } + } + return data; + } + public RawOpsDocList parseYaml(String data) { Iterable objects = yaml.loadAllFromString(data); List newDocList = new ArrayList<>(); @@ -125,4 +133,18 @@ public class RawOpsLoader { return rawOpsDocList; } + private List> parseYamlMap(String data) { + Iterable objects = yaml.loadAllFromString(data); + List> maps = new ArrayList<>(); + + for (Object object : objects) { + if (object instanceof Map) { + maps.add(new LinkedHashMap<>((Map)object)); + } else { + throw new OpConfigError("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName()); + } + } + return maps; + } + } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawYamlLoader.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawYamlLoader.java deleted file mode 100644 index a78366d76..000000000 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawYamlLoader.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2022-2023 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.adapters.api.activityconfig.rawyaml; - -import io.nosqlbench.nb.api.nbio.Content; -import io.nosqlbench.nb.api.nbio.NBIO; -import io.nosqlbench.nb.api.errors.BasicError; -import io.nosqlbench.nb.api.errors.OpConfigError; -import io.nosqlbench.adapters.api.templating.StrInterpolator; -import org.apache.logging.log4j.Logger; -import org.snakeyaml.engine.v2.api.Load; -import org.snakeyaml.engine.v2.api.LoadSettings; - -import java.util.*; -import java.util.function.Function; - -public class RawYamlLoader { - - public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"}; - - List> stringTransformers = new ArrayList<>(); - private final ArrayList> transformers = new ArrayList<>(); - - private void addTransformer(Function newTransformer) { - Collections.addAll(this.transformers, newTransformer); - } - - public RawYamlLoader() { - addTransformer(new StrInterpolator()); - } - - public List> loadString(Logger logger, String originalData) { - String data = originalData; - try { - if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + originalData); - for (Function transformer : transformers) { - data = transformer.apply(data); - } - } catch (Exception e) { - RuntimeException t = new OpConfigError("Error applying string transforms to input", e); - throw t; - } - - return parseYaml(logger, data); - } - - public List> loadPath( - Logger logger, - String path, - String... searchPaths) { - - String data = null; - try { - Optional> oyaml = NBIO.all().searchPrefixes(searchPaths).pathname(path).extensionSet(YAML_EXTENSIONS).first(); - data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path)); - return loadString(logger, data); - } catch (Exception e) { - throw new RuntimeException("error while reading file " + path, e); - } - } - - private List> parseYaml(Logger logger, String data) { - Load yaml = new Load(LoadSettings.builder().build()); - Iterable objects = yaml.loadAllFromString(data); - List newDocList = new ArrayList<>(); - - List> maps = new ArrayList<>(); - - for (Object object : objects) { - if (object instanceof Map) { - maps.add(new LinkedHashMap<>((Map)object)); - } else { - throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName()); - } - } - return maps; - } - - protected String applyTransforms(Logger logger, String data) { - for (Function xform : stringTransformers) { - try { - if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + xform); - data = xform.apply(data); - } catch (Exception e) { - RuntimeException t = new OpConfigError("Error applying string transforms to input", e); - throw t; - } - } - return data; - } - -} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java index 6a1bb726d..6ee581822 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java @@ -16,19 +16,30 @@ package io.nosqlbench.adapters.api.templating; +import com.google.gson.Gson; import io.nosqlbench.nb.api.advisor.NBAdvisorBuilder; import io.nosqlbench.nb.api.advisor.NBAdvisorPoint; import io.nosqlbench.nb.api.advisor.conditions.Conditions; import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef; +import io.nosqlbench.nb.api.errors.OpConfigError; +import io.nosqlbench.nb.api.nbio.Content; +import io.nosqlbench.nb.api.nbio.NBIO; +import io.nosqlbench.nb.api.nbio.ResolverChain; +import io.nosqlbench.nb.api.system.NBEnvironment; import org.apache.commons.text.StrLookup; import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.yaml.snakeyaml.Yaml; +import java.io.*; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.*; import java.util.function.Function; +import static java.util.stream.Collectors.*; + public class StrInterpolator implements Function { private final static Logger logger = LogManager.getLogger(StrInterpolator.class); private final NBAdvisorBuilder advisorBuilder = new NBAdvisorBuilder(); @@ -40,6 +51,7 @@ public class StrInterpolator implements Function { .setEnableUndefinedVariableException(true) .setDisableSubstitutionInValues(true); private final Pattern COMMENT = Pattern.compile("^\\s*#.*"); + private final Pattern INSERT = Pattern.compile("^(\\s*)INSERT:\\s+(.+)$"); public StrInterpolator(ActivityDef... activityDefs) { Arrays.stream(activityDefs) @@ -62,36 +74,84 @@ public class StrInterpolator implements Function { @Override public String apply(String raw) { + logger.debug(() -> "Applying string transformer to data:\n" + raw); advisor = advisorBuilder.build(); advisor.add(Conditions.DeprecatedWarning); - String[] lines = raw.split("\\R"); + List lines = new LinkedList<>(Arrays.asList(raw.split("\\R"))); boolean endsWithNewline = raw.endsWith("\n"); int i = 0; - for (String line : lines) { + while (i < lines.size()) { + String line = lines.get(i); if (!isComment(line)) { String result = matchTemplates(line); if (!result.equals(line)) { - lines[i] = result; + lines.set(i, result); + line = result; + } + Matcher matcher = INSERT.matcher(line); + if (matcher.matches()) { + String leadingSpaces = matcher.group(1); + String filePath = matcher.group(2); + List includes = insertContentFromFile(leadingSpaces, filePath); + System.out.println(leadingSpaces + "INSERT: " + filePath); + lines.remove(i); + lines.addAll(i, includes); + i--; } } i++; } - String results = String.join(System.lineSeparator(), lines); + String results = lines.stream().collect(joining(System.lineSeparator())); if (endsWithNewline) { results += System.lineSeparator(); } advisor.setName("Workload", "Deprecated template format").logName().evaluate(); + String finalResults = results; + logger.debug(() -> "Results of applying string transformer:\n" + finalResults); return results; } - public Map checkpointAccesses() { - return multimap.checkpointAccesses(); - } + private LinkedList insertContentFromFile(String leadingSpaces, String filePath) { + // Determine file type and process the inclusion + LinkedList result = new LinkedList<>(); + result.add(leadingSpaces + "# INSERT: " + filePath); + try { + ResolverChain chain = new ResolverChain(filePath); + Content insert = NBIO.chain(chain.getChain()).searchPrefixes("activities") + .pathname(chain.getPath()).first() + .orElseThrow(() -> new RuntimeException("Unable to load path '" + filePath + "'")); + BufferedReader reader = new BufferedReader(new StringReader(insert.asString())); - public LinkedHashMap getTemplateDetails(String input) { - LinkedHashMap details = new LinkedHashMap<>(); - - return details; + if (filePath.endsWith(".properties")) { + // Include properties file + Properties properties = new Properties(); + properties.load(reader); + for (String key : properties.stringPropertyNames()) { + result.add(leadingSpaces + key + ": " + properties.getProperty(key)); + } + } else if (filePath.endsWith(".json")) { + // Include JSON + Gson gson = new Gson(); + Map jsonMap = gson.fromJson(reader, Map.class); + Yaml yaml = new Yaml(); + String yamlString = yaml.dumpAsMap(jsonMap); + LinkedList include = new LinkedList<>(Arrays.asList(yamlString.split("\\R"))); + int j = 0; + while (j < include.size()) { + result.add(leadingSpaces + include.get(j)); + j++; + } + } else { + // Include as a YAML file (if it is not then if a bad OpDocList is created it will fail. + String line; + while ((line = reader.readLine()) != null) { + result.add(leadingSpaces + line); + } + } + } catch (Exception e) { + throw new OpConfigError("While processing file '" + filePath + "' " + e.getMessage()); + } + return result; } public String matchTemplates(String original) { @@ -177,6 +237,10 @@ public class StrInterpolator implements Function { return line; } + public Map checkpointAccesses() { + return multimap.checkpointAccesses(); + } + public static class MultiMap extends StrLookup { private final String warnPrefix = "UNSET"; @@ -232,13 +296,18 @@ public class StrInterpolator implements Function { value = val.toString(); //System.out.println("for: '"+original+"': "+key+"->"+value); } else { + boolean check_env = true; for (Map map : maps) { val = map.get(key); if (val != null) { value = val.toString(); + check_env = false; break; } } + if (check_env && NBEnvironment.INSTANCE.hasPropertyLayer() && NBEnvironment.INSTANCE.containsKey(key) ) { + value = NBEnvironment.INSTANCE.get(key); + } } value = (value==null? extractedDefaults.get(key) : value); } diff --git a/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/activityconfig/specifications/YamlSpecValidator.java b/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/activityconfig/specifications/YamlSpecValidator.java index cc8190819..49b048399 100644 --- a/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/activityconfig/specifications/YamlSpecValidator.java +++ b/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/activityconfig/specifications/YamlSpecValidator.java @@ -19,7 +19,7 @@ package io.nosqlbench.adapters.api.activityconfig.specifications; import com.google.gson.*; import com.google.gson.reflect.TypeToken; import io.nosqlbench.adapters.api.activityconfig.OpsLoader; -import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawYamlLoader; +import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsLoader; import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate; import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplateFormat; import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList; @@ -145,7 +145,7 @@ public class YamlSpecValidator implements STAssemblyValidator { System.out.format("%-40s", "- checking yaml->json"); try { - List> docmaps = new RawYamlLoader().loadString(logger, yaml); + List> docmaps = new RawOpsLoader().loadStringMap(yaml); JsonElement elem = null; try { elem = JsonParser.parseString(json); diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorPoint.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorPoint.java index 2b5b63dde..918f82620 100644 --- a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorPoint.java +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorPoint.java @@ -98,7 +98,7 @@ public class NBAdvisorPoint extends NBAdvisorPointOrBuilder { } public NBAdvisorPoint logName() { - if (resultLog.size() > 0) { + if (resultLog.size() > 0 && advisorLevel != NBAdvisorLevel.none) { logger.info("Advisor: " + name + ": " + description); } return this; diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigModel.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigModel.java index b0ff0a6e5..0da992d95 100644 --- a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigModel.java +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigModel.java @@ -225,7 +225,7 @@ public class ConfigModel implements NBConfigModel { for (String configkey : config.keySet()) { Param element = this.paramsByName.get(configkey); if (element != null) { - String warning = "Config parameter '" + configkey + "' is also a " + type + ". Check for possible conflicts."; + String warning = "Config parameter '" + configkey + "' is also a " + type + ". Check for possible conflicts.\n"; NBAdvisorOutput.output(Level.WARN, warning); } } @@ -299,7 +299,7 @@ public class ConfigModel implements NBConfigModel { for (String configkey : config.keySet()) { Param element = this.paramsByName.get(configkey); String warning = "Unknown config parameter '" + configkey + "' in config model while configuring " + getOf().getSimpleName() - + ", possible parameter names are " + this.paramsByName.keySet() + "."; + + ", possible parameter names are " + this.paramsByName.keySet() + ".\n"; if (element == null) { String warnonly = System.getenv("NB_CONFIG_WARNINGS_ONLY"); logger.warn("WARNING: " + warning); diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/system/NBEnvironment.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/system/NBEnvironment.java index 6325c0a4f..32d13bbc8 100644 --- a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/system/NBEnvironment.java +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/system/NBEnvironment.java @@ -65,6 +65,10 @@ public class NBEnvironment { private final LinkedHashMap references = new LinkedHashMap<>(); + private final Map properties = new HashMap<>(); + + private boolean hasPropertyLayer = false; + /** * These properties are well-defined in the Java specs. This map redirects common * environment variable names to the given system property. This allows @@ -101,7 +105,9 @@ public class NBEnvironment { } } - System.setProperty(propname, value); + // Instead of using System properties we keep an internal overlay. + properties.put(propname, value); + //System.setProperty(propname, value); } /** @@ -153,6 +159,10 @@ public class NBEnvironment { } } if (name.contains(".")) { + value = properties.get(name.toLowerCase()); + if (value != null) { + return value; + } value = System.getProperty(name.toLowerCase()); if (value != null) { return value; @@ -163,11 +173,19 @@ public class NBEnvironment { if (logger != null) { logger.debug("redirecting env var '" + name + "' to upper-case property '" + propName + "'"); } + value = properties.get(propName); + if (value != null) { + return value; + } value = System.getProperty(propName); if (value != null) { return value; } } + value = properties.get(name); + if (value != null) { + return value; + } value = System.getProperty(name); if (value != null) { return value; @@ -203,6 +221,14 @@ public class NBEnvironment { return (value != null); } + public boolean hasPropertyLayer() { + return hasPropertyLayer; + } + + public void setPropertyLayer() { + hasPropertyLayer = true; + } + /** * For the given word, if it contains a pattern with '$' followed by alpha, followed * by alphanumeric and underscores, replace this pattern with the system property or diff --git a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java index 936ded70b..49f707a29 100644 --- a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java +++ b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java @@ -28,12 +28,16 @@ import io.nosqlbench.nb.api.labels.NBLabelSpec; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.logging.NBLogLevel; import io.nosqlbench.nb.api.metadata.SystemId; +import io.nosqlbench.nb.api.system.NBEnvironment; import io.nosqlbench.nb.api.system.NBStatePath; import io.nosqlbench.engine.api.metrics.IndicatorMode; import io.nosqlbench.engine.cmdstream.CmdType; import io.nosqlbench.nb.annotations.Maturity; import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; import java.nio.file.Path; import java.security.InvalidParameterException; import java.util.*; @@ -51,6 +55,8 @@ public class NBCLIOptions { private static final String NB_STATE_DIR = "--statedir"; public static final String ARGS_FILE_DEFAULT = "$NBSTATEDIR/argsfile"; private static final String INCLUDE = "--include"; + private static final String PROPERTIES = "--properties"; + private static final String ENVIRONMENT = "--environment"; private static final String userHome = System.getProperty("user.home"); @@ -415,6 +421,15 @@ public class NBCLIOptions { arglist.removeFirst(); annotatorsConfig = this.readWordOrThrow(arglist, "annotators config"); break; + case NBCLIOptions.PROPERTIES: + arglist.removeFirst(); + final String properties = this.readWordOrThrow(arglist, "path to properties file"); + this.setProperties(properties); + break; + case NBCLIOptions.ENVIRONMENT: + arglist.removeFirst(); + NBEnvironment.INSTANCE.setPropertyLayer(); + break; case NBCLIOptions.REPORT_GRAPHITE_TO: arglist.removeFirst(); this.reportGraphiteTo = arglist.removeFirst(); @@ -774,6 +789,21 @@ public class NBCLIOptions { return this.wantsToIncludePaths.toArray(new String[0]); } + private void setProperties(String filePath) { + Properties properties = new Properties(); + try (FileReader reader = new FileReader(filePath)) { + properties.load(reader); + } catch (FileNotFoundException e) { + throw new BasicError("Property File '"+filePath+"' was not found.", e); + } catch (IOException e) { + throw new BasicError("IO Exception Loading Property File '"+filePath+"'.", e); + } + for (String key : properties.stringPropertyNames()) { + NBEnvironment.INSTANCE.put(key, properties.getProperty(key)); + } + NBEnvironment.INSTANCE.setPropertyLayer(); + } + private Map parseLogLevelOverrides(final String levelsSpec) { final Map levels = new HashMap<>(); Arrays.stream(levelsSpec.split("[,;]")).forEach(kp -> { diff --git a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java index 64e20cfe9..ebea9723d 100644 --- a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java +++ b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java @@ -16,16 +16,12 @@ package io.nosqlbench.engine.cli.atfiles; -import io.nosqlbench.nb.api.nbio.Content; -import io.nosqlbench.nb.api.nbio.NBIO; -import io.nosqlbench.nb.api.nbio.NBPathsAPI; import io.nosqlbench.nb.api.system.NBEnvironment; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.snakeyaml.engine.v2.api.Load; import org.snakeyaml.engine.v2.api.LoadSettings; +import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -38,7 +34,6 @@ import java.util.regex.Pattern; * TODO:ERRORHANDLER Cannot invoke "Object.getClass()" because "scopeOfInclude" is null on file full of comments only */ public class NBAtFile { -// private final static Logger logger = LogManager.getLogger(NBAtFile.class); /** * This will take a command line in raw form, which may include some arguments @@ -63,7 +58,6 @@ public class NBAtFile { * @throws RuntimeException for any errors finding, traversing, parsing, or rendering values */ public static LinkedList includeAt(LinkedList processInPlace) { -// logger.trace("argv stream before processing: " + String.join("|",processInPlace)); ListIterator iter = processInPlace.listIterator(); while (iter.hasNext()) { String spec = iter.next(); @@ -76,8 +70,6 @@ public class NBAtFile { } } } -// logger.trace("argv stream after atfile processing: "+ String.join("|",processInPlace)); - return processInPlace; } private final static Pattern includePattern = @@ -121,26 +113,36 @@ public class NBAtFile { String[] datapath = (dataPathSpec!=null && !dataPathSpec.isBlank()) ? dataPathSpec.split("(/|\\.)") : new String[] {}; String filename = Path.of(filepathSpec).getFileName().toString(); - if (filename.contains(".") && !(filename.toLowerCase().endsWith("yaml"))) { - throw new RuntimeException("Only the yaml format and extension is supported for at-files." + + if (filename.contains(".") && !( filename.toLowerCase().endsWith("yaml") || filename.toLowerCase().endsWith("properties") )) { + throw new RuntimeException("Only the yaml and properties format and extension are supported for at-files." + " You specified " + filepathSpec); } - filepathSpec=(filepathSpec.endsWith(".yaml") ? filepathSpec : filepathSpec+".yaml"); + filepathSpec=(filepathSpec.endsWith(".yaml") ? filepathSpec : (filepathSpec.endsWith(".properties") ? filepathSpec : filepathSpec+".yaml")); Path atPath = Path.of(filepathSpec); - String argsdata = ""; - try { - argsdata = Files.readString(atPath); - } catch (IOException e) { - throw new RuntimeException(e); - } - NBAtFileFormats fmt = (formatSpec!=null) ? NBAtFileFormats.valueOfSymbol(formatSpec) : NBAtFileFormats.Default; Object scopeOfInclude = null; try { - Load yaml = new Load(LoadSettings.builder().build()); - scopeOfInclude= yaml.loadFromString(argsdata); + if (filepathSpec.endsWith(".properties")) { + FileInputStream inputStream = new FileInputStream(atPath.toFile()); + Properties props = new Properties(); + props.load(inputStream); + Map propsMap = new HashMap<>(); + for (String key : props.stringPropertyNames()) { + propsMap.put(key, props.getProperty(key)); + } + scopeOfInclude = propsMap; + } else { + Load yaml = new Load(LoadSettings.builder().build()); + String argsdata = ""; + try { + argsdata = Files.readString(atPath); + } catch (IOException e) { + throw new RuntimeException(e); + } + scopeOfInclude = yaml.loadFromString(argsdata); + } } catch (Exception e) { throw new RuntimeException(e); } diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java index 9d3afa552..e656e9455 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java @@ -585,8 +585,7 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok if ((op != null ? 1 : 0) + (stmt != null ? 1 : 0) + (workload != null ? 1 : 0) > 1) { throw new OpConfigError("Only op, statement, or workload may be provided, not more than one."); } - - + logger.debug("loadStmtsDocList #1"); if (workload != null && OpsLoader.isJson(workload)) { workloadSource = "commandline: (workload/json):" + workload; return OpsLoader.loadString(workload, OpTemplateFormat.json, activityDef.getParams(), null); @@ -597,11 +596,13 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok return OpsLoader.loadPath(workload, activityDef.getParams(), "activities"); } + logger.debug("loadStmtsDocList #2"); if (stmt != null) { workloadSource = "commandline: (stmt/inline): '" + stmt + "'"; return OpsLoader.loadString(stmt, OpTemplateFormat.inline, activityDef.getParams(), null); } + logger.debug("loadStmtsDocList #3"); if (op != null && OpsLoader.isJson(op)) { workloadSource = "commandline: (op/json): '" + op + "'"; return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null); diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java index 1abe910ee..8258fa2fb 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java @@ -95,7 +95,7 @@ public class StandardActivity exte List opTemplates = loadOpTemplates(defaultAdapter); NBConfigModel combinedAdapterModel = ConfigModel.of(StandardActivity.class); for (OpTemplate ot : opTemplates) { - logger.info(() -> "StandardActivity.opTemplate = "+ot); + logger.debug(() -> "StandardActivity.opTemplate = "+ot); String driverName = ot.getOptionalStringParam("driver", String.class) .or(() -> ot.getOptionalStringParam("type", String.class)) .orElse(defaultDriverName); @@ -127,11 +127,11 @@ public class StandardActivity exte paramsAdvisor.validateAll(ot.getBindings().keySet()); adapterlist.add(adapter); ParsedOp pop = new ParsedOp(ot, adapter.getConfiguration(), List.of(adapter.getPreprocessor()), this); - logger.info("StandardActivity.pop="+pop); + logger.debug("StandardActivity.pop="+pop); Optional discard = pop.takeOptionalStaticValue("driver", String.class); pops.add(pop); } - logger.info(() -> "StandardActivity.opTemplate loop complete"); + logger.debug(() -> "StandardActivity.opTemplate loop complete"); paramsAdvisor.setName("Workload", "Check parameters, template, and binding names") .logName().evaluate();