IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add Property File Support plus Related Workload Improvements (#2115)

Browse Source 
* Use Property Files for Environment Vars and as AtFiles

* Keep a private property overlay over system properties

* Add properties from file and use in workload templates

* Allow access to properties and env vars is enabled

* Reorder to ease reading the code

* Insert property, yaml, and json files

* Make exceptions meaningful

* Catch all errors on error insert file type

* Some RawOpsLoader Refactoring and other Small Fixes
This commit is contained in:
Dave Fisher 2024-12-18 12:01:25 -08:00 committed by GitHub
parent 79e2a4bf6e
commit 1784f5dcd0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 214 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java
View File

@ -57,6 +57,7 @@ public class OpsLoader {
return loadString(content.get().toString(), fmt, params, content.getURI()); return loadString(content.get().toString(), fmt, params, content.getURI());
} }
// used in OpsDocList (at least)
public static OpsDocList loadPath(String path, Map<String, ?> params, String... searchPaths) { public static OpsDocList loadPath(String path, Map<String, ?> params, String... searchPaths) {
String[] extensions = path.indexOf('.') > -1 ? new String[]{} : YAML_EXTENSIONS; String[] extensions = path.indexOf('.') > -1 ? new String[]{} : YAML_EXTENSIONS;
ResolverChain chain = new ResolverChain(path); ResolverChain chain = new ResolverChain(path);
@ -69,18 +70,17 @@ public class OpsLoader {
public static OpsDocList loadString( public static OpsDocList loadString(
final String sourceData, OpTemplateFormat fmt, Map<String, ?> params, URI srcuri) { final String sourceData, OpTemplateFormat fmt, Map<String, ?> params, URI srcuri) {
logger.trace(() -> "Applying string transformer to data:" + sourceData);
if (srcuri != null) { if (srcuri != null) {
logger.info("workload URI: '" + srcuri + "'"); logger.info("workload URI: '" + srcuri + "'");
} }
StrInterpolator transformer = new StrInterpolator(params); StrInterpolator transformer = new StrInterpolator(params);
String data = transformer.apply(sourceData); //String data = transformer.apply(sourceData);
RawOpsLoader loader = new RawOpsLoader(transformer); RawOpsLoader loader = new RawOpsLoader(transformer);
RawOpsDocList rawOpsDocList = switch (fmt) { RawOpsDocList rawOpsDocList = switch (fmt) {
case jsonnet -> loader.loadString(evaluateJsonnet(srcuri, params)); case jsonnet -> loader.loadString(evaluateJsonnet(srcuri, params));
case yaml, json -> loader.loadString(data); case yaml, json -> loader.loadString(sourceData);
case inline, stmt -> RawOpsDocList.forSingleStatement(data); case inline, stmt -> RawOpsDocList.forSingleStatement(transformer.apply(sourceData));
}; };
// TODO: itemize inline to support ParamParser // TODO: itemize inline to support ParamParser

52
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpsLoader.java
View File

@ -64,21 +64,6 @@ public class RawOpsLoader {
Collections.addAll(this.transformers, newTransformer); Collections.addAll(this.transformers, newTransformer);
} }
public RawOpsDocList loadString(final String originalData) {
logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
String data = originalData;
try {
for (Function<String, String> transformer : transformers) {
data = transformer.apply(data);
}
} catch (Exception e) {
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
throw t;
}
return parseYaml(data);
}
public RawOpsDocList loadPath( public RawOpsDocList loadPath(
String path, String path,
String... searchPaths) { String... searchPaths) {
@ -94,6 +79,29 @@ public class RawOpsLoader {
} }
public RawOpsDocList loadString(final String originalData) {
String data = applyTransforms(originalData);
return parseYaml(data);
}
public List<Map<String,Object>> loadStringMap(final String originalData) {
String data = applyTransforms(originalData);
List<Map<String,Object>> maps = parseYamlMap(data);
return maps;
}
protected String applyTransforms(String originalData) {
String data = originalData;
for (Function<String, String> transformer : transformers) {
try {
data = transformer.apply(data);
} catch (Exception e) {
throw new OpConfigError("Error applying string transforms to input", e);
}
}
return data;
}
public RawOpsDocList parseYaml(String data) { public RawOpsDocList parseYaml(String data) {
Iterable<Object> objects = yaml.loadAllFromString(data); Iterable<Object> objects = yaml.loadAllFromString(data);
List<RawOpsDoc> newDocList = new ArrayList<>(); List<RawOpsDoc> newDocList = new ArrayList<>();
@ -125,4 +133,18 @@ public class RawOpsLoader {
return rawOpsDocList; return rawOpsDocList;
} }
private List<Map<String,Object>> parseYamlMap(String data) {
Iterable<Object> objects = yaml.loadAllFromString(data);
List<Map<String,Object>> maps = new ArrayList<>();
for (Object object : objects) {
if (object instanceof Map) {
maps.add(new LinkedHashMap<>((Map<String,Object>)object));
} else {
throw new OpConfigError("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
}
}
return maps;
}
} }

106
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawYamlLoader.java
View File

@ -1,106 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapters.api.activityconfig.rawyaml;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.nb.api.errors.OpConfigError;
import io.nosqlbench.adapters.api.templating.StrInterpolator;
import org.apache.logging.log4j.Logger;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
import java.util.*;
import java.util.function.Function;
public class RawYamlLoader {
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
List<Function<String, String>> stringTransformers = new ArrayList<>();
private final ArrayList<Function<String,String>> transformers = new ArrayList<>();
private void addTransformer(Function<String, String> newTransformer) {
Collections.addAll(this.transformers, newTransformer);
}
public RawYamlLoader() {
addTransformer(new StrInterpolator());
}
public List<Map<String,Object>> loadString(Logger logger, String originalData) {
String data = originalData;
try {
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
for (Function<String, String> transformer : transformers) {
data = transformer.apply(data);
}
} catch (Exception e) {
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
throw t;
}
return parseYaml(logger, data);
}
public List<Map<String,Object>> loadPath(
Logger logger,
String path,
String... searchPaths) {
String data = null;
try {
Optional<Content<?>> oyaml = NBIO.all().searchPrefixes(searchPaths).pathname(path).extensionSet(YAML_EXTENSIONS).first();
data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(logger, data);
} catch (Exception e) {
throw new RuntimeException("error while reading file " + path, e);
}
}
private List<Map<String,Object>> parseYaml(Logger logger, String data) {
Load yaml = new Load(LoadSettings.builder().build());
Iterable<Object> objects = yaml.loadAllFromString(data);
List<RawOpsDoc> newDocList = new ArrayList<>();
List<Map<String,Object>> maps = new ArrayList<>();
for (Object object : objects) {
if (object instanceof Map) {
maps.add(new LinkedHashMap<>((Map<String,Object>)object));
} else {
throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
}
}
return maps;
}
protected String applyTransforms(Logger logger, String data) {
for (Function<String, String> xform : stringTransformers) {
try {
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + xform);
data = xform.apply(data);
} catch (Exception e) {
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
throw t;
}
}
return data;
}
}

91
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java
View File

@ -16,19 +16,30 @@
package io.nosqlbench.adapters.api.templating; package io.nosqlbench.adapters.api.templating;
import com.google.gson.Gson;
import io.nosqlbench.nb.api.advisor.NBAdvisorBuilder; import io.nosqlbench.nb.api.advisor.NBAdvisorBuilder;
import io.nosqlbench.nb.api.advisor.NBAdvisorPoint; import io.nosqlbench.nb.api.advisor.NBAdvisorPoint;
import io.nosqlbench.nb.api.advisor.conditions.Conditions; import io.nosqlbench.nb.api.advisor.conditions.Conditions;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef; import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.errors.OpConfigError;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.nbio.ResolverChain;
import io.nosqlbench.nb.api.system.NBEnvironment;
import org.apache.commons.text.StrLookup; import org.apache.commons.text.StrLookup;
import org.apache.commons.text.StringSubstitutor; import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.yaml.snakeyaml.Yaml;
import java.io.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.*; import java.util.*;
import java.util.function.Function; import java.util.function.Function;
import static java.util.stream.Collectors.*;
public class StrInterpolator implements Function<String, String> { public class StrInterpolator implements Function<String, String> {
private final static Logger logger = LogManager.getLogger(StrInterpolator.class); private final static Logger logger = LogManager.getLogger(StrInterpolator.class);
private final NBAdvisorBuilder<String> advisorBuilder = new NBAdvisorBuilder<String>(); private final NBAdvisorBuilder<String> advisorBuilder = new NBAdvisorBuilder<String>();
@ -40,6 +51,7 @@ public class StrInterpolator implements Function<String, String> {
.setEnableUndefinedVariableException(true) .setEnableUndefinedVariableException(true)
.setDisableSubstitutionInValues(true); .setDisableSubstitutionInValues(true);
private final Pattern COMMENT = Pattern.compile("^\\s*#.*"); private final Pattern COMMENT = Pattern.compile("^\\s*#.*");
private final Pattern INSERT = Pattern.compile("^(\\s*)INSERT:\\s+(.+)$");
public StrInterpolator(ActivityDef... activityDefs) { public StrInterpolator(ActivityDef... activityDefs) {
Arrays.stream(activityDefs) Arrays.stream(activityDefs)
@ -62,36 +74,84 @@ public class StrInterpolator implements Function<String, String> {
@Override @Override
public String apply(String raw) { public String apply(String raw) {
logger.debug(() -> "Applying string transformer to data:\n" + raw);
advisor = advisorBuilder.build(); advisor = advisorBuilder.build();
advisor.add(Conditions.DeprecatedWarning); advisor.add(Conditions.DeprecatedWarning);
String[] lines = raw.split("\\R"); List<String> lines = new LinkedList<>(Arrays.asList(raw.split("\\R")));
boolean endsWithNewline = raw.endsWith("\n"); boolean endsWithNewline = raw.endsWith("\n");
int i = 0; int i = 0;
for (String line : lines) { while (i < lines.size()) {
String line = lines.get(i);
if (!isComment(line)) { if (!isComment(line)) {
String result = matchTemplates(line); String result = matchTemplates(line);
if (!result.equals(line)) { if (!result.equals(line)) {
lines[i] = result; lines.set(i, result);
line = result;
}
Matcher matcher = INSERT.matcher(line);
if (matcher.matches()) {
String leadingSpaces = matcher.group(1);
String filePath = matcher.group(2);
List<String> includes = insertContentFromFile(leadingSpaces, filePath);
System.out.println(leadingSpaces + "INSERT: " + filePath);
lines.remove(i);
lines.addAll(i, includes);
i--;
} }
} }
i++; i++;
} }
String results = String.join(System.lineSeparator(), lines); String results = lines.stream().collect(joining(System.lineSeparator()));
if (endsWithNewline) { if (endsWithNewline) {
results += System.lineSeparator(); results += System.lineSeparator();
} }
advisor.setName("Workload", "Deprecated template format").logName().evaluate(); advisor.setName("Workload", "Deprecated template format").logName().evaluate();
String finalResults = results;
logger.debug(() -> "Results of applying string transformer:\n" + finalResults);
return results; return results;
} }
public Map<String,String> checkpointAccesses() { private LinkedList<String> insertContentFromFile(String leadingSpaces, String filePath) {
return multimap.checkpointAccesses(); // Determine file type and process the inclusion
LinkedList<String> result = new LinkedList<>();
result.add(leadingSpaces + "# INSERT: " + filePath);
try {
ResolverChain chain = new ResolverChain(filePath);
Content<?> insert = NBIO.chain(chain.getChain()).searchPrefixes("activities")
.pathname(chain.getPath()).first()
.orElseThrow(() -> new RuntimeException("Unable to load path '" + filePath + "'"));
BufferedReader reader = new BufferedReader(new StringReader(insert.asString()));
if (filePath.endsWith(".properties")) {
// Include properties file
Properties properties = new Properties();
properties.load(reader);
for (String key : properties.stringPropertyNames()) {
result.add(leadingSpaces + key + ": " + properties.getProperty(key));
} }
} else if (filePath.endsWith(".json")) {
public LinkedHashMap<String, String> getTemplateDetails(String input) { // Include JSON
LinkedHashMap<String, String> details = new LinkedHashMap<>(); Gson gson = new Gson();
Map<String, Object> jsonMap = gson.fromJson(reader, Map.class);
return details; Yaml yaml = new Yaml();
String yamlString = yaml.dumpAsMap(jsonMap);
LinkedList<String> include = new LinkedList<>(Arrays.asList(yamlString.split("\\R")));
int j = 0;
while (j < include.size()) {
result.add(leadingSpaces + include.get(j));
j++;
}
} else {
// Include as a YAML file (if it is not then if a bad OpDocList is created it will fail.
String line;
while ((line = reader.readLine()) != null) {
result.add(leadingSpaces + line);
}
}
} catch (Exception e) {
throw new OpConfigError("While processing file '" + filePath + "' " + e.getMessage());
}
return result;
} }
public String matchTemplates(String original) { public String matchTemplates(String original) {
@ -177,6 +237,10 @@ public class StrInterpolator implements Function<String, String> {
return line; return line;
} }
public Map<String,String> checkpointAccesses() {
return multimap.checkpointAccesses();
}
public static class MultiMap extends StrLookup<String> { public static class MultiMap extends StrLookup<String> {
private final String warnPrefix = "UNSET"; private final String warnPrefix = "UNSET";
@ -232,13 +296,18 @@ public class StrInterpolator implements Function<String, String> {
value = val.toString(); value = val.toString();
//System.out.println("for: '"+original+"': "+key+"->"+value); //System.out.println("for: '"+original+"': "+key+"->"+value);
} else { } else {
boolean check_env = true;
for (Map<String, ?> map : maps) { for (Map<String, ?> map : maps) {
val = map.get(key); val = map.get(key);
if (val != null) { if (val != null) {
value = val.toString(); value = val.toString();
check_env = false;
break; break;
} }
} }
if (check_env && NBEnvironment.INSTANCE.hasPropertyLayer() && NBEnvironment.INSTANCE.containsKey(key) ) {
value = NBEnvironment.INSTANCE.get(key);
}
} }
value = (value==null? extractedDefaults.get(key) : value); value = (value==null? extractedDefaults.get(key) : value);
} }

4
nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/activityconfig/specifications/YamlSpecValidator.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.adapters.api.activityconfig.specifications;
import com.google.gson.*; import com.google.gson.*;
import com.google.gson.reflect.TypeToken; import com.google.gson.reflect.TypeToken;
import io.nosqlbench.adapters.api.activityconfig.OpsLoader; import io.nosqlbench.adapters.api.activityconfig.OpsLoader;
import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawYamlLoader; import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate; import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplateFormat; import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList;
@ -145,7 +145,7 @@ public class YamlSpecValidator implements STAssemblyValidator {
System.out.format("%-40s", "- checking yaml->json"); System.out.format("%-40s", "- checking yaml->json");
try { try {
List<Map<String, Object>> docmaps = new RawYamlLoader().loadString(logger, yaml); List<Map<String, Object>> docmaps = new RawOpsLoader().loadStringMap(yaml);
JsonElement elem = null; JsonElement elem = null;
try { try {
elem = JsonParser.parseString(json); elem = JsonParser.parseString(json);

2
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorPoint.java
View File

@ -98,7 +98,7 @@ public class NBAdvisorPoint<T> extends NBAdvisorPointOrBuilder<T> {
} }
public NBAdvisorPoint<T> logName() { public NBAdvisorPoint<T> logName() {
if (resultLog.size() > 0) { if (resultLog.size() > 0 && advisorLevel != NBAdvisorLevel.none) {
logger.info("Advisor: " + name + ": " + description); logger.info("Advisor: " + name + ": " + description);
} }
return this; return this;

4
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigModel.java
View File

@ -225,7 +225,7 @@ public class ConfigModel implements NBConfigModel {
for (String configkey : config.keySet()) { for (String configkey : config.keySet()) {
Param<?> element = this.paramsByName.get(configkey); Param<?> element = this.paramsByName.get(configkey);
if (element != null) { if (element != null) {
String warning = "Config parameter '" + configkey + "' is also a " + type + ". Check for possible conflicts."; String warning = "Config parameter '" + configkey + "' is also a " + type + ". Check for possible conflicts.\n";
NBAdvisorOutput.output(Level.WARN, warning); NBAdvisorOutput.output(Level.WARN, warning);
} }
} }
@ -299,7 +299,7 @@ public class ConfigModel implements NBConfigModel {
for (String configkey : config.keySet()) { for (String configkey : config.keySet()) {
Param<?> element = this.paramsByName.get(configkey); Param<?> element = this.paramsByName.get(configkey);
String warning = "Unknown config parameter '" + configkey + "' in config model while configuring " + getOf().getSimpleName() String warning = "Unknown config parameter '" + configkey + "' in config model while configuring " + getOf().getSimpleName()
+ ", possible parameter names are " + this.paramsByName.keySet() + "."; + ", possible parameter names are " + this.paramsByName.keySet() + ".\n";
if (element == null) { if (element == null) {
String warnonly = System.getenv("NB_CONFIG_WARNINGS_ONLY"); String warnonly = System.getenv("NB_CONFIG_WARNINGS_ONLY");
logger.warn("WARNING: " + warning); logger.warn("WARNING: " + warning);

28
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/system/NBEnvironment.java
View File

@ -65,6 +65,10 @@ public class NBEnvironment {
private final LinkedHashMap<String, String> references = new LinkedHashMap<>(); private final LinkedHashMap<String, String> references = new LinkedHashMap<>();
private final Map<String,String> properties = new HashMap<>();
private boolean hasPropertyLayer = false;
/** /**
* These properties are well-defined in the Java specs. This map redirects common * These properties are well-defined in the Java specs. This map redirects common
* environment variable names to the given system property. This allows * environment variable names to the given system property. This allows
@ -101,7 +105,9 @@ public class NBEnvironment {
} }
} }
System.setProperty(propname, value); // Instead of using System properties we keep an internal overlay.
properties.put(propname, value);
//System.setProperty(propname, value);
} }
/** /**
@ -153,6 +159,10 @@ public class NBEnvironment {
} }
} }
if (name.contains(".")) { if (name.contains(".")) {
value = properties.get(name.toLowerCase());
if (value != null) {
return value;
}
value = System.getProperty(name.toLowerCase()); value = System.getProperty(name.toLowerCase());
if (value != null) { if (value != null) {
return value; return value;
@ -163,11 +173,19 @@ public class NBEnvironment {
if (logger != null) { if (logger != null) {
logger.debug("redirecting env var '" + name + "' to upper-case property '" + propName + "'"); logger.debug("redirecting env var '" + name + "' to upper-case property '" + propName + "'");
} }
value = properties.get(propName);
if (value != null) {
return value;
}
value = System.getProperty(propName); value = System.getProperty(propName);
if (value != null) { if (value != null) {
return value; return value;
} }
} }
value = properties.get(name);
if (value != null) {
return value;
}
value = System.getProperty(name); value = System.getProperty(name);
if (value != null) { if (value != null) {
return value; return value;
@ -203,6 +221,14 @@ public class NBEnvironment {
return (value != null); return (value != null);
} }
public boolean hasPropertyLayer() {
return hasPropertyLayer;
}
public void setPropertyLayer() {
hasPropertyLayer = true;
}
/** /**
* For the given word, if it contains a pattern with '$' followed by alpha, followed * For the given word, if it contains a pattern with '$' followed by alpha, followed
* by alphanumeric and underscores, replace this pattern with the system property or * by alphanumeric and underscores, replace this pattern with the system property or

30
nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@ -28,12 +28,16 @@ import io.nosqlbench.nb.api.labels.NBLabelSpec;
import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.logging.NBLogLevel; import io.nosqlbench.nb.api.logging.NBLogLevel;
import io.nosqlbench.nb.api.metadata.SystemId; import io.nosqlbench.nb.api.metadata.SystemId;
import io.nosqlbench.nb.api.system.NBEnvironment;
import io.nosqlbench.nb.api.system.NBStatePath; import io.nosqlbench.nb.api.system.NBStatePath;
import io.nosqlbench.engine.api.metrics.IndicatorMode; import io.nosqlbench.engine.api.metrics.IndicatorMode;
import io.nosqlbench.engine.cmdstream.CmdType; import io.nosqlbench.engine.cmdstream.CmdType;
import io.nosqlbench.nb.annotations.Maturity; import io.nosqlbench.nb.annotations.Maturity;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
import java.security.InvalidParameterException; import java.security.InvalidParameterException;
import java.util.*; import java.util.*;
@ -51,6 +55,8 @@ public class NBCLIOptions {
private static final String NB_STATE_DIR = "--statedir"; private static final String NB_STATE_DIR = "--statedir";
public static final String ARGS_FILE_DEFAULT = "$NBSTATEDIR/argsfile"; public static final String ARGS_FILE_DEFAULT = "$NBSTATEDIR/argsfile";
private static final String INCLUDE = "--include"; private static final String INCLUDE = "--include";
private static final String PROPERTIES = "--properties";
private static final String ENVIRONMENT = "--environment";
private static final String userHome = System.getProperty("user.home"); private static final String userHome = System.getProperty("user.home");
@ -415,6 +421,15 @@ public class NBCLIOptions {
arglist.removeFirst(); arglist.removeFirst();
annotatorsConfig = this.readWordOrThrow(arglist, "annotators config"); annotatorsConfig = this.readWordOrThrow(arglist, "annotators config");
break; break;
case NBCLIOptions.PROPERTIES:
arglist.removeFirst();
final String properties = this.readWordOrThrow(arglist, "path to properties file");
this.setProperties(properties);
break;
case NBCLIOptions.ENVIRONMENT:
arglist.removeFirst();
NBEnvironment.INSTANCE.setPropertyLayer();
break;
case NBCLIOptions.REPORT_GRAPHITE_TO: case NBCLIOptions.REPORT_GRAPHITE_TO:
arglist.removeFirst(); arglist.removeFirst();
this.reportGraphiteTo = arglist.removeFirst(); this.reportGraphiteTo = arglist.removeFirst();
@ -774,6 +789,21 @@ public class NBCLIOptions {
return this.wantsToIncludePaths.toArray(new String[0]); return this.wantsToIncludePaths.toArray(new String[0]);
} }
private void setProperties(String filePath) {
Properties properties = new Properties();
try (FileReader reader = new FileReader(filePath)) {
properties.load(reader);
} catch (FileNotFoundException e) {
throw new BasicError("Property File '"+filePath+"' was not found.", e);
} catch (IOException e) {
throw new BasicError("IO Exception Loading Property File '"+filePath+"'.", e);
}
for (String key : properties.stringPropertyNames()) {
NBEnvironment.INSTANCE.put(key, properties.getProperty(key));
}
NBEnvironment.INSTANCE.setPropertyLayer();
}
private Map<String, String> parseLogLevelOverrides(final String levelsSpec) { private Map<String, String> parseLogLevelOverrides(final String levelsSpec) {
final Map<String, String> levels = new HashMap<>(); final Map<String, String> levels = new HashMap<>();
Arrays.stream(levelsSpec.split("[,;]")).forEach(kp -> { Arrays.stream(levelsSpec.split("[,;]")).forEach(kp -> {

38
nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java
View File

@ -16,16 +16,12 @@
package io.nosqlbench.engine.cli.atfiles; package io.nosqlbench.engine.cli.atfiles;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.nbio.NBPathsAPI;
import io.nosqlbench.nb.api.system.NBEnvironment; import io.nosqlbench.nb.api.system.NBEnvironment;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.snakeyaml.engine.v2.api.Load; import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings; import org.snakeyaml.engine.v2.api.LoadSettings;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
@ -38,7 +34,6 @@ import java.util.regex.Pattern;
* TODO:ERRORHANDLER Cannot invoke "Object.getClass()" because "scopeOfInclude" is null on file full of comments only * TODO:ERRORHANDLER Cannot invoke "Object.getClass()" because "scopeOfInclude" is null on file full of comments only
*/ */
public class NBAtFile { public class NBAtFile {
// private final static Logger logger = LogManager.getLogger(NBAtFile.class);
/** /**
* This will take a command line in raw form, which may include some arguments * This will take a command line in raw form, which may include some arguments
@ -63,7 +58,6 @@ public class NBAtFile {
* @throws RuntimeException for any errors finding, traversing, parsing, or rendering values * @throws RuntimeException for any errors finding, traversing, parsing, or rendering values
*/ */
public static LinkedList<String> includeAt(LinkedList<String> processInPlace) { public static LinkedList<String> includeAt(LinkedList<String> processInPlace) {
// logger.trace("argv stream before processing: " + String.join("|",processInPlace));
ListIterator<String> iter = processInPlace.listIterator(); ListIterator<String> iter = processInPlace.listIterator();
while (iter.hasNext()) { while (iter.hasNext()) {
String spec = iter.next(); String spec = iter.next();
@ -76,8 +70,6 @@ public class NBAtFile {
} }
} }
} }
// logger.trace("argv stream after atfile processing: "+ String.join("|",processInPlace));
return processInPlace; return processInPlace;
} }
private final static Pattern includePattern = private final static Pattern includePattern =
@ -121,26 +113,36 @@ public class NBAtFile {
String[] datapath = (dataPathSpec!=null && !dataPathSpec.isBlank()) ? dataPathSpec.split("(/|\\.)") : new String[] {}; String[] datapath = (dataPathSpec!=null && !dataPathSpec.isBlank()) ? dataPathSpec.split("(/|\\.)") : new String[] {};
String filename = Path.of(filepathSpec).getFileName().toString(); String filename = Path.of(filepathSpec).getFileName().toString();
if (filename.contains(".") && !(filename.toLowerCase().endsWith("yaml"))) { if (filename.contains(".") && !( filename.toLowerCase().endsWith("yaml") || filename.toLowerCase().endsWith("properties") )) {
throw new RuntimeException("Only the yaml format and extension is supported for at-files." + throw new RuntimeException("Only the yaml and properties format and extension are supported for at-files." +
" You specified " + filepathSpec); " You specified " + filepathSpec);
} }
filepathSpec=(filepathSpec.endsWith(".yaml") ? filepathSpec : filepathSpec+".yaml"); filepathSpec=(filepathSpec.endsWith(".yaml") ? filepathSpec : (filepathSpec.endsWith(".properties") ? filepathSpec : filepathSpec+".yaml"));
Path atPath = Path.of(filepathSpec); Path atPath = Path.of(filepathSpec);
NBAtFileFormats fmt = (formatSpec!=null) ? NBAtFileFormats.valueOfSymbol(formatSpec) : NBAtFileFormats.Default;
Object scopeOfInclude = null;
try {
if (filepathSpec.endsWith(".properties")) {
FileInputStream inputStream = new FileInputStream(atPath.toFile());
Properties props = new Properties();
props.load(inputStream);
Map<String, String> propsMap = new HashMap<>();
for (String key : props.stringPropertyNames()) {
propsMap.put(key, props.getProperty(key));
}
scopeOfInclude = propsMap;
} else {
Load yaml = new Load(LoadSettings.builder().build());
String argsdata = ""; String argsdata = "";
try { try {
argsdata = Files.readString(atPath); argsdata = Files.readString(atPath);
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
NBAtFileFormats fmt = (formatSpec!=null) ? NBAtFileFormats.valueOfSymbol(formatSpec) : NBAtFileFormats.Default;
Object scopeOfInclude = null;
try {
Load yaml = new Load(LoadSettings.builder().build());
scopeOfInclude = yaml.loadFromString(argsdata); scopeOfInclude = yaml.loadFromString(argsdata);
}
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }

5
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -585,8 +585,7 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok
if ((op != null ? 1 : 0) + (stmt != null ? 1 : 0) + (workload != null ? 1 : 0) > 1) { if ((op != null ? 1 : 0) + (stmt != null ? 1 : 0) + (workload != null ? 1 : 0) > 1) {
throw new OpConfigError("Only op, statement, or workload may be provided, not more than one."); throw new OpConfigError("Only op, statement, or workload may be provided, not more than one.");
} }
logger.debug("loadStmtsDocList #1");
if (workload != null && OpsLoader.isJson(workload)) { if (workload != null && OpsLoader.isJson(workload)) {
workloadSource = "commandline: (workload/json):" + workload; workloadSource = "commandline: (workload/json):" + workload;
return OpsLoader.loadString(workload, OpTemplateFormat.json, activityDef.getParams(), null); return OpsLoader.loadString(workload, OpTemplateFormat.json, activityDef.getParams(), null);
@ -597,11 +596,13 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok
return OpsLoader.loadPath(workload, activityDef.getParams(), "activities"); return OpsLoader.loadPath(workload, activityDef.getParams(), "activities");
} }
logger.debug("loadStmtsDocList #2");
if (stmt != null) { if (stmt != null) {
workloadSource = "commandline: (stmt/inline): '" + stmt + "'"; workloadSource = "commandline: (stmt/inline): '" + stmt + "'";
return OpsLoader.loadString(stmt, OpTemplateFormat.inline, activityDef.getParams(), null); return OpsLoader.loadString(stmt, OpTemplateFormat.inline, activityDef.getParams(), null);
} }
logger.debug("loadStmtsDocList #3");
if (op != null && OpsLoader.isJson(op)) { if (op != null && OpsLoader.isJson(op)) {
workloadSource = "commandline: (op/json): '" + op + "'"; workloadSource = "commandline: (op/json): '" + op + "'";
return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null); return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null);

6
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java
View File

@ -95,7 +95,7 @@ public class StandardActivity<R extends java.util.function.LongFunction, S> exte
List<OpTemplate> opTemplates = loadOpTemplates(defaultAdapter); List<OpTemplate> opTemplates = loadOpTemplates(defaultAdapter);
NBConfigModel combinedAdapterModel = ConfigModel.of(StandardActivity.class); NBConfigModel combinedAdapterModel = ConfigModel.of(StandardActivity.class);
for (OpTemplate ot : opTemplates) { for (OpTemplate ot : opTemplates) {
logger.info(() -> "StandardActivity.opTemplate = "+ot); logger.debug(() -> "StandardActivity.opTemplate = "+ot);
String driverName = ot.getOptionalStringParam("driver", String.class) String driverName = ot.getOptionalStringParam("driver", String.class)
.or(() -> ot.getOptionalStringParam("type", String.class)) .or(() -> ot.getOptionalStringParam("type", String.class))
.orElse(defaultDriverName); .orElse(defaultDriverName);
@ -127,11 +127,11 @@ public class StandardActivity<R extends java.util.function.LongFunction, S> exte
paramsAdvisor.validateAll(ot.getBindings().keySet()); paramsAdvisor.validateAll(ot.getBindings().keySet());
adapterlist.add(adapter); adapterlist.add(adapter);
ParsedOp pop = new ParsedOp(ot, adapter.getConfiguration(), List.of(adapter.getPreprocessor()), this); ParsedOp pop = new ParsedOp(ot, adapter.getConfiguration(), List.of(adapter.getPreprocessor()), this);
logger.info("StandardActivity.pop="+pop); logger.debug("StandardActivity.pop="+pop);
Optional<String> discard = pop.takeOptionalStaticValue("driver", String.class); Optional<String> discard = pop.takeOptionalStaticValue("driver", String.class);
pops.add(pop); pops.add(pop);
} }
logger.info(() -> "StandardActivity.opTemplate loop complete"); logger.debug(() -> "StandardActivity.opTemplate loop complete");
paramsAdvisor.setName("Workload", "Check parameters, template, and binding names") paramsAdvisor.setName("Workload", "Check parameters, template, and binding names")
.logName().evaluate(); .logName().evaluate();