IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-12-22 07:03:35 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add Property File Support plus Related Workload Improvements (#2115)

Browse Source 
* Use Property Files for Environment Vars and as AtFiles

* Keep a private property overlay over system properties

* Add properties from file and use in workload templates

* Allow access to properties and env vars is enabled

* Reorder to ease reading the code

* Insert property, yaml, and json files

* Make exceptions meaningful

* Catch all errors on error insert file type

* Some RawOpsLoader Refactoring and other Small Fixes
This commit is contained in:
Dave Fisher 2024-12-18 12:01:25 -08:00 committed by GitHub
parent 79e2a4bf6e
commit 1784f5dcd0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 214 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java
View File

@ -57,6 +57,7 @@ public class OpsLoader {
return loadString(content.get().toString(), fmt, params, content.getURI());
}
// used in OpsDocList (at least)
public static OpsDocList loadPath(String path, Map<String, ?> params, String... searchPaths) {
String[] extensions = path.indexOf('.') > -1 ? new String[]{} : YAML_EXTENSIONS;
ResolverChain chain = new ResolverChain(path);
@ -69,18 +70,17 @@ public class OpsLoader {
public static OpsDocList loadString(
final String sourceData, OpTemplateFormat fmt, Map<String, ?> params, URI srcuri) {
logger.trace(() -> "Applying string transformer to data:" + sourceData);
if (srcuri != null) {
logger.info("workload URI: '" + srcuri + "'");
}
StrInterpolator transformer = new StrInterpolator(params);
String data = transformer.apply(sourceData);
//String data = transformer.apply(sourceData);
RawOpsLoader loader = new RawOpsLoader(transformer);
RawOpsDocList rawOpsDocList = switch (fmt) {
case jsonnet -> loader.loadString(evaluateJsonnet(srcuri, params));
case yaml, json -> loader.loadString(data);
case inline, stmt -> RawOpsDocList.forSingleStatement(data);
case yaml, json -> loader.loadString(sourceData);
case inline, stmt -> RawOpsDocList.forSingleStatement(transformer.apply(sourceData));
};
// TODO: itemize inline to support ParamParser

52
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpsLoader.java
View File

@ -64,21 +64,6 @@ public class RawOpsLoader {
Collections.addAll(this.transformers, newTransformer);
}
public RawOpsDocList loadString(final String originalData) {
logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
String data = originalData;
try {
for (Function<String, String> transformer : transformers) {
data = transformer.apply(data);
}
} catch (Exception e) {
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
throw t;
}
return parseYaml(data);
}
public RawOpsDocList loadPath(
String path,
String... searchPaths) {
@ -94,6 +79,29 @@ public class RawOpsLoader {
}
public RawOpsDocList loadString(final String originalData) {
String data = applyTransforms(originalData);
return parseYaml(data);
}
public List<Map<String,Object>> loadStringMap(final String originalData) {
String data = applyTransforms(originalData);
List<Map<String,Object>> maps = parseYamlMap(data);
return maps;
}
protected String applyTransforms(String originalData) {
String data = originalData;
for (Function<String, String> transformer : transformers) {
try {
data = transformer.apply(data);
} catch (Exception e) {
throw new OpConfigError("Error applying string transforms to input", e);
}
}
return data;
}
public RawOpsDocList parseYaml(String data) {
Iterable<Object> objects = yaml.loadAllFromString(data);
List<RawOpsDoc> newDocList = new ArrayList<>();
@ -125,4 +133,18 @@ public class RawOpsLoader {
return rawOpsDocList;
}
private List<Map<String,Object>> parseYamlMap(String data) {
Iterable<Object> objects = yaml.loadAllFromString(data);
List<Map<String,Object>> maps = new ArrayList<>();
for (Object object : objects) {
if (object instanceof Map) {
maps.add(new LinkedHashMap<>((Map<String,Object>)object));
} else {
throw new OpConfigError("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
}
}
return maps;
}
}

106
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawYamlLoader.java
View File

@ -1,106 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapters.api.activityconfig.rawyaml;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.nb.api.errors.OpConfigError;
import io.nosqlbench.adapters.api.templating.StrInterpolator;
import org.apache.logging.log4j.Logger;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
import java.util.*;
import java.util.function.Function;
public class RawYamlLoader {
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
List<Function<String, String>> stringTransformers = new ArrayList<>();
private final ArrayList<Function<String,String>> transformers = new ArrayList<>();
private void addTransformer(Function<String, String> newTransformer) {
Collections.addAll(this.transformers, newTransformer);
}
public RawYamlLoader() {
addTransformer(new StrInterpolator());
}
public List<Map<String,Object>> loadString(Logger logger, String originalData) {
String data = originalData;
try {
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
for (Function<String, String> transformer : transformers) {
data = transformer.apply(data);
}
} catch (Exception e) {
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
throw t;
}
return parseYaml(logger, data);
}
public List<Map<String,Object>> loadPath(
Logger logger,
String path,
String... searchPaths) {
String data = null;
try {
Optional<Content<?>> oyaml = NBIO.all().searchPrefixes(searchPaths).pathname(path).extensionSet(YAML_EXTENSIONS).first();
data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(logger, data);
} catch (Exception e) {
throw new RuntimeException("error while reading file " + path, e);
}
}
private List<Map<String,Object>> parseYaml(Logger logger, String data) {
Load yaml = new Load(LoadSettings.builder().build());
Iterable<Object> objects = yaml.loadAllFromString(data);
List<RawOpsDoc> newDocList = new ArrayList<>();
List<Map<String,Object>> maps = new ArrayList<>();
for (Object object : objects) {
if (object instanceof Map) {
maps.add(new LinkedHashMap<>((Map<String,Object>)object));
} else {
throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
}
}
return maps;
}
protected String applyTransforms(Logger logger, String data) {
for (Function<String, String> xform : stringTransformers) {
try {
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + xform);
data = xform.apply(data);
} catch (Exception e) {
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
throw t;
}
}
return data;
}
}

91
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java
View File

@ -16,19 +16,30 @@
package io.nosqlbench.adapters.api.templating;
import com.google.gson.Gson;
import io.nosqlbench.nb.api.advisor.NBAdvisorBuilder;
import io.nosqlbench.nb.api.advisor.NBAdvisorPoint;
import io.nosqlbench.nb.api.advisor.conditions.Conditions;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.errors.OpConfigError;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.nbio.ResolverChain;
import io.nosqlbench.nb.api.system.NBEnvironment;
import org.apache.commons.text.StrLookup;
import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.yaml.snakeyaml.Yaml;
import java.io.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.*;
import java.util.function.Function;
import static java.util.stream.Collectors.*;
public class StrInterpolator implements Function<String, String> {
private final static Logger logger = LogManager.getLogger(StrInterpolator.class);
private final NBAdvisorBuilder<String> advisorBuilder = new NBAdvisorBuilder<String>();
@ -40,6 +51,7 @@ public class StrInterpolator implements Function<String, String> {
.setEnableUndefinedVariableException(true)
.setDisableSubstitutionInValues(true);
private final Pattern COMMENT = Pattern.compile("^\\s*#.*");
private final Pattern INSERT = Pattern.compile("^(\\s*)INSERT:\\s+(.+)$");
public StrInterpolator(ActivityDef... activityDefs) {
Arrays.stream(activityDefs)
@ -62,36 +74,84 @@ public class StrInterpolator implements Function<String, String> {
@Override
public String apply(String raw) {
logger.debug(() -> "Applying string transformer to data:\n" + raw);
advisor = advisorBuilder.build();
advisor.add(Conditions.DeprecatedWarning);
String[] lines = raw.split("\\R");
List<String> lines = new LinkedList<>(Arrays.asList(raw.split("\\R")));
boolean endsWithNewline = raw.endsWith("\n");
int i = 0;
for (String line : lines) {
while (i < lines.size()) {
String line = lines.get(i);
if (!isComment(line)) {
String result = matchTemplates(line);
if (!result.equals(line)) {
lines[i] = result;
lines.set(i, result);
line = result;
}
Matcher matcher = INSERT.matcher(line);
if (matcher.matches()) {
String leadingSpaces = matcher.group(1);
String filePath = matcher.group(2);
List<String> includes = insertContentFromFile(leadingSpaces, filePath);
System.out.println(leadingSpaces + "INSERT: " + filePath);
lines.remove(i);
lines.addAll(i, includes);
i--;
}
}
i++;
}
String results = String.join(System.lineSeparator(), lines);
String results = lines.stream().collect(joining(System.lineSeparator()));
if (endsWithNewline) {
results += System.lineSeparator();
}
advisor.setName("Workload", "Deprecated template format").logName().evaluate();
String finalResults = results;
logger.debug(() -> "Results of applying string transformer:\n" + finalResults);
return results;
}
public Map<String,String> checkpointAccesses() {
return multimap.checkpointAccesses();
private LinkedList<String> insertContentFromFile(String leadingSpaces, String filePath) {
// Determine file type and process the inclusion
LinkedList<String> result = new LinkedList<>();
result.add(leadingSpaces + "# INSERT: " + filePath);
try {
ResolverChain chain = new ResolverChain(filePath);
Content<?> insert = NBIO.chain(chain.getChain()).searchPrefixes("activities")
.pathname(chain.getPath()).first()
.orElseThrow(() -> new RuntimeException("Unable to load path '" + filePath + "'"));
BufferedReader reader = new BufferedReader(new StringReader(insert.asString()));
if (filePath.endsWith(".properties")) {
// Include properties file
Properties properties = new Properties();
properties.load(reader);
for (String key : properties.stringPropertyNames()) {
result.add(leadingSpaces + key + ": " + properties.getProperty(key));
}
public LinkedHashMap<String, String> getTemplateDetails(String input) {
LinkedHashMap<String, String> details = new LinkedHashMap<>();
return details;
} else if (filePath.endsWith(".json")) {
// Include JSON
Gson gson = new Gson();
Map<String, Object> jsonMap = gson.fromJson(reader, Map.class);
Yaml yaml = new Yaml();
String yamlString = yaml.dumpAsMap(jsonMap);
LinkedList<String> include = new LinkedList<>(Arrays.asList(yamlString.split("\\R")));
int j = 0;
while (j < include.size()) {
result.add(leadingSpaces + include.get(j));
j++;
}
} else {
// Include as a YAML file (if it is not then if a bad OpDocList is created it will fail.
String line;
while ((line = reader.readLine()) != null) {
result.add(leadingSpaces + line);
}
}
} catch (Exception e) {
throw new OpConfigError("While processing file '" + filePath + "' " + e.getMessage());
}
return result;
}
public String matchTemplates(String original) {
@ -177,6 +237,10 @@ public class StrInterpolator implements Function<String, String> {
return line;
}
public Map<String,String> checkpointAccesses() {
return multimap.checkpointAccesses();
}
public static class MultiMap extends StrLookup<String> {
private final String warnPrefix = "UNSET";
@ -232,13 +296,18 @@ public class StrInterpolator implements Function<String, String> {
value = val.toString();
//System.out.println("for: '"+original+"': "+key+"->"+value);
} else {
boolean check_env = true;
for (Map<String, ?> map : maps) {
val = map.get(key);
if (val != null) {
value = val.toString();
check_env = false;
break;
}
}
if (check_env && NBEnvironment.INSTANCE.hasPropertyLayer() && NBEnvironment.INSTANCE.containsKey(key) ) {
value = NBEnvironment.INSTANCE.get(key);
}
}
value = (value==null? extractedDefaults.get(key) : value);
}

4
nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/activityconfig/specifications/YamlSpecValidator.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.adapters.api.activityconfig.specifications;
import com.google.gson.*;
import com.google.gson.reflect.TypeToken;
import io.nosqlbench.adapters.api.activityconfig.OpsLoader;
import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawYamlLoader;
import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList;
@ -145,7 +145,7 @@ public class YamlSpecValidator implements STAssemblyValidator {
System.out.format("%-40s", "- checking yaml->json");
try {
List<Map<String, Object>> docmaps = new RawYamlLoader().loadString(logger, yaml);
List<Map<String, Object>> docmaps = new RawOpsLoader().loadStringMap(yaml);
JsonElement elem = null;
try {
elem = JsonParser.parseString(json);

2
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorPoint.java
View File

@ -98,7 +98,7 @@ public class NBAdvisorPoint<T> extends NBAdvisorPointOrBuilder<T> {
}
public NBAdvisorPoint<T> logName() {
if (resultLog.size() > 0) {
if (resultLog.size() > 0 && advisorLevel != NBAdvisorLevel.none) {
logger.info("Advisor: " + name + ": " + description);
}
return this;

4
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigModel.java
View File

@ -225,7 +225,7 @@ public class ConfigModel implements NBConfigModel {
for (String configkey : config.keySet()) {
Param<?> element = this.paramsByName.get(configkey);
if (element != null) {
String warning = "Config parameter '" + configkey + "' is also a " + type + ". Check for possible conflicts.";
String warning = "Config parameter '" + configkey + "' is also a " + type + ". Check for possible conflicts.\n";
NBAdvisorOutput.output(Level.WARN, warning);
}
}
@ -299,7 +299,7 @@ public class ConfigModel implements NBConfigModel {
for (String configkey : config.keySet()) {
Param<?> element = this.paramsByName.get(configkey);
String warning = "Unknown config parameter '" + configkey + "' in config model while configuring " + getOf().getSimpleName()
+ ", possible parameter names are " + this.paramsByName.keySet() + ".";
+ ", possible parameter names are " + this.paramsByName.keySet() + ".\n";
if (element == null) {
String warnonly = System.getenv("NB_CONFIG_WARNINGS_ONLY");
logger.warn("WARNING: " + warning);

28
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/system/NBEnvironment.java
View File

@ -65,6 +65,10 @@ public class NBEnvironment {
private final LinkedHashMap<String, String> references = new LinkedHashMap<>();
private final Map<String,String> properties = new HashMap<>();
private boolean hasPropertyLayer = false;
/**
* These properties are well-defined in the Java specs. This map redirects common
* environment variable names to the given system property. This allows
@ -101,7 +105,9 @@ public class NBEnvironment {
}
}
System.setProperty(propname, value);
// Instead of using System properties we keep an internal overlay.
properties.put(propname, value);
//System.setProperty(propname, value);
}
/**
@ -153,6 +159,10 @@ public class NBEnvironment {
}
}
if (name.contains(".")) {
value = properties.get(name.toLowerCase());
if (value != null) {
return value;
}
value = System.getProperty(name.toLowerCase());
if (value != null) {
return value;
@ -163,11 +173,19 @@ public class NBEnvironment {
if (logger != null) {
logger.debug("redirecting env var '" + name + "' to upper-case property '" + propName + "'");
}
value = properties.get(propName);
if (value != null) {
return value;
}
value = System.getProperty(propName);
if (value != null) {
return value;
}
}
value = properties.get(name);
if (value != null) {
return value;
}
value = System.getProperty(name);
if (value != null) {
return value;
@ -203,6 +221,14 @@ public class NBEnvironment {
return (value != null);
}
public boolean hasPropertyLayer() {
return hasPropertyLayer;
}
public void setPropertyLayer() {
hasPropertyLayer = true;
}
/**
* For the given word, if it contains a pattern with '$' followed by alpha, followed
* by alphanumeric and underscores, replace this pattern with the system property or

30
nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@ -28,12 +28,16 @@ import io.nosqlbench.nb.api.labels.NBLabelSpec;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.logging.NBLogLevel;
import io.nosqlbench.nb.api.metadata.SystemId;
import io.nosqlbench.nb.api.system.NBEnvironment;
import io.nosqlbench.nb.api.system.NBStatePath;
import io.nosqlbench.engine.api.metrics.IndicatorMode;
import io.nosqlbench.engine.cmdstream.CmdType;
import io.nosqlbench.nb.annotations.Maturity;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Path;
import java.security.InvalidParameterException;
import java.util.*;
@ -51,6 +55,8 @@ public class NBCLIOptions {
private static final String NB_STATE_DIR = "--statedir";
public static final String ARGS_FILE_DEFAULT = "$NBSTATEDIR/argsfile";
private static final String INCLUDE = "--include";
private static final String PROPERTIES = "--properties";
private static final String ENVIRONMENT = "--environment";
private static final String userHome = System.getProperty("user.home");
@ -415,6 +421,15 @@ public class NBCLIOptions {
arglist.removeFirst();
annotatorsConfig = this.readWordOrThrow(arglist, "annotators config");
break;
case NBCLIOptions.PROPERTIES:
arglist.removeFirst();
final String properties = this.readWordOrThrow(arglist, "path to properties file");
this.setProperties(properties);
break;
case NBCLIOptions.ENVIRONMENT:
arglist.removeFirst();
NBEnvironment.INSTANCE.setPropertyLayer();
break;
case NBCLIOptions.REPORT_GRAPHITE_TO:
arglist.removeFirst();
this.reportGraphiteTo = arglist.removeFirst();
@ -774,6 +789,21 @@ public class NBCLIOptions {
return this.wantsToIncludePaths.toArray(new String[0]);
}
private void setProperties(String filePath) {
Properties properties = new Properties();
try (FileReader reader = new FileReader(filePath)) {
properties.load(reader);
} catch (FileNotFoundException e) {
throw new BasicError("Property File '"+filePath+"' was not found.", e);
} catch (IOException e) {
throw new BasicError("IO Exception Loading Property File '"+filePath+"'.", e);
}
for (String key : properties.stringPropertyNames()) {
NBEnvironment.INSTANCE.put(key, properties.getProperty(key));
}
NBEnvironment.INSTANCE.setPropertyLayer();
}
private Map<String, String> parseLogLevelOverrides(final String levelsSpec) {
final Map<String, String> levels = new HashMap<>();
Arrays.stream(levelsSpec.split("[,;]")).forEach(kp -> {

38
nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java
View File

@ -16,16 +16,12 @@
package io.nosqlbench.engine.cli.atfiles;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.nbio.NBPathsAPI;
import io.nosqlbench.nb.api.system.NBEnvironment;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@ -38,7 +34,6 @@ import java.util.regex.Pattern;
* TODO:ERRORHANDLER Cannot invoke "Object.getClass()" because "scopeOfInclude" is null on file full of comments only
*/
public class NBAtFile {
// private final static Logger logger = LogManager.getLogger(NBAtFile.class);
/**
* This will take a command line in raw form, which may include some arguments
@ -63,7 +58,6 @@ public class NBAtFile {
* @throws RuntimeException for any errors finding, traversing, parsing, or rendering values
*/
public static LinkedList<String> includeAt(LinkedList<String> processInPlace) {
// logger.trace("argv stream before processing: " + String.join("|",processInPlace));
ListIterator<String> iter = processInPlace.listIterator();
while (iter.hasNext()) {
String spec = iter.next();
@ -76,8 +70,6 @@ public class NBAtFile {
}
}
}
// logger.trace("argv stream after atfile processing: "+ String.join("|",processInPlace));
return processInPlace;
}
private final static Pattern includePattern =
@ -121,26 +113,36 @@ public class NBAtFile {
String[] datapath = (dataPathSpec!=null && !dataPathSpec.isBlank()) ? dataPathSpec.split("(/|\\.)") : new String[] {};
String filename = Path.of(filepathSpec).getFileName().toString();
if (filename.contains(".") && !(filename.toLowerCase().endsWith("yaml"))) {
throw new RuntimeException("Only the yaml format and extension is supported for at-files." +
if (filename.contains(".") && !( filename.toLowerCase().endsWith("yaml") || filename.toLowerCase().endsWith("properties") )) {
throw new RuntimeException("Only the yaml and properties format and extension are supported for at-files." +
" You specified " + filepathSpec);
}
filepathSpec=(filepathSpec.endsWith(".yaml") ? filepathSpec : filepathSpec+".yaml");
filepathSpec=(filepathSpec.endsWith(".yaml") ? filepathSpec : (filepathSpec.endsWith(".properties") ? filepathSpec : filepathSpec+".yaml"));
Path atPath = Path.of(filepathSpec);
NBAtFileFormats fmt = (formatSpec!=null) ? NBAtFileFormats.valueOfSymbol(formatSpec) : NBAtFileFormats.Default;
Object scopeOfInclude = null;
try {
if (filepathSpec.endsWith(".properties")) {
FileInputStream inputStream = new FileInputStream(atPath.toFile());
Properties props = new Properties();
props.load(inputStream);
Map<String, String> propsMap = new HashMap<>();
for (String key : props.stringPropertyNames()) {
propsMap.put(key, props.getProperty(key));
}
scopeOfInclude = propsMap;
} else {
Load yaml = new Load(LoadSettings.builder().build());
String argsdata = "";
try {
argsdata = Files.readString(atPath);
} catch (IOException e) {
throw new RuntimeException(e);
}
NBAtFileFormats fmt = (formatSpec!=null) ? NBAtFileFormats.valueOfSymbol(formatSpec) : NBAtFileFormats.Default;
Object scopeOfInclude = null;
try {
Load yaml = new Load(LoadSettings.builder().build());
scopeOfInclude = yaml.loadFromString(argsdata);
}
} catch (Exception e) {
throw new RuntimeException(e);
}

5
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -585,8 +585,7 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok
if ((op != null ? 1 : 0) + (stmt != null ? 1 : 0) + (workload != null ? 1 : 0) > 1) {
throw new OpConfigError("Only op, statement, or workload may be provided, not more than one.");
}
logger.debug("loadStmtsDocList #1");
if (workload != null && OpsLoader.isJson(workload)) {
workloadSource = "commandline: (workload/json):" + workload;
return OpsLoader.loadString(workload, OpTemplateFormat.json, activityDef.getParams(), null);
@ -597,11 +596,13 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok
return OpsLoader.loadPath(workload, activityDef.getParams(), "activities");
}
logger.debug("loadStmtsDocList #2");
if (stmt != null) {
workloadSource = "commandline: (stmt/inline): '" + stmt + "'";
return OpsLoader.loadString(stmt, OpTemplateFormat.inline, activityDef.getParams(), null);
}
logger.debug("loadStmtsDocList #3");
if (op != null && OpsLoader.isJson(op)) {
workloadSource = "commandline: (op/json): '" + op + "'";
return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null);

6
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java
View File

@ -95,7 +95,7 @@ public class StandardActivity<R extends java.util.function.LongFunction, S> exte
List<OpTemplate> opTemplates = loadOpTemplates(defaultAdapter);
NBConfigModel combinedAdapterModel = ConfigModel.of(StandardActivity.class);
for (OpTemplate ot : opTemplates) {
logger.info(() -> "StandardActivity.opTemplate = "+ot);
logger.debug(() -> "StandardActivity.opTemplate = "+ot);
String driverName = ot.getOptionalStringParam("driver", String.class)
.or(() -> ot.getOptionalStringParam("type", String.class))
.orElse(defaultDriverName);
@ -127,11 +127,11 @@ public class StandardActivity<R extends java.util.function.LongFunction, S> exte
paramsAdvisor.validateAll(ot.getBindings().keySet());
adapterlist.add(adapter);
ParsedOp pop = new ParsedOp(ot, adapter.getConfiguration(), List.of(adapter.getPreprocessor()), this);
logger.info("StandardActivity.pop="+pop);
logger.debug("StandardActivity.pop="+pop);
Optional<String> discard = pop.takeOptionalStaticValue("driver", String.class);
pops.add(pop);
}
logger.info(() -> "StandardActivity.opTemplate loop complete");
logger.debug(() -> "StandardActivity.opTemplate loop complete");
paramsAdvisor.setName("Workload", "Check parameters, template, and binding names")
.logName().evaluate();