mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2024-12-22 07:03:35 -06:00
Add Property File Support plus Related Workload Improvements (#2115)
* Use Property Files for Environment Vars and as AtFiles * Keep a private property overlay over system properties * Add properties from file and use in workload templates * Allow access to properties and env vars is enabled * Reorder to ease reading the code * Insert property, yaml, and json files * Make exceptions meaningful * Catch all errors on error insert file type * Some RawOpsLoader Refactoring and other Small Fixes
This commit is contained in:
Dave Fisher
GitHub
parent
79e2a4bf6e
commit
1784f5dcd0
@ -57,6 +57,7 @@ public class OpsLoader {
|
||||
return loadString(content.get().toString(), fmt, params, content.getURI());
|
||||
}
|
||||
|
||||
// used in OpsDocList (at least)
|
||||
public static OpsDocList loadPath(String path, Map<String, ?> params, String... searchPaths) {
|
||||
String[] extensions = path.indexOf('.') > -1 ? new String[]{} : YAML_EXTENSIONS;
|
||||
ResolverChain chain = new ResolverChain(path);
|
||||
@ -69,18 +70,17 @@ public class OpsLoader {
|
||||
public static OpsDocList loadString(
|
||||
final String sourceData, OpTemplateFormat fmt, Map<String, ?> params, URI srcuri) {
|
||||
|
||||
logger.trace(() -> "Applying string transformer to data:" + sourceData);
|
||||
if (srcuri != null) {
|
||||
logger.info("workload URI: '" + srcuri + "'");
|
||||
}
|
||||
StrInterpolator transformer = new StrInterpolator(params);
|
||||
String data = transformer.apply(sourceData);
|
||||
//String data = transformer.apply(sourceData);
|
||||
|
||||
RawOpsLoader loader = new RawOpsLoader(transformer);
|
||||
RawOpsDocList rawOpsDocList = switch (fmt) {
|
||||
case jsonnet -> loader.loadString(evaluateJsonnet(srcuri, params));
|
||||
case yaml, json -> loader.loadString(data);
|
||||
case inline, stmt -> RawOpsDocList.forSingleStatement(data);
|
||||
case yaml, json -> loader.loadString(sourceData);
|
||||
case inline, stmt -> RawOpsDocList.forSingleStatement(transformer.apply(sourceData));
|
||||
};
|
||||
// TODO: itemize inline to support ParamParser
|
||||
|
||||
|
||||
@ -64,24 +64,9 @@ public class RawOpsLoader {
|
||||
Collections.addAll(this.transformers, newTransformer);
|
||||
}
|
||||
|
||||
public RawOpsDocList loadString(final String originalData) {
|
||||
logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
|
||||
String data = originalData;
|
||||
try {
|
||||
for (Function<String, String> transformer : transformers) {
|
||||
data = transformer.apply(data);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
|
||||
throw t;
|
||||
}
|
||||
|
||||
return parseYaml(data);
|
||||
}
|
||||
|
||||
public RawOpsDocList loadPath(
|
||||
String path,
|
||||
String... searchPaths) {
|
||||
String path,
|
||||
String... searchPaths) {
|
||||
|
||||
String data = null;
|
||||
try {
|
||||
@ -94,6 +79,29 @@ public class RawOpsLoader {
|
||||
|
||||
}
|
||||
|
||||
public RawOpsDocList loadString(final String originalData) {
|
||||
String data = applyTransforms(originalData);
|
||||
return parseYaml(data);
|
||||
}
|
||||
|
||||
public List<Map<String,Object>> loadStringMap(final String originalData) {
|
||||
String data = applyTransforms(originalData);
|
||||
List<Map<String,Object>> maps = parseYamlMap(data);
|
||||
return maps;
|
||||
}
|
||||
|
||||
protected String applyTransforms(String originalData) {
|
||||
String data = originalData;
|
||||
for (Function<String, String> transformer : transformers) {
|
||||
try {
|
||||
data = transformer.apply(data);
|
||||
} catch (Exception e) {
|
||||
throw new OpConfigError("Error applying string transforms to input", e);
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
public RawOpsDocList parseYaml(String data) {
|
||||
Iterable<Object> objects = yaml.loadAllFromString(data);
|
||||
List<RawOpsDoc> newDocList = new ArrayList<>();
|
||||
@ -125,4 +133,18 @@ public class RawOpsLoader {
|
||||
return rawOpsDocList;
|
||||
}
|
||||
|
||||
private List<Map<String,Object>> parseYamlMap(String data) {
|
||||
Iterable<Object> objects = yaml.loadAllFromString(data);
|
||||
List<Map<String,Object>> maps = new ArrayList<>();
|
||||
|
||||
for (Object object : objects) {
|
||||
if (object instanceof Map) {
|
||||
maps.add(new LinkedHashMap<>((Map<String,Object>)object));
|
||||
} else {
|
||||
throw new OpConfigError("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
|
||||
}
|
||||
}
|
||||
return maps;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,106 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.adapters.api.activityconfig.rawyaml;
|
||||
|
||||
import io.nosqlbench.nb.api.nbio.Content;
|
||||
import io.nosqlbench.nb.api.nbio.NBIO;
|
||||
import io.nosqlbench.nb.api.errors.BasicError;
|
||||
import io.nosqlbench.nb.api.errors.OpConfigError;
|
||||
import io.nosqlbench.adapters.api.templating.StrInterpolator;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.snakeyaml.engine.v2.api.Load;
|
||||
import org.snakeyaml.engine.v2.api.LoadSettings;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
|
||||
public class RawYamlLoader {
|
||||
|
||||
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
|
||||
|
||||
List<Function<String, String>> stringTransformers = new ArrayList<>();
|
||||
private final ArrayList<Function<String,String>> transformers = new ArrayList<>();
|
||||
|
||||
private void addTransformer(Function<String, String> newTransformer) {
|
||||
Collections.addAll(this.transformers, newTransformer);
|
||||
}
|
||||
|
||||
public RawYamlLoader() {
|
||||
addTransformer(new StrInterpolator());
|
||||
}
|
||||
|
||||
public List<Map<String,Object>> loadString(Logger logger, String originalData) {
|
||||
String data = originalData;
|
||||
try {
|
||||
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
|
||||
for (Function<String, String> transformer : transformers) {
|
||||
data = transformer.apply(data);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
|
||||
throw t;
|
||||
}
|
||||
|
||||
return parseYaml(logger, data);
|
||||
}
|
||||
|
||||
public List<Map<String,Object>> loadPath(
|
||||
Logger logger,
|
||||
String path,
|
||||
String... searchPaths) {
|
||||
|
||||
String data = null;
|
||||
try {
|
||||
Optional<Content<?>> oyaml = NBIO.all().searchPrefixes(searchPaths).pathname(path).extensionSet(YAML_EXTENSIONS).first();
|
||||
data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
|
||||
return loadString(logger, data);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("error while reading file " + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<Map<String,Object>> parseYaml(Logger logger, String data) {
|
||||
Load yaml = new Load(LoadSettings.builder().build());
|
||||
Iterable<Object> objects = yaml.loadAllFromString(data);
|
||||
List<RawOpsDoc> newDocList = new ArrayList<>();
|
||||
|
||||
List<Map<String,Object>> maps = new ArrayList<>();
|
||||
|
||||
for (Object object : objects) {
|
||||
if (object instanceof Map) {
|
||||
maps.add(new LinkedHashMap<>((Map<String,Object>)object));
|
||||
} else {
|
||||
throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
|
||||
}
|
||||
}
|
||||
return maps;
|
||||
}
|
||||
|
||||
protected String applyTransforms(Logger logger, String data) {
|
||||
for (Function<String, String> xform : stringTransformers) {
|
||||
try {
|
||||
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + xform);
|
||||
data = xform.apply(data);
|
||||
} catch (Exception e) {
|
||||
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
|
||||
throw t;
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
}
|
||||
@ -16,19 +16,30 @@
|
||||
|
||||
package io.nosqlbench.adapters.api.templating;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import io.nosqlbench.nb.api.advisor.NBAdvisorBuilder;
|
||||
import io.nosqlbench.nb.api.advisor.NBAdvisorPoint;
|
||||
import io.nosqlbench.nb.api.advisor.conditions.Conditions;
|
||||
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
|
||||
import io.nosqlbench.nb.api.errors.OpConfigError;
|
||||
import io.nosqlbench.nb.api.nbio.Content;
|
||||
import io.nosqlbench.nb.api.nbio.NBIO;
|
||||
import io.nosqlbench.nb.api.nbio.ResolverChain;
|
||||
import io.nosqlbench.nb.api.system.NBEnvironment;
|
||||
import org.apache.commons.text.StrLookup;
|
||||
import org.apache.commons.text.StringSubstitutor;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
|
||||
import static java.util.stream.Collectors.*;
|
||||
|
||||
public class StrInterpolator implements Function<String, String> {
|
||||
private final static Logger logger = LogManager.getLogger(StrInterpolator.class);
|
||||
private final NBAdvisorBuilder<String> advisorBuilder = new NBAdvisorBuilder<String>();
|
||||
@ -40,6 +51,7 @@ public class StrInterpolator implements Function<String, String> {
|
||||
.setEnableUndefinedVariableException(true)
|
||||
.setDisableSubstitutionInValues(true);
|
||||
private final Pattern COMMENT = Pattern.compile("^\\s*#.*");
|
||||
private final Pattern INSERT = Pattern.compile("^(\\s*)INSERT:\\s+(.+)$");
|
||||
|
||||
public StrInterpolator(ActivityDef... activityDefs) {
|
||||
Arrays.stream(activityDefs)
|
||||
@ -62,36 +74,84 @@ public class StrInterpolator implements Function<String, String> {
|
||||
|
||||
@Override
|
||||
public String apply(String raw) {
|
||||
logger.debug(() -> "Applying string transformer to data:\n" + raw);
|
||||
advisor = advisorBuilder.build();
|
||||
advisor.add(Conditions.DeprecatedWarning);
|
||||
String[] lines = raw.split("\\R");
|
||||
List<String> lines = new LinkedList<>(Arrays.asList(raw.split("\\R")));
|
||||
boolean endsWithNewline = raw.endsWith("\n");
|
||||
int i = 0;
|
||||
for (String line : lines) {
|
||||
while (i < lines.size()) {
|
||||
String line = lines.get(i);
|
||||
if (!isComment(line)) {
|
||||
String result = matchTemplates(line);
|
||||
if (!result.equals(line)) {
|
||||
lines[i] = result;
|
||||
lines.set(i, result);
|
||||
line = result;
|
||||
}
|
||||
Matcher matcher = INSERT.matcher(line);
|
||||
if (matcher.matches()) {
|
||||
String leadingSpaces = matcher.group(1);
|
||||
String filePath = matcher.group(2);
|
||||
List<String> includes = insertContentFromFile(leadingSpaces, filePath);
|
||||
System.out.println(leadingSpaces + "INSERT: " + filePath);
|
||||
lines.remove(i);
|
||||
lines.addAll(i, includes);
|
||||
i--;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
}
|
||||
String results = String.join(System.lineSeparator(), lines);
|
||||
String results = lines.stream().collect(joining(System.lineSeparator()));
|
||||
if (endsWithNewline) {
|
||||
results += System.lineSeparator();
|
||||
}
|
||||
advisor.setName("Workload", "Deprecated template format").logName().evaluate();
|
||||
String finalResults = results;
|
||||
logger.debug(() -> "Results of applying string transformer:\n" + finalResults);
|
||||
return results;
|
||||
}
|
||||
|
||||
public Map<String,String> checkpointAccesses() {
|
||||
return multimap.checkpointAccesses();
|
||||
}
|
||||
private LinkedList<String> insertContentFromFile(String leadingSpaces, String filePath) {
|
||||
// Determine file type and process the inclusion
|
||||
LinkedList<String> result = new LinkedList<>();
|
||||
result.add(leadingSpaces + "# INSERT: " + filePath);
|
||||
try {
|
||||
ResolverChain chain = new ResolverChain(filePath);
|
||||
Content<?> insert = NBIO.chain(chain.getChain()).searchPrefixes("activities")
|
||||
.pathname(chain.getPath()).first()
|
||||
.orElseThrow(() -> new RuntimeException("Unable to load path '" + filePath + "'"));
|
||||
BufferedReader reader = new BufferedReader(new StringReader(insert.asString()));
|
||||
|
||||
public LinkedHashMap<String, String> getTemplateDetails(String input) {
|
||||
LinkedHashMap<String, String> details = new LinkedHashMap<>();
|
||||
|
||||
return details;
|
||||
if (filePath.endsWith(".properties")) {
|
||||
// Include properties file
|
||||
Properties properties = new Properties();
|
||||
properties.load(reader);
|
||||
for (String key : properties.stringPropertyNames()) {
|
||||
result.add(leadingSpaces + key + ": " + properties.getProperty(key));
|
||||
}
|
||||
} else if (filePath.endsWith(".json")) {
|
||||
// Include JSON
|
||||
Gson gson = new Gson();
|
||||
Map<String, Object> jsonMap = gson.fromJson(reader, Map.class);
|
||||
Yaml yaml = new Yaml();
|
||||
String yamlString = yaml.dumpAsMap(jsonMap);
|
||||
LinkedList<String> include = new LinkedList<>(Arrays.asList(yamlString.split("\\R")));
|
||||
int j = 0;
|
||||
while (j < include.size()) {
|
||||
result.add(leadingSpaces + include.get(j));
|
||||
j++;
|
||||
}
|
||||
} else {
|
||||
// Include as a YAML file (if it is not then if a bad OpDocList is created it will fail.
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
result.add(leadingSpaces + line);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new OpConfigError("While processing file '" + filePath + "' " + e.getMessage());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public String matchTemplates(String original) {
|
||||
@ -177,6 +237,10 @@ public class StrInterpolator implements Function<String, String> {
|
||||
return line;
|
||||
}
|
||||
|
||||
public Map<String,String> checkpointAccesses() {
|
||||
return multimap.checkpointAccesses();
|
||||
}
|
||||
|
||||
public static class MultiMap extends StrLookup<String> {
|
||||
|
||||
private final String warnPrefix = "UNSET";
|
||||
@ -232,13 +296,18 @@ public class StrInterpolator implements Function<String, String> {
|
||||
value = val.toString();
|
||||
//System.out.println("for: '"+original+"': "+key+"->"+value);
|
||||
} else {
|
||||
boolean check_env = true;
|
||||
for (Map<String, ?> map : maps) {
|
||||
val = map.get(key);
|
||||
if (val != null) {
|
||||
value = val.toString();
|
||||
check_env = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (check_env && NBEnvironment.INSTANCE.hasPropertyLayer() && NBEnvironment.INSTANCE.containsKey(key) ) {
|
||||
value = NBEnvironment.INSTANCE.get(key);
|
||||
}
|
||||
}
|
||||
value = (value==null? extractedDefaults.get(key) : value);
|
||||
}
|
||||
|
||||
@ -19,7 +19,7 @@ package io.nosqlbench.adapters.api.activityconfig.specifications;
|
||||
import com.google.gson.*;
|
||||
import com.google.gson.reflect.TypeToken;
|
||||
import io.nosqlbench.adapters.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawYamlLoader;
|
||||
import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsLoader;
|
||||
import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplateFormat;
|
||||
import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList;
|
||||
@ -145,7 +145,7 @@ public class YamlSpecValidator implements STAssemblyValidator {
|
||||
System.out.format("%-40s", "- checking yaml->json");
|
||||
|
||||
try {
|
||||
List<Map<String, Object>> docmaps = new RawYamlLoader().loadString(logger, yaml);
|
||||
List<Map<String, Object>> docmaps = new RawOpsLoader().loadStringMap(yaml);
|
||||
JsonElement elem = null;
|
||||
try {
|
||||
elem = JsonParser.parseString(json);
|
||||
|
||||
@ -98,7 +98,7 @@ public class NBAdvisorPoint<T> extends NBAdvisorPointOrBuilder<T> {
|
||||
}
|
||||
|
||||
public NBAdvisorPoint<T> logName() {
|
||||
if (resultLog.size() > 0) {
|
||||
if (resultLog.size() > 0 && advisorLevel != NBAdvisorLevel.none) {
|
||||
logger.info("Advisor: " + name + ": " + description);
|
||||
}
|
||||
return this;
|
||||
|
||||
@ -225,7 +225,7 @@ public class ConfigModel implements NBConfigModel {
|
||||
for (String configkey : config.keySet()) {
|
||||
Param<?> element = this.paramsByName.get(configkey);
|
||||
if (element != null) {
|
||||
String warning = "Config parameter '" + configkey + "' is also a " + type + ". Check for possible conflicts.";
|
||||
String warning = "Config parameter '" + configkey + "' is also a " + type + ". Check for possible conflicts.\n";
|
||||
NBAdvisorOutput.output(Level.WARN, warning);
|
||||
}
|
||||
}
|
||||
@ -299,7 +299,7 @@ public class ConfigModel implements NBConfigModel {
|
||||
for (String configkey : config.keySet()) {
|
||||
Param<?> element = this.paramsByName.get(configkey);
|
||||
String warning = "Unknown config parameter '" + configkey + "' in config model while configuring " + getOf().getSimpleName()
|
||||
+ ", possible parameter names are " + this.paramsByName.keySet() + ".";
|
||||
+ ", possible parameter names are " + this.paramsByName.keySet() + ".\n";
|
||||
if (element == null) {
|
||||
String warnonly = System.getenv("NB_CONFIG_WARNINGS_ONLY");
|
||||
logger.warn("WARNING: " + warning);
|
||||
|
||||
@ -65,6 +65,10 @@ public class NBEnvironment {
|
||||
|
||||
private final LinkedHashMap<String, String> references = new LinkedHashMap<>();
|
||||
|
||||
private final Map<String,String> properties = new HashMap<>();
|
||||
|
||||
private boolean hasPropertyLayer = false;
|
||||
|
||||
/**
|
||||
* These properties are well-defined in the Java specs. This map redirects common
|
||||
* environment variable names to the given system property. This allows
|
||||
@ -101,7 +105,9 @@ public class NBEnvironment {
|
||||
}
|
||||
|
||||
}
|
||||
System.setProperty(propname, value);
|
||||
// Instead of using System properties we keep an internal overlay.
|
||||
properties.put(propname, value);
|
||||
//System.setProperty(propname, value);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -153,6 +159,10 @@ public class NBEnvironment {
|
||||
}
|
||||
}
|
||||
if (name.contains(".")) {
|
||||
value = properties.get(name.toLowerCase());
|
||||
if (value != null) {
|
||||
return value;
|
||||
}
|
||||
value = System.getProperty(name.toLowerCase());
|
||||
if (value != null) {
|
||||
return value;
|
||||
@ -163,11 +173,19 @@ public class NBEnvironment {
|
||||
if (logger != null) {
|
||||
logger.debug("redirecting env var '" + name + "' to upper-case property '" + propName + "'");
|
||||
}
|
||||
value = properties.get(propName);
|
||||
if (value != null) {
|
||||
return value;
|
||||
}
|
||||
value = System.getProperty(propName);
|
||||
if (value != null) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
value = properties.get(name);
|
||||
if (value != null) {
|
||||
return value;
|
||||
}
|
||||
value = System.getProperty(name);
|
||||
if (value != null) {
|
||||
return value;
|
||||
@ -203,6 +221,14 @@ public class NBEnvironment {
|
||||
return (value != null);
|
||||
}
|
||||
|
||||
public boolean hasPropertyLayer() {
|
||||
return hasPropertyLayer;
|
||||
}
|
||||
|
||||
public void setPropertyLayer() {
|
||||
hasPropertyLayer = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* For the given word, if it contains a pattern with '$' followed by alpha, followed
|
||||
* by alphanumeric and underscores, replace this pattern with the system property or
|
||||
|
||||
@ -28,12 +28,16 @@ import io.nosqlbench.nb.api.labels.NBLabelSpec;
|
||||
import io.nosqlbench.nb.api.labels.NBLabels;
|
||||
import io.nosqlbench.nb.api.logging.NBLogLevel;
|
||||
import io.nosqlbench.nb.api.metadata.SystemId;
|
||||
import io.nosqlbench.nb.api.system.NBEnvironment;
|
||||
import io.nosqlbench.nb.api.system.NBStatePath;
|
||||
import io.nosqlbench.engine.api.metrics.IndicatorMode;
|
||||
import io.nosqlbench.engine.cmdstream.CmdType;
|
||||
import io.nosqlbench.nb.annotations.Maturity;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.security.InvalidParameterException;
|
||||
import java.util.*;
|
||||
@ -51,6 +55,8 @@ public class NBCLIOptions {
|
||||
private static final String NB_STATE_DIR = "--statedir";
|
||||
public static final String ARGS_FILE_DEFAULT = "$NBSTATEDIR/argsfile";
|
||||
private static final String INCLUDE = "--include";
|
||||
private static final String PROPERTIES = "--properties";
|
||||
private static final String ENVIRONMENT = "--environment";
|
||||
|
||||
private static final String userHome = System.getProperty("user.home");
|
||||
|
||||
@ -415,6 +421,15 @@ public class NBCLIOptions {
|
||||
arglist.removeFirst();
|
||||
annotatorsConfig = this.readWordOrThrow(arglist, "annotators config");
|
||||
break;
|
||||
case NBCLIOptions.PROPERTIES:
|
||||
arglist.removeFirst();
|
||||
final String properties = this.readWordOrThrow(arglist, "path to properties file");
|
||||
this.setProperties(properties);
|
||||
break;
|
||||
case NBCLIOptions.ENVIRONMENT:
|
||||
arglist.removeFirst();
|
||||
NBEnvironment.INSTANCE.setPropertyLayer();
|
||||
break;
|
||||
case NBCLIOptions.REPORT_GRAPHITE_TO:
|
||||
arglist.removeFirst();
|
||||
this.reportGraphiteTo = arglist.removeFirst();
|
||||
@ -774,6 +789,21 @@ public class NBCLIOptions {
|
||||
return this.wantsToIncludePaths.toArray(new String[0]);
|
||||
}
|
||||
|
||||
private void setProperties(String filePath) {
|
||||
Properties properties = new Properties();
|
||||
try (FileReader reader = new FileReader(filePath)) {
|
||||
properties.load(reader);
|
||||
} catch (FileNotFoundException e) {
|
||||
throw new BasicError("Property File '"+filePath+"' was not found.", e);
|
||||
} catch (IOException e) {
|
||||
throw new BasicError("IO Exception Loading Property File '"+filePath+"'.", e);
|
||||
}
|
||||
for (String key : properties.stringPropertyNames()) {
|
||||
NBEnvironment.INSTANCE.put(key, properties.getProperty(key));
|
||||
}
|
||||
NBEnvironment.INSTANCE.setPropertyLayer();
|
||||
}
|
||||
|
||||
private Map<String, String> parseLogLevelOverrides(final String levelsSpec) {
|
||||
final Map<String, String> levels = new HashMap<>();
|
||||
Arrays.stream(levelsSpec.split("[,;]")).forEach(kp -> {
|
||||
|
||||
@ -16,16 +16,12 @@
|
||||
|
||||
package io.nosqlbench.engine.cli.atfiles;
|
||||
|
||||
import io.nosqlbench.nb.api.nbio.Content;
|
||||
import io.nosqlbench.nb.api.nbio.NBIO;
|
||||
import io.nosqlbench.nb.api.nbio.NBPathsAPI;
|
||||
import io.nosqlbench.nb.api.system.NBEnvironment;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.snakeyaml.engine.v2.api.Load;
|
||||
import org.snakeyaml.engine.v2.api.LoadSettings;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
@ -38,7 +34,6 @@ import java.util.regex.Pattern;
|
||||
* TODO:ERRORHANDLER Cannot invoke "Object.getClass()" because "scopeOfInclude" is null on file full of comments only
|
||||
*/
|
||||
public class NBAtFile {
|
||||
// private final static Logger logger = LogManager.getLogger(NBAtFile.class);
|
||||
|
||||
/**
|
||||
* This will take a command line in raw form, which may include some arguments
|
||||
@ -63,7 +58,6 @@ public class NBAtFile {
|
||||
* @throws RuntimeException for any errors finding, traversing, parsing, or rendering values
|
||||
*/
|
||||
public static LinkedList<String> includeAt(LinkedList<String> processInPlace) {
|
||||
// logger.trace("argv stream before processing: " + String.join("|",processInPlace));
|
||||
ListIterator<String> iter = processInPlace.listIterator();
|
||||
while (iter.hasNext()) {
|
||||
String spec = iter.next();
|
||||
@ -76,8 +70,6 @@ public class NBAtFile {
|
||||
}
|
||||
}
|
||||
}
|
||||
// logger.trace("argv stream after atfile processing: "+ String.join("|",processInPlace));
|
||||
|
||||
return processInPlace;
|
||||
}
|
||||
private final static Pattern includePattern =
|
||||
@ -121,26 +113,36 @@ public class NBAtFile {
|
||||
String[] datapath = (dataPathSpec!=null && !dataPathSpec.isBlank()) ? dataPathSpec.split("(/|\\.)") : new String[] {};
|
||||
|
||||
String filename = Path.of(filepathSpec).getFileName().toString();
|
||||
if (filename.contains(".") && !(filename.toLowerCase().endsWith("yaml"))) {
|
||||
throw new RuntimeException("Only the yaml format and extension is supported for at-files." +
|
||||
if (filename.contains(".") && !( filename.toLowerCase().endsWith("yaml") || filename.toLowerCase().endsWith("properties") )) {
|
||||
throw new RuntimeException("Only the yaml and properties format and extension are supported for at-files." +
|
||||
" You specified " + filepathSpec);
|
||||
}
|
||||
filepathSpec=(filepathSpec.endsWith(".yaml") ? filepathSpec : filepathSpec+".yaml");
|
||||
filepathSpec=(filepathSpec.endsWith(".yaml") ? filepathSpec : (filepathSpec.endsWith(".properties") ? filepathSpec : filepathSpec+".yaml"));
|
||||
Path atPath = Path.of(filepathSpec);
|
||||
|
||||
String argsdata = "";
|
||||
try {
|
||||
argsdata = Files.readString(atPath);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
NBAtFileFormats fmt = (formatSpec!=null) ? NBAtFileFormats.valueOfSymbol(formatSpec) : NBAtFileFormats.Default;
|
||||
|
||||
Object scopeOfInclude = null;
|
||||
try {
|
||||
Load yaml = new Load(LoadSettings.builder().build());
|
||||
scopeOfInclude= yaml.loadFromString(argsdata);
|
||||
if (filepathSpec.endsWith(".properties")) {
|
||||
FileInputStream inputStream = new FileInputStream(atPath.toFile());
|
||||
Properties props = new Properties();
|
||||
props.load(inputStream);
|
||||
Map<String, String> propsMap = new HashMap<>();
|
||||
for (String key : props.stringPropertyNames()) {
|
||||
propsMap.put(key, props.getProperty(key));
|
||||
}
|
||||
scopeOfInclude = propsMap;
|
||||
} else {
|
||||
Load yaml = new Load(LoadSettings.builder().build());
|
||||
String argsdata = "";
|
||||
try {
|
||||
argsdata = Files.readString(atPath);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
scopeOfInclude = yaml.loadFromString(argsdata);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
@ -585,8 +585,7 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok
|
||||
if ((op != null ? 1 : 0) + (stmt != null ? 1 : 0) + (workload != null ? 1 : 0) > 1) {
|
||||
throw new OpConfigError("Only op, statement, or workload may be provided, not more than one.");
|
||||
}
|
||||
|
||||
|
||||
logger.debug("loadStmtsDocList #1");
|
||||
if (workload != null && OpsLoader.isJson(workload)) {
|
||||
workloadSource = "commandline: (workload/json):" + workload;
|
||||
return OpsLoader.loadString(workload, OpTemplateFormat.json, activityDef.getParams(), null);
|
||||
@ -597,11 +596,13 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok
|
||||
return OpsLoader.loadPath(workload, activityDef.getParams(), "activities");
|
||||
}
|
||||
|
||||
logger.debug("loadStmtsDocList #2");
|
||||
if (stmt != null) {
|
||||
workloadSource = "commandline: (stmt/inline): '" + stmt + "'";
|
||||
return OpsLoader.loadString(stmt, OpTemplateFormat.inline, activityDef.getParams(), null);
|
||||
}
|
||||
|
||||
logger.debug("loadStmtsDocList #3");
|
||||
if (op != null && OpsLoader.isJson(op)) {
|
||||
workloadSource = "commandline: (op/json): '" + op + "'";
|
||||
return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null);
|
||||
|
||||
@ -95,7 +95,7 @@ public class StandardActivity<R extends java.util.function.LongFunction, S> exte
|
||||
List<OpTemplate> opTemplates = loadOpTemplates(defaultAdapter);
|
||||
NBConfigModel combinedAdapterModel = ConfigModel.of(StandardActivity.class);
|
||||
for (OpTemplate ot : opTemplates) {
|
||||
logger.info(() -> "StandardActivity.opTemplate = "+ot);
|
||||
logger.debug(() -> "StandardActivity.opTemplate = "+ot);
|
||||
String driverName = ot.getOptionalStringParam("driver", String.class)
|
||||
.or(() -> ot.getOptionalStringParam("type", String.class))
|
||||
.orElse(defaultDriverName);
|
||||
@ -127,11 +127,11 @@ public class StandardActivity<R extends java.util.function.LongFunction, S> exte
|
||||
paramsAdvisor.validateAll(ot.getBindings().keySet());
|
||||
adapterlist.add(adapter);
|
||||
ParsedOp pop = new ParsedOp(ot, adapter.getConfiguration(), List.of(adapter.getPreprocessor()), this);
|
||||
logger.info("StandardActivity.pop="+pop);
|
||||
logger.debug("StandardActivity.pop="+pop);
|
||||
Optional<String> discard = pop.takeOptionalStaticValue("driver", String.class);
|
||||
pops.add(pop);
|
||||
}
|
||||
logger.info(() -> "StandardActivity.opTemplate loop complete");
|
||||
logger.debug(() -> "StandardActivity.opTemplate loop complete");
|
||||
|
||||
paramsAdvisor.setName("Workload", "Check parameters, template, and binding names")
|
||||
.logName().evaluate();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user