IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

cleanups and formatting

Browse Source
This commit is contained in:
Jonathan Shook
2025-01-08 12:46:04 -06:00
parent 0756b6e9d8
commit 5a12392102
16 changed files with 397 additions and 645 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java
View File

@@ -61,14 +61,21 @@ public class OpsLoader {
public static OpsDocList loadPath(String path, Map<String, ?> params, String... searchPaths) {
String[] extensions = path.indexOf('.') > -1 ? new String[]{} : YAML_EXTENSIONS;
ResolverChain chain = new ResolverChain(path);
Content<?> foundPath = NBIO.chain(chain.getChain()).searchPrefixes(searchPaths).pathname(chain.getPath()).extensionSet(extensions).first()
.orElseThrow(() -> new RuntimeException("Unable to load path '" + path + "'"));
Content<?> foundPath =
NBIO.chain(chain.getChain()).searchPrefixes(searchPaths).pathname(chain.getPath())
.extensionSet(extensions).first()
.orElseThrow(() -> new RuntimeException("Unable to load path '" + path + "'"));
OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(foundPath.getURI());
return loadString(foundPath.asString(), fmt, params, foundPath.getURI());
}
public static OpsDocList loadString(
final String sourceData, OpTemplateFormat fmt, Map<String, ?> params, URI srcuri) {
final String sourceData,
OpTemplateFormat fmt,
Map<String, ?> params,
URI srcuri
)
{
if (srcuri != null) {
logger.info("workload URI: '" + srcuri + "'");
@@ -113,9 +120,15 @@ public class OpsLoader {
}
int resultStatus = SjsonnetMain.main0(
injected.toArray(new String[0]), new DefaultParseCache(), inputStream, stdoutStream,
stderrStream, new os.Path(Path.of(System.getProperty("user.dir"))), Option.empty(),
Option.empty(), null
injected.toArray(new String[0]),
new DefaultParseCache(),
inputStream,
stdoutStream,
stderrStream,
new os.Path(Path.of(System.getProperty("user.dir"))),
Option.empty(),
Option.empty(),
null
);
String stdoutOutput = stdoutBuffer.toString(StandardCharsets.UTF_8);
@@ -133,16 +146,17 @@ public class OpsLoader {
}
}
if (!stderrOutput.isEmpty()) {
BasicError error = new BasicError(
"stderr output from jsonnet preprocessing: " + stderrOutput);
BasicError error =
new BasicError("stderr output from jsonnet preprocessing: " + stderrOutput);
if (resultStatus != 0) {
throw error;
} else {
logger.warn(error.toString(), error);
}
}
logger.info("jsonnet processing read '" + uri + "', rendered " + stdoutOutput.split(
"\n").length + " lines.");
logger.info(
"jsonnet processing read '" + uri + "', rendered " + stdoutOutput.split("\n").length
+ " lines.");
logger.trace("jsonnet result:\n" + stdoutOutput);
return stdoutOutput;
@@ -152,8 +166,11 @@ public class OpsLoader {
// into the parsers in a non-exception way
public static boolean isJson(String workload) {
try {
new GsonBuilder().setPrettyPrinting().create().fromJson(workload, Map.class);
return true;
if (workload.matches("^\\s*\\{.+")) {
new GsonBuilder().setPrettyPrinting().create().fromJson(workload, Map.class);
return true;
}
return false;
} catch (Exception e) {
return false;
}
@@ -163,8 +180,11 @@ public class OpsLoader {
// into the parsers in a non-exception way
public static boolean isYaml(String workload) {
try {
Object result = new Load(LoadSettings.builder().build()).loadFromString(workload);
return (result instanceof Map);
if (workload.indexOf('\n')>=0) {
Object result = new Load(LoadSettings.builder().build()).loadFromString(workload);
return (result instanceof Map);
}
return false;
} catch (Exception e) {
return false;
}

2
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/OpsOwner.java
View File

@@ -107,7 +107,7 @@ public class OpsOwner extends RawOpFields {
}
setOpsFieldByType(itemizedMaps);
} else if (object instanceof String) {
setOpsFieldByType(Map.of("stmt1", (String) object));
setOpsFieldByType(Map.of("stmt", (String) object));
} else {
throw new RuntimeException("Unknown object type: " + object.getClass());
}

4
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDef.java
View File

@@ -102,6 +102,10 @@ public class OpDef extends OpTemplate {
return tags;
}
/// Op template definitions are auto-tagged according to their placement within the workload
/// template. The block name and op name are both added as individual labels.
/// No other label should be added as before with auto-concatenation, since this breaks the
/// definitive behavior of tag filters over label combinations.
private LinkedHashMap<String, String> composeTags() {
LinkedHashMap<String, String> tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawOpDef.getTags(), block.getTags()));
tagsWithName.put("block",block.getName());

44
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpTemplates.java
View File

@@ -2,13 +2,13 @@ package io.nosqlbench.adapters.api.activityconfig.yaml;
/*
* Copyright (c) nosqlbench
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,7 +20,9 @@ package io.nosqlbench.adapters.api.activityconfig.yaml;
import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsDocList;
import io.nosqlbench.nb.api.tagging.TagFilter;
import java.util.function.Function;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
@@ -30,17 +32,19 @@ import java.util.stream.Stream;
/// [OpTemplates] is a list of selected op templates and their backing data.
///
/// It is a value type which makes it easy to /// get matching subsets of op templates according to tag filters, to combine them, etc.
/// It is a value type which makes it easy to get matching subsets of op templates according to
/// tag filters, to combine them, etc.
///
/// When a user selects an op template, they are expected to use the [TagFilter] mechanism.
/// Any such lookup methods should be implemented on this class.
public class OpTemplates implements Iterable<OpTemplate> {
public class OpTemplates implements Iterable<OpTemplate> {
private final ArrayList<OpTemplate> templates = new ArrayList<>();
private final static Logger logger = LogManager.getLogger(OpTemplates.class);
private final OpsDocList opsDocList;
public OpTemplates(OpsDocList opsDocList) {
opsDocList.getStmtDocs().stream().flatMap(d -> d.getOpTemplates().stream()).forEach(templates::add);
opsDocList.getStmtDocs().stream().flatMap(d -> d.getOpTemplates().stream())
.forEach(templates::add);
this.opsDocList = opsDocList;
}
@@ -50,7 +54,7 @@ public class OpTemplates implements Iterable<OpTemplate> {
}
public OpTemplates() {
this.opsDocList=new OpsDocList(new RawOpsDocList(List.of()));
this.opsDocList = new OpsDocList(new RawOpsDocList(List.of()));
}
public OpTemplates and(OpTemplates other) {
@@ -59,22 +63,21 @@ public class OpTemplates implements Iterable<OpTemplate> {
}
/**
* @param tagFilterSpec a comma-separated tag filter spec
* @return The list of all included op templates for all included blocks of in this document,
* including the inherited and overridden values from this doc and the parent block.
@param tagFilterSpec
a comma-separated tag filter spec
@return The list of all included op templates for all included blocks of in this document,
including the inherited and overridden values from this doc and the parent block.
*/
public OpTemplates matching(String tagFilterSpec, boolean logit) {
return matching(new TagFilter(tagFilterSpec), logit);
}
public OpTemplates matching(TagFilter tagFilter, boolean logit) {
List<OpTemplate> matchingOpTemplates = new ArrayList<>();
List<String> matchlog = new ArrayList<>();
templates.stream()
.map(tagFilter::matchesTaggedResult)
.peek(r -> matchlog.add(r.getLog()))
.filter(TagFilter.Result::matched)
.map(TagFilter.Result::getElement)
templates.stream().map(tagFilter::matchesTaggedResult).peek(r -> matchlog.add(r.getLog()))
.filter(TagFilter.Result::matched).map(TagFilter.Result::getElement)
.forEach(matchingOpTemplates::add);
if (logit) {
@@ -83,11 +86,11 @@ public class OpTemplates implements Iterable<OpTemplate> {
}
}
return new OpTemplates(matchingOpTemplates,opsDocList);
return new OpTemplates(matchingOpTemplates, opsDocList);
}
public Map<String,String> getDocBindings() {
public Map<String, String> getDocBindings() {
return opsDocList.getDocBindings();
}
@@ -112,9 +115,10 @@ public class OpTemplates implements Iterable<OpTemplate> {
return this.templates.isEmpty();
}
public OpTemplates transform(Function<OpTemplate,OpTemplate> transformF) {
List<OpTemplate> transformed = this.templates.stream().map(t -> transformF.apply(t)).toList();
return new OpTemplates(transformed,opsDocList);
public OpTemplates transform(Function<OpTemplate, OpTemplate> transformF) {
List<OpTemplate> transformed = this.templates.stream().map(t -> transformF.apply(t))
.toList();
return new OpTemplates(transformed, opsDocList);
}
}

24
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/components/core/NBBaseComponent.java
View File

@@ -103,25 +103,22 @@ public class NBBaseComponent extends NBBaseComponentMetrics
public synchronized NBComponent attachChild(NBComponent... children) {
for (NBComponent adding : children) {
logger.debug(
() -> "attaching " + adding.description() + " to parent " + this.description());
logger.debug(() -> "attaching " + adding.description() + " to parent "
+ this.description());
for (NBComponent extant : this.children) {
NBLabels eachLabels = extant.getComponentOnlyLabels();
NBLabels newLabels = adding.getComponentOnlyLabels();
if (eachLabels != null &&
newLabels != null &&
!eachLabels.isEmpty() &&
!newLabels.isEmpty() &&
adding.getComponentOnlyLabels().equals(extant.getComponentOnlyLabels()))
if (eachLabels != null && newLabels != null && !eachLabels.isEmpty()
&& !newLabels.isEmpty() && adding.getComponentOnlyLabels()
.equals(extant.getComponentOnlyLabels()))
{
throw new RuntimeException("""
Adding second child under already-defined labels is not allowed:
parent: (PARENTCLASS) PARENTNAME
extant: (EXTANTCLASS) EXTANTNAME
adding: (ADDINGCLASS) ADDINGNAME
"""
.replaceAll("PARENTCLASS", this.getClass().getSimpleName())
""".replaceAll("PARENTCLASS", this.getClass().getSimpleName())
.replaceAll("PARENTNAME", this.description())
.replaceAll("EXTANTCLASS", extant.getClass().getSimpleName())
.replaceAll("EXTANTNAME", extant.description())
@@ -137,10 +134,8 @@ public class NBBaseComponent extends NBBaseComponentMetrics
@Override
public NBComponent detachChild(NBComponent... children) {
for (NBComponent child : children) {
logger.debug(() -> "notifyinb before detaching " +
child.description() +
" from " +
this.description());
logger.debug(() -> "notifying before detaching " + child.description() + " from "
+ this.description());
child.beforeDetach();
}
for (NBComponent child : children) {
@@ -159,8 +154,7 @@ public class NBBaseComponent extends NBBaseComponentMetrics
@Override
public NBLabels getLabels() {
NBLabels effectiveLabels = (this.parent == null ? NBLabels.forKV() : parent.getLabels());
effectiveLabels = (this.labels == null) ?
effectiveLabels :
effectiveLabels = (this.labels == null) ? effectiveLabels :
effectiveLabels.and(this.labels);
return effectiveLabels;
}

1
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/components/core/NBComponentExecutionScope.java
View File

@@ -26,6 +26,7 @@ public class NBComponentExecutionScope implements AutoCloseable {
public NBComponentExecutionScope(NBComponent... components) {
this.components = components;
}
@Override
public void close() throws RuntimeException {
for (NBComponent component : components) {

17
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigModel.java
View File

@@ -18,6 +18,7 @@ package io.nosqlbench.nb.api.config.standard;
import io.nosqlbench.nb.api.advisor.NBAdvisorOutput;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.nb.api.errors.OpConfigError;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -63,10 +64,10 @@ public class ConfigModel implements NBConfigModel {
}
/**
* Add a param that, when present in a runtime configuration, will cause the config
* model to be expanded dynamically. This is for scenarios in which you have external
* configurable resources or templates which contain their own models that can
* only be known at runtime.
Add a param that, when present in a runtime configuration, will cause the config
model to be expanded dynamically. This is for scenarios in which you have external
configurable resources or templates which contain their own models that can
only be known at runtime.
*/
public NBConfigModel asReadOnly() {
@@ -88,7 +89,13 @@ public class ConfigModel implements NBConfigModel {
return ofType;
}
public static <T> T convertValueTo(String configName, String paramName, Object value, Class<T> type) {
public static <T> T convertValueTo(
String configName,
String paramName,
Object value,
Class<T> type
)
{
try {
if (type.isAssignableFrom(value.getClass())) {
return type.cast(value);

96
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBConfiguration.java
View File

@@ -47,7 +47,8 @@ public class NBConfiguration {
public static NBConfiguration empty() {
return new NBConfiguration(
ConfigModel.of(Object.class).asReadOnly(),
new LinkedHashMap<>());
new LinkedHashMap<>()
);
}
/**
@@ -76,22 +77,19 @@ public class NBConfiguration {
public <T> T getWithEnv(String name, Class<? extends T> vclass) {
T value = get(name, vclass);
if (value == null) {
}
if (value instanceof String) {
Optional<String> interpolated = NBEnvironment.INSTANCE.interpolate(value.toString());
if (interpolated.isEmpty()) {
throw new NBConfigError("Unable to interpolate env and sys props in '" +
value +
"'");
throw new NBConfigError(
"Unable to interpolate env and sys props in '" + value + "'");
}
String result = interpolated.get();
return ConfigModel.convertValueTo(
this.getClass().getSimpleName(),
name,
result,
vclass);
vclass
);
} else {
return value;
}
@@ -111,27 +109,23 @@ public class NBConfiguration {
public <T> T get(String name) {
Param<T> param = (Param<T>) model.getNamedParams().get(name);
if (param == null) {
throw new NBConfigError("Attempted to get parameter for name '" +
name +
"' but this parameter has no " +
"model defined for " +
this.getModel().getOf());
throw new NBConfigError(
"Attempted to get parameter for name '" + name + "' but this parameter has no "
+ "model defined for " + this.getModel().getOf());
}
// if (param.isRequired() && (param.getDefaultValue()==null) && )
Object object = this.data.get(name);
object = object != null ? object : param.getDefaultValue();
if (object == null && param.isRequired()) {
throw new NBConfigError("An object by name '" +
name +
"' was requested as required, and no value was" +
" defined for it. This user provided value must be set or otherwise marked optional or given a" +
" default value in the parameter model.");
throw new NBConfigError(
"An object by name '" + name + "' was requested as required, and no value was"
+ " defined for it. This user provided value must be set or otherwise marked optional or given a"
+ " default value in the parameter model.");
} else if (object == null && !param.isRequired()) {
throw new NBConfigError("An object by name '" +
name +
"' was requested as given by the config layer," +
" but no value was present, and no default was found in the config model. This is an ambiguous " +
"scenario. Either access the object as optional, or give it a default value. (code change)");
throw new NBConfigError(
"An object by name '" + name + "' was requested as given by the config layer,"
+ " but no value was present, and no default was found in the config model. This is an ambiguous "
+ "scenario. Either access the object as optional, or give it a default value. (code change)");
}
if (param.type.isInstance(object)) {
return (T) object;
@@ -140,14 +134,11 @@ public class NBConfiguration {
} else if (NBTypeConverter.canConvert(object, param.type)) {
return NBTypeConverter.convert(object, param.type);
} else {
throw new NBConfigError("Unable to assign config value for field '" +
name +
"' of type '" +
object.getClass().getCanonicalName() +
"' to the required return type '" +
param.type.getCanonicalName() +
"' as specified in the config model for '" +
model.getOf().getCanonicalName());
throw new NBConfigError(
"Unable to assign config value for field '" + name + "' of type '"
+ object.getClass().getCanonicalName() + "' to the required return type '"
+ param.type.getCanonicalName() + "' as specified in the config model for '"
+ model.getOf().getCanonicalName());
}
}
@@ -155,20 +146,16 @@ public class NBConfiguration {
Param<T> param = model.getParam(name);
if (param == null) {
throw new NBConfigError("Parameter named '" +
name +
"' is not valid for " +
model.getOf().getSimpleName() +
".");
throw new NBConfigError(
"Parameter named '" + name + "' is not valid for " + model.getOf().getSimpleName()
+ ".");
}
if ((!param.isRequired()) && param.getDefaultValue() == null) {
throw new RuntimeException("Non-optional get on optional parameter " +
name +
"' which has no default value while configuring " +
model.getOf() +
"." +
"\nTo avoid user impact, ensure that ConfigModel and NBConfigurable usage are aligned.");
throw new RuntimeException("""
Non-optional get on optional parameter 'PNAME' which has no default value while configuring OF.
To avoid user impact, ensure that ConfigModel and NBConfigurable usage are aligned.
""".replaceAll("PNAME", name).replaceAll("OF", model.getOf().getSimpleName()));
}
Object o = data.get(name);
@@ -203,9 +190,8 @@ public class NBConfiguration {
}
}
} else {
throw new NBConfigError("Parameter definition was not found for " +
Arrays.toString(names) +
".");
throw new NBConfigError(
"Parameter definition was not found for " + Arrays.toString(names) + ".");
}
}
if (o == null) {
@@ -222,11 +208,9 @@ public class NBConfiguration {
} else if (NBTypeConverter.canConvert(o, type)) {
return Optional.of((T) NBTypeConverter.convert(o, type));
} else {
throw new NBConfigError("config param " +
Arrays.toString(names) +
" was not assignable to class '" +
type.getCanonicalName() +
"'");
throw new NBConfigError(
"config param " + Arrays.toString(names) + " was not assignable to class '"
+ type.getCanonicalName() + "'");
}
}
@@ -239,11 +223,9 @@ public class NBConfiguration {
if (defaultValue.getClass().isAssignableFrom(o.getClass())) {
return (T) o;
}
throw new NBConfigError("config parameter '" +
name +
"' is not assignable to required type '" +
defaultValue.getClass() +
"'");
throw new NBConfigError(
"config parameter '" + name + "' is not assignable to required type '"
+ defaultValue.getClass() + "'");
}
public <T> T param(String name, Class<? extends T> vclass) {
@@ -275,7 +257,7 @@ public class NBConfiguration {
/// see [#update(Map)]
public <T> NBConfiguration update(String fieldName, T value) {
return update(Map.of(fieldName,value));
return update(Map.of(fieldName, value));
}
/// This will create a new configuration without modifying the existing one,
@@ -288,7 +270,7 @@ public class NBConfiguration {
///
/// Any holders of an updated configurations must maintain their own copies if necessary for
/// deltas.
public <T> NBConfiguration update(Map<String,Object> entries) {
public <T> NBConfiguration update(Map<String, Object> entries) {
NBConfiguration updated = model.apply(new LinkedHashMap<>(this.data) {
{
putAll(entries);