From 8ba0684b8dcefd774426d0d99f466d7741a3d714 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 7 Feb 2023 18:18:39 -0600 Subject: [PATCH 01/10] allow inline deprecation of activity params --- .../api/engine/activityimpl/ActivityDef.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java b/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java index 1ab2d6ebc..f9e2068b6 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java +++ b/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java @@ -18,6 +18,7 @@ package io.nosqlbench.api.engine.activityimpl; import io.nosqlbench.api.config.NBNamedElement; import io.nosqlbench.api.engine.util.Unit; +import io.nosqlbench.api.errors.BasicError; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -213,4 +214,22 @@ public class ActivityDef implements NBNamedElement { public String getName() { return getAlias(); } + + public ActivityDef deprecate(String deprecatedName, String newName) { + Object deprecatedParam = this.parameterMap.get(deprecatedName); + if (deprecatedParam==null) { + return this; + } + if (deprecatedParam instanceof CharSequence chars) { + if (this.parameterMap.containsKey(newName)) { + throw new BasicError("You have specified activity param '" + deprecatedName + "' in addition to the valid name '" + newName +"'. Remove '" + deprecatedName + "'."); + } else { + logger.warn("Auto replacing deprecated activity param " + deprecatedName + " with new '" + newName +"="+ chars +"."); + parameterMap.put(newName,parameterMap.remove(deprecatedParam)); + } + } else { + throw new BasicError("Can't replace deprecated name with value of type " + deprecatedName.getClass().getCanonicalName()); + } + return this; + } } From 47c1f2db5a95ec1cc9cbad5bb446f56c08b3dbf1 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 7 Feb 2023 18:19:03 -0600 Subject: [PATCH 02/10] update bindings-bigdecimal to work with J17 enums --- nbr/src/main/resources/examples/bindings-bigdecimal.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nbr/src/main/resources/examples/bindings-bigdecimal.yaml b/nbr/src/main/resources/examples/bindings-bigdecimal.yaml index 2d05000b9..47f388678 100644 --- a/nbr/src/main/resources/examples/bindings-bigdecimal.yaml +++ b/nbr/src/main/resources/examples/bindings-bigdecimal.yaml @@ -10,13 +10,16 @@ scenarios: # a scale parameter or a custom MathContext, but not both. The scale parameter # is not supported for String or Double input forms. +# As of Java 17, the roundingMode enums are UP DOWN CEILING FLOOR HALF_UP HALF_DOWN HALF_EVEN UNNECESSARY +# and the precision must be zero or greater + bindings: # convert an example double with some fractional values, then convert it to BigDecimal from_double: ToDouble(); Div(100.0d); ToBigDecimal(); # convert doubles to BigDecimal, with custom precision and rounding - from_double_custom5: ToDouble(); Div(100.0d); ToBigDecimal('precision=5 roundingMode=HALF'); + from_double_custom5: ToDouble(); Div(100.0d); ToBigDecimal('precision=5 roundingMode=HALF_EVEN'); # convert directly to BigDecimal from long as whole numbers from_long: ToBigDecimal(); From 0d955b2158ae7d654f9a3e2a89e7fb167af0d194 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 7 Feb 2023 18:19:53 -0600 Subject: [PATCH 03/10] remove incompatible example from bindings-bytebuffers --- nbr/src/main/resources/examples/bindings-bytebuffers.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/nbr/src/main/resources/examples/bindings-bytebuffers.yaml b/nbr/src/main/resources/examples/bindings-bytebuffers.yaml index 19e3f51a0..510809000 100644 --- a/nbr/src/main/resources/examples/bindings-bytebuffers.yaml +++ b/nbr/src/main/resources/examples/bindings-bytebuffers.yaml @@ -21,9 +21,6 @@ bindings: # control the size of the extracted sample with a hash range hashed_bb_sizerange: ByteBufferSizedHashed(HashRange(1,10)); ToHexString(); - # control the size of the extracted sample based on a continuous function - hashed_bb_size_normal: ByteBufferSizedHashed(Normal(5.0, 1.0)); ToHexString(); - # control the size of the extracted sample based on a discrete function hashed_bb_size_binomial: ByteBufferSizedHashed(Binomial(8,0.5)); ToHexString(); From c78ff2118b5d1d20dfff2828b2c76e8bb00370c0 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 7 Feb 2023 18:20:13 -0600 Subject: [PATCH 04/10] improve error for MathContext init --- .../library/basics/shared/util/MathContextReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/util/MathContextReader.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/util/MathContextReader.java index df2c6bcbe..7e124231b 100644 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/util/MathContextReader.java +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/util/MathContextReader.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,7 +39,7 @@ public class MathContextReader { } } catch (IllegalArgumentException iae) { throw new BasicError("'" + name + "' was not a valid format for a new MathContext(String), try something " + - "like 'precision=17 roundingMode=UP"); + "like 'precision=17 roundingMode=UP', original exception:" + iae); } } From c45e3ff9643a6ab6967ec98d10aaf2ad840ea109 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 7 Feb 2023 18:20:49 -0600 Subject: [PATCH 05/10] deprecate yaml->workload and type->driver, and make loader more robust --- .../api/activityimpl/uniform/StandardActivity.java | 1 + .../activityimpl/uniform/StandardActivityType.java | 13 ++++++++----- .../core/lifecycle/activity/ActivityLoader.java | 3 ++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java index f21c8a468..b2377b9f1 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java @@ -80,6 +80,7 @@ public class StandardActivity extends SimpleActivity implements for (OpTemplate ot : opTemplates) { ParsedOp incompleteOpDef = new ParsedOp(ot, NBConfiguration.empty(), List.of()); String driverName = incompleteOpDef.takeOptionalStaticValue("driver", String.class) + .or(() -> incompleteOpDef.takeOptionalStaticValue("type",String.class)) .or(() -> activityDef.getParams().getOptionalString("driver")) .orElseThrow(() -> new OpConfigError("Unable to identify driver name for op template:\n" + ot)); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java index c319852e6..b45b91de6 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java @@ -16,15 +16,15 @@ package io.nosqlbench.engine.api.activityimpl.uniform; +import io.nosqlbench.api.config.standard.NBConfigModel; +import io.nosqlbench.api.config.standard.NBConfiguration; +import io.nosqlbench.api.config.standard.NBReconfigurable; +import io.nosqlbench.api.engine.activityimpl.ActivityDef; import io.nosqlbench.engine.api.activityapi.core.ActionDispenser; import io.nosqlbench.engine.api.activityapi.core.ActivityType; import io.nosqlbench.engine.api.activityconfig.OpsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList; -import io.nosqlbench.api.engine.activityimpl.ActivityDef; import io.nosqlbench.engine.api.activityimpl.SimpleActivity; -import io.nosqlbench.api.config.standard.NBConfigModel; -import io.nosqlbench.api.config.standard.NBConfiguration; -import io.nosqlbench.api.config.standard.NBReconfigurable; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -39,7 +39,10 @@ public class StandardActivityType> extends Simpl private final Map adapters = new HashMap<>(); public StandardActivityType(DriverAdapter adapter, ActivityDef activityDef) { - super(activityDef); + super(activityDef + .deprecate("type","driver") + .deprecate("yaml", "workload") + ); this.adapters.put(adapter.getAdapterName(),adapter); if (adapter instanceof ActivityDefAware) { ((ActivityDefAware) adapter).setActivityDef(activityDef); diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityLoader.java b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityLoader.java index fb0f9bd95..97634ced7 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityLoader.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityLoader.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,6 +41,7 @@ public class ActivityLoader { } public synchronized Activity loadActivity(ActivityDef activityDef) { + activityDef= activityDef.deprecate("yaml","workload").deprecate("type","driver"); Activity activity = new StandardActivityType(activityDef).getAssembledActivity(activityDef, activityMap); activityMap.put(activity.getAlias(),activity); logger.debug("Resolved activity for alias '" + activityDef.getAlias() + "'"); From b73933030376aed95eefa97d357443a24a5181b9 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 7 Feb 2023 19:15:56 -0600 Subject: [PATCH 06/10] allow inline deprecation of activity params --- .../io/nosqlbench/api/engine/activityimpl/ActivityDef.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java b/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java index f9e2068b6..26bca1168 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java +++ b/nb-api/src/main/java/io/nosqlbench/api/engine/activityimpl/ActivityDef.java @@ -224,8 +224,8 @@ public class ActivityDef implements NBNamedElement { if (this.parameterMap.containsKey(newName)) { throw new BasicError("You have specified activity param '" + deprecatedName + "' in addition to the valid name '" + newName +"'. Remove '" + deprecatedName + "'."); } else { - logger.warn("Auto replacing deprecated activity param " + deprecatedName + " with new '" + newName +"="+ chars +"."); - parameterMap.put(newName,parameterMap.remove(deprecatedParam)); + logger.warn("Auto replacing deprecated activity param '" + deprecatedName + "="+ chars +"' with new '" + newName +"="+ chars +"'."); + parameterMap.put(newName,parameterMap.remove(deprecatedName)); } } else { throw new BasicError("Can't replace deprecated name with value of type " + deprecatedName.getClass().getCanonicalName()); From 663a721f1d63b1bc3664fecbb9b3344aae3297f6 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 7 Feb 2023 19:16:02 -0600 Subject: [PATCH 07/10] data path correction --- .../io/nosqlbench/virtdata/library/realer/CountryNames.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/virtdata-lib-realer/src/main/java/io/nosqlbench/virtdata/library/realer/CountryNames.java b/virtdata-lib-realer/src/main/java/io/nosqlbench/virtdata/library/realer/CountryNames.java index aacba6d47..295e4b56b 100644 --- a/virtdata-lib-realer/src/main/java/io/nosqlbench/virtdata/library/realer/CountryNames.java +++ b/virtdata-lib-realer/src/main/java/io/nosqlbench/virtdata/library/realer/CountryNames.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ public class CountryNames extends CSVSampler implements LongFunction { @Example("CountryNames()") public CountryNames() { - super("COUNTRY_NAME","n/a","name","countries.csv"); + super("COUNTRY_NAME","n/a","name","data/countries.csv"); } } From 262ac17b985228f39a8629d23ad2c47bc3065d8f Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 7 Feb 2023 19:16:05 -0600 Subject: [PATCH 08/10] correct binding regexes --- .../main/resources/examples/bindings-premade.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nbr/src/main/resources/examples/bindings-premade.yaml b/nbr/src/main/resources/examples/bindings-premade.yaml index ec16c5271..8343a1f85 100644 --- a/nbr/src/main/resources/examples/bindings-premade.yaml +++ b/nbr/src/main/resources/examples/bindings-premade.yaml @@ -5,12 +5,12 @@ description: | scenarios: default: run driver===stdout format=readout - names: run driver===stdout format=readout bindings=names - cities: run driver===stdout format=readout bindings=cities - states: run driver===stdout format=readout bindings=states - zips: run driver===stdout format=readout bindings=zips - counties: run driver===stdout format=readout bindings=counties - countries: run driver===stdout format=readout bindings=country + names: run driver===stdout format=readout bindings='names.*' + cities: run driver===stdout format=readout bindings='cities.*' + states: run driver===stdout format=readout bindings='states.*' + zips: run driver===stdout format=readout bindings='zips.*' + counties: run driver===stdout format=readout bindings='counties.*' + countries: run driver===stdout format=readout bindings='country.*' # Each binding is named, so the bindings is a map of names to From da866eae6c31eda4d5b575a2b3b820297b5c568d Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 7 Feb 2023 19:16:07 -0600 Subject: [PATCH 09/10] make bindings-utilities more useful --- nbr/src/main/resources/examples/bindings-utilities.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbr/src/main/resources/examples/bindings-utilities.yaml b/nbr/src/main/resources/examples/bindings-utilities.yaml index 3f78c04cd..adfed6198 100644 --- a/nbr/src/main/resources/examples/bindings-utilities.yaml +++ b/nbr/src/main/resources/examples/bindings-utilities.yaml @@ -21,10 +21,10 @@ bindings: # The TypeOf() function tells you the java class of its input - typeof: TypeOf(); + typeof: ToJavaInstant(); TypeOf(); # The Show() function provides a snapshot of what is in the thread-local # variable map as a String - show: Show(); + show: SaveLong('var42'); Show(); From d7c1a36586ce1592189938fcff23bb8bd160a2a5 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 7 Feb 2023 19:16:10 -0600 Subject: [PATCH 10/10] typo fix --- .../io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java b/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java index ae5f83050..16ca54d9d 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java @@ -139,7 +139,7 @@ public class NBCLIScenarioParser { if (selectedScenario.containsKey(stepname)) { namedSteps.put(stepname,selectedScenario.get(stepname)); } else { - throw new BasicError("Unable to find named scenario.step'" + scenarioName + "' in workload '" + workloadName + throw new BasicError("Unable to find named scenario.step '" + scenarioName + "' in workload '" + workloadName + "', but you can pick from one of: " + selectedScenario.keySet().stream().map(n -> nameparts[0].concat(".").concat(n)).collect(Collectors.joining(", "))); } }