From 99bd45d83a2a6b7f27ec20e719716ad7496a3002 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 10 Feb 2023 22:44:58 -0600 Subject: [PATCH 1/8] add unit test for specific boxing, closes #1074 --- .../nb/api/config/ConfigModelTest.java | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/nb-api/src/test/java/io/nosqlbench/nb/api/config/ConfigModelTest.java b/nb-api/src/test/java/io/nosqlbench/nb/api/config/ConfigModelTest.java index df171cbf0..1c92acec5 100644 --- a/nb-api/src/test/java/io/nosqlbench/nb/api/config/ConfigModelTest.java +++ b/nb-api/src/test/java/io/nosqlbench/nb/api/config/ConfigModelTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,7 @@ package io.nosqlbench.nb.api.config; -import io.nosqlbench.api.config.standard.ConfigModel; -import io.nosqlbench.api.config.standard.NBConfiguration; -import io.nosqlbench.api.config.standard.Param; +import io.nosqlbench.api.config.standard.*; import org.junit.jupiter.api.Test; import java.util.List; @@ -36,6 +34,25 @@ public class ConfigModelTest { NBConfiguration cfg = cm.apply(Map.of("c", 232)); assertThat(cfg.getOptional("a")).isEmpty(); assertThat(cfg.get("c",int.class)).isEqualTo(232); + } + + @Test + public void testBoxingSupport() { + NBConfigModel model = ConfigModel.of(ConfigModelTest.class) + .add(Param.defaultTo("val",5)) + .asReadOnly(); + NBConfiguration config = model.apply(Map.of("val", 7)); + Integer val1 = config.getOrDefault("val", 8); + assertThat(val1).isEqualTo(7); + + int val2 = config.getOrDefault("val", 9); + assertThat(val2).isEqualTo(7); + + Integer val3 = config.get("val"); + assertThat(val3).isEqualTo(7); + + int val4 = config.get("val"); + assertThat(val4).isEqualTo(7); } } From a0e8a56a6aa0c686aaf9d6a4973fd948f3a11f8f Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 10 Feb 2023 22:45:50 -0600 Subject: [PATCH 2/8] properly filter log4j2-test --- engine-api/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/engine-api/pom.xml b/engine-api/pom.xml index f7c9a1717..46216858a 100644 --- a/engine-api/pom.xml +++ b/engine-api/pom.xml @@ -135,6 +135,14 @@ src/test/resources false + + true + src/test/resources + + log4j2-test.xml + + + From 46890232e28d8ea493ed18e321860831ef949e00 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 10 Feb 2023 22:46:43 -0600 Subject: [PATCH 3/8] verified via UAT; Test would be difficult without elaborate scaffolding; closes #994 --- .../api/activityimpl/uniform/StandardActivity.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java index b2377b9f1..a479c3f06 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java @@ -77,11 +77,12 @@ public class StandardActivity extends SimpleActivity implements List adapterlist = new ArrayList<>(); NBConfigModel supersetConfig = ConfigModel.of(StandardActivity.class).add(yamlmodel); + Optional defaultDriverOption = activityDef.getParams().getOptionalString("driver"); for (OpTemplate ot : opTemplates) { ParsedOp incompleteOpDef = new ParsedOp(ot, NBConfiguration.empty(), List.of()); String driverName = incompleteOpDef.takeOptionalStaticValue("driver", String.class) .or(() -> incompleteOpDef.takeOptionalStaticValue("type",String.class)) - .or(() -> activityDef.getParams().getOptionalString("driver")) + .or(() -> defaultDriverOption) .orElseThrow(() -> new OpConfigError("Unable to identify driver name for op template:\n" + ot)); // String driverName = ot.getOptionalStringParam("driver") @@ -109,7 +110,6 @@ public class StandardActivity extends SimpleActivity implements } supersetConfig.assertValidConfig(activityDef.getParams().getStringStringMap()); - DriverAdapter adapter = adapters.get(driverName); adapterlist.add(adapter); ParsedOp pop = new ParsedOp(ot, adapter.getConfiguration(), List.of(adapter.getPreprocessor())); @@ -117,6 +117,13 @@ public class StandardActivity extends SimpleActivity implements pops.add(pop); } + if (defaultDriverOption.isPresent()) { + long matchingDefault = mappers.keySet().stream().filter(n -> n.equals(defaultDriverOption.get())).count(); + if (matchingDefault==0) { + logger.warn("All op templates used a different driver than the default '" + defaultDriverOption.get()+"'"); + } + } + try { boolean strict = activityDef.getParams().getOptionalBoolean("strict").orElse(false); sequence = createOpSourceFromParsedOps(adapters, mappers, adapterlist, pops); From 982f162bff1e3290cdd17ebf4d4a9d9363730157 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 14 Feb 2023 17:48:21 -0600 Subject: [PATCH 4/8] implement a discard function, closes #1101 --- .../shared/from_long/to_object/Discard.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_object/Discard.java diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_object/Discard.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_object/Discard.java new file mode 100644 index 000000000..28e2eacf8 --- /dev/null +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_object/Discard.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.virtdata.library.basics.shared.from_long.to_object; + +import io.nosqlbench.virtdata.api.annotations.Categories; +import io.nosqlbench.virtdata.api.annotations.Category; +import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; + +import java.util.function.LongFunction; + + +/** + * This function takes a long input and ignores it. It returns a generic object which is meant to be used as input to + * other function which don't need a specific input. + */ +@ThreadSafeMapper +@Categories({Category.general}) +public class Discard implements LongFunction { + Object object = "discard"; + + @Override + public Object apply(long value) { + return object; + } +} From 715c63717f0c19dae754279dffc7cd4cb5f9632d Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 14 Feb 2023 17:48:48 -0600 Subject: [PATCH 5/8] enable recycles parameter for BaseDriverAdapter closes #974 --- .../engine/api/activityimpl/uniform/BaseDriverAdapter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java index 2dc6194a2..43d7ca4d0 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java @@ -160,6 +160,7 @@ public abstract class BaseDriverAdapter implements DriverAdapte .add(Param.optional("stride").setRegex("\\d+")) .add(Param.optional("striderate", String.class, "rate limit for strides per second")) .add(Param.optional("cycles").setRegex("\\d+[KMBGTPE]?|\\d+[KMBGTPE]?\\.\\.\\d+[KMBGTPE]?").setDescription("cycle interval to use")) + .add(Param.optional("recycles").setDescription("allow cycles to be re-used this many times")) .add(Param.optional(List.of("cyclerate", "targetrate", "rate"), String.class, "rate limit for cycles per second")) .add(Param.optional("phaserate", String.class, "rate limit for phases per second")) .add(Param.optional("seq", String.class, "sequencing algorithm")) From 21e73984a97db65877f1086ca7e1a534e22c4857 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 14 Feb 2023 17:49:06 -0600 Subject: [PATCH 6/8] implement better warning message for unused default driver --- .../engine/api/activityimpl/uniform/StandardActivity.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java index a479c3f06..f98f97756 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java @@ -18,6 +18,7 @@ package io.nosqlbench.engine.api.activityimpl.uniform; import io.nosqlbench.api.config.standard.*; import io.nosqlbench.api.engine.activityimpl.ActivityDef; +import io.nosqlbench.api.errors.BasicError; import io.nosqlbench.api.errors.OpConfigError; import io.nosqlbench.engine.api.activityapi.planning.OpSequence; import io.nosqlbench.engine.api.activityconfig.OpsLoader; @@ -66,9 +67,14 @@ public class StandardActivity extends SimpleActivity implements } ServiceLoader adapterLoader = ServiceLoader.load(DriverAdapter.class); - Optional defaultAdapter = activityDef.getParams().getOptionalString("driver") + Optional defaultDriverName = activityDef.getParams().getOptionalString("driver"); + Optional defaultAdapter = defaultDriverName .flatMap(s -> ServiceSelector.of(s, adapterLoader).get()); + if (defaultDriverName.isPresent() && defaultAdapter.isEmpty()) { + throw new BasicError("Unable to load default driver adapter '" + defaultDriverName.get() + "'"); + } + // HERE, op templates are loaded before drivers are loaded List opTemplates = loadOpTemplates(defaultAdapter); From 973bdc57f54cd9156d710280595e77ea08648b30 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 14 Feb 2023 17:49:10 -0600 Subject: [PATCH 7/8] clarify naming of state cache --- .../virtdata/library/basics/core/threadstate/SharedState.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/threadstate/SharedState.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/threadstate/SharedState.java index 4235fc066..07de116fb 100644 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/threadstate/SharedState.java +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/threadstate/SharedState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import java.util.concurrent.ConcurrentHashMap; /** * This provides common thread local instancing for sharing a thread local map across classes. + * This is being described as a Thread Local State Cache. */ public class SharedState { From a8a285a3f9993ac02842a5f8073f0714fb076fd7 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 14 Feb 2023 17:49:13 -0600 Subject: [PATCH 8/8] clarify optional enhancement func --- .../cqld4/opdispensers/Cqld4BaseOpDispenser.java | 10 +++++----- .../io/nosqlbench/engine/api/templating/ParsedOp.java | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java index 0b5e07e08..964f76d58 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -86,10 +86,10 @@ public abstract class Cqld4BaseOpDispenser extends BaseOpDispenser getEnhancedStmtFunc(LongFunction basefunc, ParsedOp op) { LongFunction partial = basefunc; - partial = op.enhanceEnum(partial, "cl", DefaultConsistencyLevel.class, Statement::setConsistencyLevel); - partial = op.enhanceEnum(partial, "consistency_level", DefaultConsistencyLevel.class, Statement::setConsistencyLevel); - partial = op.enhanceEnum(partial, "scl", DefaultConsistencyLevel.class, Statement::setSerialConsistencyLevel); - partial = op.enhanceEnum(partial, "serial_consistency_level", DefaultConsistencyLevel.class, Statement::setSerialConsistencyLevel); + partial = op.enhanceEnumOptionally(partial, "cl", DefaultConsistencyLevel.class, Statement::setConsistencyLevel); + partial = op.enhanceEnumOptionally(partial, "consistency_level", DefaultConsistencyLevel.class, Statement::setConsistencyLevel); + partial = op.enhanceEnumOptionally(partial, "scl", DefaultConsistencyLevel.class, Statement::setSerialConsistencyLevel); + partial = op.enhanceEnumOptionally(partial, "serial_consistency_level", DefaultConsistencyLevel.class, Statement::setSerialConsistencyLevel); partial = op.enhanceFuncOptionally(partial, "idempotent", Boolean.class, Statement::setIdempotent); partial = op.enhanceFuncOptionally(partial, "timeout", double.class, (statement, l) -> statement.setTimeout(Duration.ofMillis((long) (l * 1000L)))); partial = op.enhanceFuncOptionally(partial, "custom_payload", Map.class, Statement::setCustomPayload); diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedOp.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedOp.java index 41737a9b7..7fa125b4c 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedOp.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedOp.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -893,7 +893,7 @@ public class ParsedOp implements LongFunction>, StaticFieldReader * @param The enhancer function result type * @return an (optionally) enhanced base function */ - public > LongFunction enhanceEnum( + public > LongFunction enhanceEnumOptionally( LongFunction func, String field, Class type,