From 9afe376b4f797129013e6f411ad568e59781d568 Mon Sep 17 00:00:00 2001 From: Snyk bot Date: Wed, 23 Nov 2022 22:03:43 +0100 Subject: [PATCH 01/45] fix: upgrade io.swagger.parser.v3:swagger-parser from 2.1.4 to 2.1.7 (#798) Snyk has created this PR to upgrade io.swagger.parser.v3:swagger-parser from 2.1.4 to 2.1.7. See this package in Maven Repository: https://mvnrepository.com/artifact/io.swagger.parser.v3/swagger-parser/ See this project in Snyk: https://app.snyk.io/org/jshook/project/b808ba5a-fa96-49c2-9cae-4c2c2f8a1384?utm_source=github&utm_medium=referral&page=upgrade-pr --- engine-rest/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine-rest/pom.xml b/engine-rest/pom.xml index 25583745a..e11f4d201 100644 --- a/engine-rest/pom.xml +++ b/engine-rest/pom.xml @@ -23,7 +23,7 @@ io.swagger.parser.v3 swagger-parser - 2.1.4 + 2.1.7 From f39ba9331c00a0b4de00a58f03f3f88cc1ea1ff3 Mon Sep 17 00:00:00 2001 From: Snyk bot Date: Wed, 23 Nov 2022 22:04:25 +0100 Subject: [PATCH 02/45] fix: upgrade io.swagger.core.v3:swagger-models from 2.2.3 to 2.2.4 (#799) Snyk has created this PR to upgrade io.swagger.core.v3:swagger-models from 2.2.3 to 2.2.4. See this package in Maven Repository: https://mvnrepository.com/artifact/io.swagger.core.v3/swagger-models/ See this project in Snyk: https://app.snyk.io/org/jshook/project/b808ba5a-fa96-49c2-9cae-4c2c2f8a1384?utm_source=github&utm_medium=referral&page=upgrade-pr --- engine-rest/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine-rest/pom.xml b/engine-rest/pom.xml index e11f4d201..a45106357 100644 --- a/engine-rest/pom.xml +++ b/engine-rest/pom.xml @@ -29,7 +29,7 @@ io.swagger.core.v3 swagger-models - 2.2.3 + 2.2.4 From 7026920587254457831868f7075c4d9196328fb6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 22:21:48 +0000 Subject: [PATCH 03/45] build(deps): bump postgresql in /driver-cockroachdb Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.5.0 to 42.5.1. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.5.0...REL42.5.1) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- driver-cockroachdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-cockroachdb/pom.xml b/driver-cockroachdb/pom.xml index afc953cfb..d3de3ab44 100644 --- a/driver-cockroachdb/pom.xml +++ b/driver-cockroachdb/pom.xml @@ -42,7 +42,7 @@ org.postgresql postgresql - 42.5.0 + 42.5.1 From 64f11bed1ba4b385736a0a3cbe4cf21242259c08 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Wed, 23 Nov 2022 22:44:20 +0000 Subject: [PATCH 04/45] fix: upgrade com.github.oshi:oshi-core-java11 from 6.2.2 to 6.3.1 Snyk has created this PR to upgrade com.github.oshi:oshi-core-java11 from 6.2.2 to 6.3.1. See this package in Maven Repository: https://mvnrepository.com/artifact/com.github.oshi/oshi-core-java11/ See this project in Snyk: https://app.snyk.io/org/jshook/project/fc9e1bd9-1d9a-474f-bde4-efb614c52ffe?utm_source=github&utm_medium=referral&page=upgrade-pr --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index e0dcc4f7e..485266ccb 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -265,7 +265,7 @@ com.github.oshi oshi-core-java11 - 6.2.2 + 6.3.1 From 96be7811a622ee79a9b09e2c55638bfe9a137e20 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Wed, 23 Nov 2022 22:44:23 +0000 Subject: [PATCH 05/45] fix: upgrade com.google.code.gson:gson from 2.9.1 to 2.10 Snyk has created this PR to upgrade com.google.code.gson:gson from 2.9.1 to 2.10. See this package in Maven Repository: https://mvnrepository.com/artifact/com.google.code.gson/gson/ See this project in Snyk: https://app.snyk.io/org/jshook/project/fc9e1bd9-1d9a-474f-bde4-efb614c52ffe?utm_source=github&utm_medium=referral&page=upgrade-pr --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index e0dcc4f7e..f766cc3bf 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -271,7 +271,7 @@ com.google.code.gson gson - 2.9.1 + 2.10 From ec858f69dfdfe1b4d1cf9e5aa4db6a8ce75cc9c9 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Wed, 23 Nov 2022 22:44:28 +0000 Subject: [PATCH 06/45] fix: upgrade info.picocli:picocli from 4.6.3 to 4.7.0 Snyk has created this PR to upgrade info.picocli:picocli from 4.6.3 to 4.7.0. See this package in Maven Repository: https://mvnrepository.com/artifact/info.picocli/picocli/ See this project in Snyk: https://app.snyk.io/org/jshook/project/fc9e1bd9-1d9a-474f-bde4-efb614c52ffe?utm_source=github&utm_medium=referral&page=upgrade-pr --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index e0dcc4f7e..b6e1296ee 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -98,7 +98,7 @@ info.picocli picocli - 4.6.3 + 4.7.0 From d622670f7edddc37ab573d353f64c2b0130081e9 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Wed, 23 Nov 2022 22:44:31 +0000 Subject: [PATCH 07/45] fix: upgrade org.graalvm.sdk:graal-sdk from 21.3.3.1 to 21.3.4 Snyk has created this PR to upgrade org.graalvm.sdk:graal-sdk from 21.3.3.1 to 21.3.4. See this package in Maven Repository: https://mvnrepository.com/artifact/org.graalvm.sdk/graal-sdk/ See this project in Snyk: https://app.snyk.io/org/jshook/project/fc9e1bd9-1d9a-474f-bde4-efb614c52ffe?utm_source=github&utm_medium=referral&page=upgrade-pr --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index e0dcc4f7e..4451b5073 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -360,7 +360,7 @@ org.graalvm.sdk graal-sdk - 21.3.3.1 + 21.3.4 org.graalvm.js From fec156a25db73cc3679eed5ac403954adedbc2d8 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 24 Nov 2022 02:55:56 +0000 Subject: [PATCH 08/45] fix: upgrade io.netty:netty-handler from 4.1.82.Final to 4.1.84.Final Snyk has created this PR to upgrade io.netty:netty-handler from 4.1.82.Final to 4.1.84.Final. See this package in Maven Repository: https://mvnrepository.com/artifact/io.netty/netty-handler/ See this project in Snyk: https://app.snyk.io/org/jshook/project/faf9c31c-14e2-456a-9f66-9dda91c923d9?utm_source=github&utm_medium=referral&page=upgrade-pr --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index e0dcc4f7e..c775b6d06 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -189,7 +189,7 @@ io.netty netty-handler - 4.1.82.Final + 4.1.84.Final From 82c253acb8fd71ecdcaab7a2cb46bfcf46290039 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Fri, 25 Nov 2022 18:29:25 +0000 Subject: [PATCH 09/45] fix: driver-cockroachdb/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-ORGPOSTGRESQL-3146847 --- driver-cockroachdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-cockroachdb/pom.xml b/driver-cockroachdb/pom.xml index afc953cfb..d3de3ab44 100644 --- a/driver-cockroachdb/pom.xml +++ b/driver-cockroachdb/pom.xml @@ -42,7 +42,7 @@ org.postgresql postgresql - 42.5.0 + 42.5.1 From 9bca349e4f18ed201819caa8e726f30ca91462e5 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 30 Nov 2022 12:19:48 -0600 Subject: [PATCH 10/45] use correct value in exception handlers --- .../src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java index cf9d3d694..570b564fc 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java @@ -139,7 +139,7 @@ public class Cqld4Space implements AutoCloseable { try { username = Files.readAllLines(path).get(0); } catch (IOException e) { - String error = "Error while reading username from file:" + usernameOpt.get(); + String error = "Error while reading username from file:" + path; logger.error(error, e); throw new RuntimeException(e); } @@ -155,7 +155,7 @@ public class Cqld4Space implements AutoCloseable { try { password = Files.readAllLines(path).get(0); } catch (IOException e) { - String error = "Error while reading password from file:" + passfileOpt.get(); + String error = "Error while reading password from file:" + path; logger.error(error, e); throw new RuntimeException(e); } From c893398398bd6df10bedbd0d739650a879159896 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 1 Dec 2022 09:59:51 +0000 Subject: [PATCH 11/45] fix: upgrade joda-time:joda-time from 2.11.2 to 2.12.1 Snyk has created this PR to upgrade joda-time:joda-time from 2.11.2 to 2.12.1. See this package in Maven Repository: https://mvnrepository.com/artifact/joda-time/joda-time/ See this project in Snyk: https://app.snyk.io/org/jshook/project/69c44849-9219-4214-b381-2530d26908cf?utm_source=github&utm_medium=referral&page=upgrade-pr --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 0e976e4ee..5d1e3e76a 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -301,7 +301,7 @@ joda-time joda-time - 2.11.2 + 2.12.1 From 34b51b1989471868e7b3e5899e4233aa740cbb11 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 1 Dec 2022 12:21:34 +0000 Subject: [PATCH 12/45] fix: upgrade com.datastax.oss:java-driver-query-builder from 4.14.1 to 4.15.0 Snyk has created this PR to upgrade com.datastax.oss:java-driver-query-builder from 4.14.1 to 4.15.0. See this package in Maven Repository: https://mvnrepository.com/artifact/com.datastax.oss/java-driver-query-builder/ See this project in Snyk: https://app.snyk.io/org/jshook/project/f2d62bab-282e-4a32-8fb9-1a067980e81a?utm_source=github&utm_medium=referral&page=upgrade-pr --- adapter-cqld4/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapter-cqld4/pom.xml b/adapter-cqld4/pom.xml index a0125cf8a..f682c6d46 100644 --- a/adapter-cqld4/pom.xml +++ b/adapter-cqld4/pom.xml @@ -67,7 +67,7 @@ com.datastax.oss java-driver-query-builder - 4.14.1 + 4.15.0 From f413a5af769626700fc545e0c5c64fc7393053d5 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 1 Dec 2022 12:21:38 +0000 Subject: [PATCH 13/45] fix: upgrade org.codehaus.groovy:groovy from 3.0.12 to 3.0.13 Snyk has created this PR to upgrade org.codehaus.groovy:groovy from 3.0.12 to 3.0.13. See this package in Maven Repository: https://mvnrepository.com/artifact/org.codehaus.groovy/groovy/ See this project in Snyk: https://app.snyk.io/org/jshook/project/f2d62bab-282e-4a32-8fb9-1a067980e81a?utm_source=github&utm_medium=referral&page=upgrade-pr --- adapter-cqld4/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapter-cqld4/pom.xml b/adapter-cqld4/pom.xml index a0125cf8a..eaa97e44f 100644 --- a/adapter-cqld4/pom.xml +++ b/adapter-cqld4/pom.xml @@ -44,7 +44,7 @@ org.codehaus.groovy groovy - 3.0.12 + 3.0.13 From d88359a6a5d11bd2bdb6569682870d4fb4f83bc2 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 1 Dec 2022 12:21:42 +0000 Subject: [PATCH 14/45] fix: upgrade org.snakeyaml:snakeyaml-engine from 2.4 to 2.5 Snyk has created this PR to upgrade org.snakeyaml:snakeyaml-engine from 2.4 to 2.5. See this package in Maven Repository: https://mvnrepository.com/artifact/org.snakeyaml/snakeyaml-engine/ See this project in Snyk: https://app.snyk.io/org/jshook/project/f2d62bab-282e-4a32-8fb9-1a067980e81a?utm_source=github&utm_medium=referral&page=upgrade-pr --- adapter-cqld4/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapter-cqld4/pom.xml b/adapter-cqld4/pom.xml index a0125cf8a..869aea6f1 100644 --- a/adapter-cqld4/pom.xml +++ b/adapter-cqld4/pom.xml @@ -73,7 +73,7 @@ org.snakeyaml snakeyaml-engine - 2.4 + 2.5 From 6ad405d9f2c9a1d4e7ba5db712bdf1963180da66 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 1 Dec 2022 17:12:09 +0000 Subject: [PATCH 15/45] fix: upgrade org.glassfish.jersey.core:jersey-server from 3.0.8 to 3.1.0 Snyk has created this PR to upgrade org.glassfish.jersey.core:jersey-server from 3.0.8 to 3.1.0. See this package in Maven Repository: https://mvnrepository.com/artifact/org.glassfish.jersey.core/jersey-server/ See this project in Snyk: https://app.snyk.io/org/jshook/project/03cbee46-d5d2-41d3-89cc-a2ad77ab807a?utm_source=github&utm_medium=referral&page=upgrade-pr --- docsys/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsys/pom.xml b/docsys/pom.xml index 528eb0cb8..1d5147e12 100644 --- a/docsys/pom.xml +++ b/docsys/pom.xml @@ -22,7 +22,7 @@ docsys http://nosqlbench.io/ - 3.0.8 + 3.1.0 From b4b09048f6e71eafd679cca9ffa6f80517bcddce Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Thu, 1 Dec 2022 12:11:07 -0600 Subject: [PATCH 16/45] update docs for op types --- .../uniform/flowtypes/ChainingOp.java | 23 +++++++--- .../uniform/flowtypes/CycleOp.java | 45 ++++++++++--------- .../activityimpl/uniform/flowtypes/Op.java | 17 ++++--- .../uniform/flowtypes/RunnableOp.java | 14 ++++++ 4 files changed, 61 insertions(+), 38 deletions(-) diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/ChainingOp.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/ChainingOp.java index 0edcfa34c..43c1aeed4 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/ChainingOp.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/ChainingOp.java @@ -19,16 +19,25 @@ package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes; import java.util.function.Function; /** - * Run a function on the current cached result and replace it - * with the result of the function. Functions are one way of invoking + *

ChainingOp: f(I) -> O

+ *

+ * Run a function on the current cached result in the current thread and replace it + * with the result of the function. ChainingOps are one way of invoking * logic within a cycle. However, they are not intended to stand alone. - * A CycleFunction must always have an input to work on. This input is - * provided by a Supplier as optionally implemented by an Op + * A ChainingOp must always have an input to work on, + * provided by either a {@link CycleOp} OR another call to a {@link ChainingOp}

* - * @param Some input type. + * @param Some input type, as determined by a previous {@link CycleOp} or {@link ChainingOp} on the same thread. * @param Some output type. */ -public interface ChainingOp extends Op, Function { +public interface ChainingOp extends Op, Function { + + /** + * Transform a value from a previous action and provide the result for a subsequent action. + * + * @param lastResult object form a previous operation or action + * @return a new result + */ @Override - O apply(I i); + O apply(I lastResult); } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/CycleOp.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/CycleOp.java index e6bf71e29..58dfe8e1f 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/CycleOp.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/CycleOp.java @@ -19,32 +19,33 @@ package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes; import java.util.function.LongFunction; /** - * A CycleRunnable is simply a variation of a Runnable type. - * The main difference is that it is supplied with the cycle - * as input. + *

CycleOp: f(cycle) -> T

+ *

A CycleOp of T is an operation which takes a long input value + * and produces a value of type T. It is implemented as + * {@link LongFunction} of T.

+ * + *

This variant of {@link Op} has the ability to see the cycle + * which was previously used to select the op implementation.

+ * + *

It also has the ability to emit an value which can be seen a subsequent operation, if + * and only if it is a {@link ChainingOp}s.

+ * + *

Designer Notes

+ *

+ * If you are using the value in this call to select a specific type of behavior, it is very + * likely a candidate for factoring into separate op implementations. + * The {@link io.nosqlbench.engine.api.activityimpl.OpMapper} + * and {@link io.nosqlbench.engine.api.activityimpl.OpDispenser} abstractions are meant to move + * op type selection and scheduling to earlier in the activity. + *

+ * */ public interface CycleOp extends Op, LongFunction { -// /** -// *

Run an action for the given cycle. The cycle is provided for anecdotal -// * usage such as logging and debugging. It is valid to use the cycle value in these places, -// * but you should not use it to determine the logic of what is run. The mechanism -// * for doing this is provided in {@link io.nosqlbench.engine.api.activityimpl.OpMapper} -// * and {@link io.nosqlbench.engine.api.activityimpl.OpDispenser} types.

-// * -// * -// * @param cycle The cycle value for which an operation is run -// */ -//// * This method should do the same thing that {@link #apply(long)} does, except that -//// * there is no need to prepare or return a result. This is the form that will be called -//// * if there is no chaining operation to consume the result of this operation. -// void accept(long cycle); - /** - *

Run an action for the given cycle. The cycle - * value is only to be used for anecdotal presentation. This form is called - * when there is a chaining operation which will do something with this result.

+ *

Run an action for the given cycle.

+ * * @param value The cycle value for which an operation is run - * @return A result which is the native result type for the underlying driver. + * @return A result object which may be used by a subsequent {@link ChainingOp} */ @Override T apply(long value); diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/Op.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/Op.java index ca50f1a1a..e5ecdf6a2 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/Op.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/Op.java @@ -17,21 +17,20 @@ package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes; /** - * This is the root type of any operation which is used in a NoSQLBench + *

This is the root type of any operation which is used in a NoSQLBench * DriverAdapter. It is a tagging interface for incremental type validation - * in the NB runtime. You probably don't want to use it directly. + * in the NB runtime. You probably don't want to use it directly.

* - * Instead, use these: + *

Instead, use one of these: *

    *
  • {@link CycleOp}
    • - An interface that will called if there is nothing to consume * the result type from your operation. In some cases preparing a result body to - * hand down the chain is more costly, so implementing this interface allows ... + * hand down the chain is more costly, so implementing this interface allows the runtime + * to be more optimized. + *
  • {@link ChainingOp}
    • *
- * - * either {@link CycleOp} or {@link ChainingOp} (but not both!) - * - * In the standard flow of an activity, either of the above interfaces is called - * so long as an Op implements one of them. + *

*/ +// TODO: optimize the runtime around the specific op type public interface Op extends OpResultSize { } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/RunnableOp.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/RunnableOp.java index dd5ba39fc..0cfc2f984 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/RunnableOp.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/RunnableOp.java @@ -16,5 +16,19 @@ package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes; +/** + *

RunnableOp

+ *

This is the simplest form of an executable operation in NoSQLBench. + * It is simply an operation is run for side-effect only.

+ */ public interface RunnableOp extends Op, Runnable { + + /** + * Invoke the operation. If you need to see the value of the current + * cycle, then you can use {@link CycleOp} instead. If you need to + * use a cached result of a previous operation, then you may need to + * use {@link ChainingOp}. + */ + @Override + void run(); } From 72602805939986783b4a0e117dc5beabdaa8179d Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Thu, 1 Dec 2022 12:16:06 -0600 Subject: [PATCH 17/45] update docs for op types --- .../api/activityimpl/uniform/StandardAction.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardAction.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardAction.java index 0e1612afd..45f8c1826 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardAction.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardAction.java @@ -32,11 +32,11 @@ import org.apache.logging.log4j.Logger; import java.util.concurrent.TimeUnit; /** - * This is the generified version of an Action. All activity types should endeavor to use - * this, as the API surface is being consolidated so that the internal machinery of NB - * works in a very consistent and uniform way. - * There will be changes to multiple drivers to support this consolidation, but the bulk - * of this work will be undertaken by the project maintainers. + * This is the generified version of an Action. All driver adapters us this, as opposed + * to previous NB versions where it was implemented for each driver. + * + * This allows the API to be consolidated so that the internal machinery of NB + * works in a very consistent and uniform way for all users and drivers. * * @param The type of activity * @param The type of operation From 52ef693735814ecc1ba202ab9e9dc0283167102c Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Fri, 2 Dec 2022 06:27:32 +0000 Subject: [PATCH 18/45] fix: upgrade io.netty:netty-handler from 4.1.84.Final to 4.1.85.Final Snyk has created this PR to upgrade io.netty:netty-handler from 4.1.84.Final to 4.1.85.Final. See this package in Maven Repository: https://mvnrepository.com/artifact/io.netty/netty-handler/ See this project in Snyk: https://app.snyk.io/org/jshook/project/faf9c31c-14e2-456a-9f66-9dda91c923d9?utm_source=github&utm_medium=referral&page=upgrade-pr --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 5d1e3e76a..5f54aa733 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -189,7 +189,7 @@ io.netty netty-handler - 4.1.84.Final + 4.1.85.Final From 1e6367ecbe0f7c6b39300375292b31dedec6b145 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Fri, 2 Dec 2022 06:27:36 +0000 Subject: [PATCH 19/45] fix: upgrade org.graalvm.js:js from 21.3.3.1 to 21.3.4 Snyk has created this PR to upgrade org.graalvm.js:js from 21.3.3.1 to 21.3.4. See this package in Maven Repository: https://mvnrepository.com/artifact/org.graalvm.js/js/ See this project in Snyk: https://app.snyk.io/org/jshook/project/faf9c31c-14e2-456a-9f66-9dda91c923d9?utm_source=github&utm_medium=referral&page=upgrade-pr --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 5d1e3e76a..adfefbcbb 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -365,7 +365,7 @@ org.graalvm.js js - 21.3.3.1 + 21.3.4 runtime From 3030e58dfae05e3e6fc91ba9729ae4d4b71c77c5 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Fri, 2 Dec 2022 06:50:48 +0000 Subject: [PATCH 20/45] fix: upgrade org.graalvm.tools:profiler from 20.3.6.1 to 20.3.8 Snyk has created this PR to upgrade org.graalvm.tools:profiler from 20.3.6.1 to 20.3.8. See this package in Maven Repository: https://mvnrepository.com/artifact/org.graalvm.tools/profiler/ See this project in Snyk: https://app.snyk.io/org/jshook/project/0e822587-faec-4a2d-9b12-113823abf43e?utm_source=github&utm_medium=referral&page=upgrade-pr --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 5d1e3e76a..98973e0ae 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -376,7 +376,7 @@ org.graalvm.tools profiler - 20.3.6.1 + 20.3.8 runtime From fd4673f15b8306cf6315bfb3274587c15527fe65 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Fri, 2 Dec 2022 06:50:51 +0000 Subject: [PATCH 21/45] fix: upgrade io.dropwizard.metrics:metrics-graphite from 4.2.10 to 4.2.12 Snyk has created this PR to upgrade io.dropwizard.metrics:metrics-graphite from 4.2.10 to 4.2.12. See this package in Maven Repository: https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-graphite/ See this project in Snyk: https://app.snyk.io/org/jshook/project/0e822587-faec-4a2d-9b12-113823abf43e?utm_source=github&utm_medium=referral&page=upgrade-pr --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 5d1e3e76a..adf9d8599 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -135,7 +135,7 @@ io.dropwizard.metrics metrics-graphite - 4.2.10 + 4.2.12 From 141bfe088f2301ddb96993aa22f2ce9e76c312f4 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Fri, 2 Dec 2022 06:50:55 +0000 Subject: [PATCH 22/45] fix: upgrade org.graalvm.js:js-scriptengine from 21.3.3.1 to 21.3.4 Snyk has created this PR to upgrade org.graalvm.js:js-scriptengine from 21.3.3.1 to 21.3.4. See this package in Maven Repository: https://mvnrepository.com/artifact/org.graalvm.js/js-scriptengine/ See this project in Snyk: https://app.snyk.io/org/jshook/project/0e822587-faec-4a2d-9b12-113823abf43e?utm_source=github&utm_medium=referral&page=upgrade-pr --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 5d1e3e76a..2c1ce4d98 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -371,7 +371,7 @@ org.graalvm.js js-scriptengine - 21.3.3.1 + 21.3.4 org.graalvm.tools From 82bfa44855ff93ee68d6e5bdda0ae7be4056043e Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Mon, 5 Dec 2022 23:07:41 +0000 Subject: [PATCH 23/45] fix: upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.340 to 1.12.341 Snyk has created this PR to upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.340 to 1.12.341. See this package in Maven Repository: https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-s3/ See this project in Snyk: https://app.snyk.io/org/jshook/project/fc9e1bd9-1d9a-474f-bde4-efb614c52ffe?utm_source=github&utm_medium=referral&page=upgrade-pr --- nb-api/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nb-api/pom.xml b/nb-api/pom.xml index 2e0631a8a..e26f8e869 100644 --- a/nb-api/pom.xml +++ b/nb-api/pom.xml @@ -97,7 +97,7 @@ com.amazonaws aws-java-sdk-s3 - 1.12.340 + 1.12.341 From 150793cda97b456783073c083fd1a2214059a177 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 6 Dec 2022 17:07:24 +0000 Subject: [PATCH 24/45] fix: upgrade org.glassfish.jersey.media:jersey-media-json-jackson from 3.0.8 to 3.1.0 Snyk has created this PR to upgrade org.glassfish.jersey.media:jersey-media-json-jackson from 3.0.8 to 3.1.0. See this package in Maven Repository: https://mvnrepository.com/artifact/org.glassfish.jersey.media/jersey-media-json-jackson/ See this project in Snyk: https://app.snyk.io/org/jshook/project/03cbee46-d5d2-41d3-89cc-a2ad77ab807a?utm_source=github&utm_medium=referral&page=upgrade-pr --- docsys/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsys/pom.xml b/docsys/pom.xml index 1d5147e12..7dfa9e33b 100644 --- a/docsys/pom.xml +++ b/docsys/pom.xml @@ -99,7 +99,7 @@ org.glassfish.jersey.media jersey-media-json-jackson - 3.0.8 + 3.1.0 From 76f5507e4d4e017de460b9fc0f6e4c165aafcd1d Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 6 Dec 2022 17:07:28 +0000 Subject: [PATCH 25/45] fix: upgrade org.apache.commons:commons-compress from 1.21 to 1.22 Snyk has created this PR to upgrade org.apache.commons:commons-compress from 1.21 to 1.22. See this package in Maven Repository: https://mvnrepository.com/artifact/org.apache.commons/commons-compress/ See this project in Snyk: https://app.snyk.io/org/jshook/project/03cbee46-d5d2-41d3-89cc-a2ad77ab807a?utm_source=github&utm_medium=referral&page=upgrade-pr --- docsys/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsys/pom.xml b/docsys/pom.xml index 1d5147e12..ca45ede7d 100644 --- a/docsys/pom.xml +++ b/docsys/pom.xml @@ -93,7 +93,7 @@ org.apache.commons commons-compress - 1.21 + 1.22 From f8623f3b7d2c80f8afea322e9b95e547ea4e58e9 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 6 Dec 2022 20:25:18 +0000 Subject: [PATCH 26/45] fix: upgrade io.swagger.core.v3:swagger-models from 2.2.4 to 2.2.6 Snyk has created this PR to upgrade io.swagger.core.v3:swagger-models from 2.2.4 to 2.2.6. See this package in Maven Repository: https://mvnrepository.com/artifact/io.swagger.core.v3/swagger-models/ See this project in Snyk: https://app.snyk.io/org/jshook/project/b808ba5a-fa96-49c2-9cae-4c2c2f8a1384?utm_source=github&utm_medium=referral&page=upgrade-pr --- engine-rest/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine-rest/pom.xml b/engine-rest/pom.xml index a45106357..58df73fa6 100644 --- a/engine-rest/pom.xml +++ b/engine-rest/pom.xml @@ -29,7 +29,7 @@ io.swagger.core.v3 swagger-models - 2.2.4 + 2.2.6 From b3d3f59d40873b6fe7f44df344fa70679ab8f0e6 Mon Sep 17 00:00:00 2001 From: yabinmeng Date: Tue, 6 Dec 2022 17:24:52 -0600 Subject: [PATCH 27/45] Migrate NBS4J from NB4 to NB5 --- adapter-pulsar/pom.xml | 14 +- .../adapter/pulsar/util/PulsarClientConf.java | 2 +- adapter-s4j/pom.xml | 89 ++++ .../adapter/s4j/S4JDriverAdapter.java | 52 +++ .../nosqlbench/adapter/s4j/S4JOpMapper.java | 71 +++ .../io/nosqlbench/adapter/s4j/S4JOpType.java | 31 ++ .../io/nosqlbench/adapter/s4j/S4JSpace.java | 364 +++++++++++++++ .../MessageConsumerOpDispenser.java | 162 +++++++ .../MessageProducerOpDispenser.java | 358 +++++++++++++++ .../s4j/dispensers/S4JBaseOpDispenser.java | 395 ++++++++++++++++ ...JAdapterAsyncOperationFailedException.java | 26 ++ .../S4JAdapterInvalidParamException.java | 29 ++ .../S4JAdapterUnexpectedException.java | 30 ++ .../S4JAdapterUnsupportedOpException.java | 25 + .../adapter/s4j/ops/MessageConsumerOp.java | 135 ++++++ .../adapter/s4j/ops/MessageProducerOp.java | 101 +++++ .../io/nosqlbench/adapter/s4j/ops/S4JOp.java | 58 +++ .../adapter/s4j/util/S4JAdapterMetrics.java | 68 +++ .../adapter/s4j/util/S4JAdapterUtil.java | 326 +++++++++++++ .../adapter/s4j/util/S4JClientConf.java | 213 +++++++++ .../s4j/util/S4JClientConfConverter.java | 429 ++++++++++++++++++ .../s4j/util/S4JCompletionListener.java | 86 ++++ .../s4j/util/S4JJMSContextWrapper.java | 52 +++ .../adapter/s4j/util/S4JMessageListener.java | 97 ++++ nb5/pom.xml | 6 + pom.xml | 1 + 26 files changed, 3212 insertions(+), 8 deletions(-) create mode 100644 adapter-s4j/pom.xml create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JDriverAdapter.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpType.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JSpace.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageConsumerOpDispenser.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageProducerOpDispenser.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/S4JBaseOpDispenser.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterAsyncOperationFailedException.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterInvalidParamException.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterUnexpectedException.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterUnsupportedOpException.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/MessageConsumerOp.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/MessageProducerOp.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/S4JOp.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JAdapterMetrics.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JAdapterUtil.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JClientConf.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JClientConfConverter.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JCompletionListener.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JJMSContextWrapper.java create mode 100644 adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JMessageListener.java diff --git a/adapter-pulsar/pom.xml b/adapter-pulsar/pom.xml index 2428c54fe..5ed541f12 100644 --- a/adapter-pulsar/pom.xml +++ b/adapter-pulsar/pom.xml @@ -62,6 +62,13 @@ ${pulsar.version} + + + org.apache.commons + commons-lang3 + 3.12.0 + + commons-beanutils @@ -82,13 +89,6 @@ avro 1.11.1 - - - - org.apache.commons - commons-lang3 - 3.12.0 - diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarClientConf.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarClientConf.java index 4af681509..fc6966f4a 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarClientConf.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarClientConf.java @@ -73,7 +73,7 @@ public class PulsarClientConf { // Convert the raw configuration map () to the required map () producerConfMapTgt.putAll(PulsarConfConverter.convertStdRawProducerConf(producerConfMapRaw)); consumerConfMapTgt.putAll(PulsarConfConverter.convertStdRawConsumerConf(consumerConfMapRaw)); - // TODO: Reader API is not disabled at the moment. Revisit when needed + // TODO: Reader API is not enabled at the moment. Revisit when needed } diff --git a/adapter-s4j/pom.xml b/adapter-s4j/pom.xml new file mode 100644 index 000000000..41bb02db6 --- /dev/null +++ b/adapter-s4j/pom.xml @@ -0,0 +1,89 @@ + + 4.0.0 + + adapter-s4j + jar + + + mvn-defaults + io.nosqlbench + 4.17.31-SNAPSHOT + ../mvn-defaults + + + ${project.artifactId} + + A Starlight for JMS driver for nosqlbench. This provides the ability to inject synthetic data + into a pulsar system via JMS 2.0 compatible APIs. + + NOTE: this is JMS compatible driver from DataStax that allows using a Pulsar cluster + as the potential JMS Destination + + + + 3.2.0 + + + + + + kr.motd.maven + os-maven-plugin + 1.7.1 + + + + + + + + io.nosqlbench + engine-api + 4.17.31-SNAPSHOT + + + + io.nosqlbench + adapters-api + 4.17.31-SNAPSHOT + + + + + com.datastax.oss + pulsar-jms-all + ${s4j.version} + + + + + org.apache.commons + commons-lang3 + 3.12.0 + + + + + commons-beanutils + commons-beanutils + 1.9.4 + + + + + org.apache.commons + commons-configuration2 + 2.8.0 + + + + + org.conscrypt + conscrypt-openjdk + 2.5.2 + ${os.detected.classifier} + + + + + diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JDriverAdapter.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JDriverAdapter.java new file mode 100644 index 000000000..49527f9c0 --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JDriverAdapter.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.s4j; + +import io.nosqlbench.adapter.s4j.ops.S4JOp; +import io.nosqlbench.api.config.standard.NBConfigModel; +import io.nosqlbench.api.config.standard.NBConfiguration; +import io.nosqlbench.engine.api.activityimpl.OpMapper; +import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.nb.annotations.Service; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.Function; + +@Service(value = DriverAdapter.class, selector = "s4j") +public class S4JDriverAdapter extends BaseDriverAdapter { + private final static Logger logger = LogManager.getLogger(S4JDriverAdapter.class); + + @Override + public OpMapper getOpMapper() { + DriverSpaceCache spaceCache = getSpaceCache(); + NBConfiguration adapterConfig = getConfiguration(); + return new S4JOpMapper(this, adapterConfig, spaceCache); + } + + @Override + public Function getSpaceInitializer(NBConfiguration cfg) { + return (s) -> new S4JSpace(s, cfg); + } + + @Override + public NBConfigModel getConfigModel() { + return super.getConfigModel().add(S4JSpace.getConfigModel()); + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java new file mode 100644 index 000000000..95b39ec57 --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.s4j; + +import io.nosqlbench.adapter.s4j.dispensers.MessageConsumerOpDispenser; +import io.nosqlbench.adapter.s4j.dispensers.MessageProducerOpDispenser; +import io.nosqlbench.adapter.s4j.ops.S4JOp; +import io.nosqlbench.api.config.standard.NBConfiguration; +import io.nosqlbench.engine.api.activityimpl.OpDispenser; +import io.nosqlbench.engine.api.activityimpl.OpMapper; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.engine.api.templating.ParsedOp; +import io.nosqlbench.engine.api.templating.TypeAndTarget; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class S4JOpMapper implements OpMapper { + + private final static Logger logger = LogManager.getLogger(S4JOpMapper.class); + + private final NBConfiguration cfg; + private final DriverSpaceCache spaceCache; + private final DriverAdapter adapter; + + public S4JOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache spaceCache) { + this.cfg = cfg; + this.spaceCache = spaceCache; + this.adapter = adapter; + } + + @Override + public OpDispenser apply(ParsedOp op) { + String spaceName = op.getStaticConfigOr("space", "default"); + S4JSpace s4jSpace = spaceCache.get(spaceName); + + /* + * If the user provides a body element, then they want to provide the JSON or + * a data structure that can be converted into JSON, bypassing any further + * specialized type-checking or op-type specific features + */ + if (op.isDefined("body")) { + throw new RuntimeException("This mode is reserved for later. Do not use the 'body' op field."); + } + else { + TypeAndTarget opType = op.getTypeAndTarget(S4JOpType.class, String.class); + + return switch (opType.enumId) { + case MessageProduce -> + new MessageProducerOpDispenser(adapter, op, opType.targetFunction, s4jSpace); + case MessageConsume -> + new MessageConsumerOpDispenser(adapter, op, opType.targetFunction, s4jSpace); + }; + } + } + +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpType.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpType.java new file mode 100644 index 000000000..2e4f1144b --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpType.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.s4j; + +public enum S4JOpType { + // publishing/sending messages to a JMS queue or a topic + MessageProduce, + // consuming/receiving messages from a JMS queue or a topic + // for a topic, it can be: + // - non-durable, non-shared + // - durable, non-shared + // - non-durable, shared + // - durable, shared + MessageConsume; +} + + diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JSpace.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JSpace.java new file mode 100644 index 000000000..8c552912f --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JSpace.java @@ -0,0 +1,364 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.s4j; + +import com.datastax.oss.pulsar.jms.PulsarConnectionFactory; +import io.nosqlbench.adapter.s4j.exception.S4JAdapterInvalidParamException; +import io.nosqlbench.adapter.s4j.exception.S4JAdapterUnexpectedException; +import io.nosqlbench.adapter.s4j.util.*; +import io.nosqlbench.api.config.standard.ConfigModel; +import io.nosqlbench.api.config.standard.NBConfigModel; +import io.nosqlbench.api.config.standard.NBConfiguration; +import io.nosqlbench.api.config.standard.Param; +import org.apache.commons.lang3.RandomUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.jms.*; +import java.util.Base64; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +public class S4JSpace implements AutoCloseable { + + private final static Logger logger = LogManager.getLogger(S4JSpace.class); + + private final String spaceName; + private final NBConfiguration cfg; + + // - Each S4J space currently represents a number of JMS connections (\"num_conn\" NB CLI parameter); + // - JMS connection can have a number of JMS sessions (\"num_session\" NB CLI parameter). + // - Each JMS session has its own sets of JMS destinations, producers, consumers, etc. + private final ConcurrentHashMap connLvlJmsContexts = new ConcurrentHashMap<>(); + private final ConcurrentHashMap sessionLvlJmsContexts = new ConcurrentHashMap<>(); + + private final String pulsarSvcUrl; + private final String webSvcUrl; + private final String s4jClientConfFileName; + private S4JClientConf s4JClientConf; + private final int sessionMode; + + // Whether to do strict error handling while sending/receiving messages + // - Yes: any error returned from the Pulsar server while doing message receiving/sending will trigger NB execution stop + // - No: pause the current thread that received the error message for 1 second and then continue processing + private boolean strictMsgErrorHandling; + + // Maximum time length to execute S4J operations (e.g. message send or consume) + // - when NB execution passes this threshold, it is simply NoOp + // - 0 means no maximum time constraint. S4JOp is always executed until NB execution cycle finishes + private long maxS4JOpTimeInSec; + private long s4JActivityStartTimeMills; + + // Whether to keep track of the received message count, which includes + // - total received message count + // - received null message count (only relevant when non-blocking message receiving is used) + // By default, this setting is disabled + private boolean trackingMsgRecvCnt; + + // How many JMS connections per NB S4J execution + private int maxNumConn; + // How many sessions per JMS connection + private int maxNumSessionPerConn; + + // Total number of acknowledgement received + // - this can apply to both message production and consumption + // - for message consumption, this only applies to non-null messages received (which is for async API) + private final AtomicLong totalOpResponseCnt = new AtomicLong(0); + // Total number of null messages received + // - only applicable to message consumption + private final AtomicLong nullMsgRecvCnt = new AtomicLong(0); + + // Keep track the transaction count per thread + private final ThreadLocal txnBatchTrackingCnt = ThreadLocal.withInitial(() -> 0); + + // Represents the JMS connection + private PulsarConnectionFactory s4jConnFactory; + + private long totalCycleNum; + + public S4JSpace(String spaceName, NBConfiguration cfg) { + this.spaceName = spaceName; + this.cfg = cfg; + + this.pulsarSvcUrl = cfg.get("service_url"); + this.webSvcUrl = cfg.get("web_url"); + this.maxNumConn= cfg.getOrDefault("num_conn", Integer.valueOf(1)); + this.maxNumSessionPerConn = cfg.getOrDefault("num_session", Integer.valueOf(1)); + this.maxS4JOpTimeInSec= cfg.getOrDefault("max_s4jop_time", Long.valueOf(0)); + this.trackingMsgRecvCnt=cfg.getOrDefault("track_msg_cnt", Boolean.FALSE); + this.strictMsgErrorHandling = cfg.getOrDefault("strict_msg_error_handling", Boolean.FALSE); + this.s4jClientConfFileName = cfg.get("config"); + this.sessionMode = S4JAdapterUtil.getSessionModeFromStr(cfg.get("session_mode")); + this.s4JClientConf = new S4JClientConf(pulsarSvcUrl, webSvcUrl, s4jClientConfFileName); + + this.initializeSpace(s4JClientConf); + } + + @Override + public void close() { + shutdownSpace(); + } + + public static NBConfigModel getConfigModel() { + return ConfigModel.of(S4JSpace.class) + .add(Param.defaultTo("service_url", "pulsar://localhost:6650") + .setDescription("Pulsar broker service URL.")) + .add(Param.defaultTo("web_url", "http://localhost:8080") + .setDescription("Pulsar web service URL.")) + .add(Param.defaultTo("config", "config.properties") + .setDescription("Pulsar client connection configuration property file.")) + .add(Param.defaultTo("num_conn", 1) + .setDescription("Number of JMS connections")) + .add(Param.defaultTo("num_session", 1) + .setDescription("Number of JMS sessions per JMS connection")) + .add(Param.defaultTo("max_s4jop_time", 0) + .setDescription("Maximum time (in seconds) to run NB S4J testing scenario.")) + .add(Param.defaultTo("track_msg_cnt", false) + .setDescription("Whether to keep track of message count(s)")) + .add(Param.defaultTo("session_mode", "") + .setDescription("JMS session mode")) + .add(Param.defaultTo("strict_msg_error_handling", false) + .setDescription("Whether to do strict error handling which is to stop NB S4J execution.")) + .asReadOnly(); + } + + public ConcurrentHashMap getConnLvlJmsContexts() { + return connLvlJmsContexts; + } + + public ConcurrentHashMap getSessionLvlJmsContexts() { + return sessionLvlJmsContexts; + } + + public long getS4JActivityStartTimeMills() { return this.s4JActivityStartTimeMills; } + public void setS4JActivityStartTimeMills(long startTime) { this.s4JActivityStartTimeMills = startTime; } + + public long getMaxS4JOpTimeInSec() { return this.maxS4JOpTimeInSec; } + + public int getSessionMode() { return sessionMode; } + + public String getS4jClientConfFileName() { return s4jClientConfFileName; } + public S4JClientConf getS4JClientConf() { return s4JClientConf; } + + public boolean isTrackingMsgRecvCnt() { return trackingMsgRecvCnt; } + + public int getMaxNumSessionPerConn() { return this.maxNumSessionPerConn; } + public int getMaxNumConn() { return this.maxNumConn; } + + public boolean isStrictMsgErrorHandling() { return this.strictMsgErrorHandling; } + + public int getTxnBatchTrackingCnt() { return txnBatchTrackingCnt.get(); } + public void incTxnBatchTrackingCnt() { + int curVal = getTxnBatchTrackingCnt(); + txnBatchTrackingCnt.set(curVal + 1); + } + + public long getTotalOpResponseCnt() { return totalOpResponseCnt.get();} + public long incTotalOpResponseCnt() { return totalOpResponseCnt.incrementAndGet();} + public void resetTotalOpResponseCnt() { totalOpResponseCnt.set(0); } + + public long getTotalNullMsgRecvdCnt() { return nullMsgRecvCnt.get();} + public void resetTotalNullMsgRecvdCnt() { nullMsgRecvCnt.set(0); } + + public long incTotalNullMsgRecvdCnt() { return nullMsgRecvCnt.incrementAndGet(); } + + public PulsarConnectionFactory getS4jConnFactory() { return s4jConnFactory; } + + public long getTotalCycleNum() { return totalCycleNum; } + public void setTotalCycleNum(long cycleNum) { totalCycleNum = cycleNum; } + + public void initializeSpace(S4JClientConf s4JClientConnInfo) { + if (s4jConnFactory == null) { + Map cfgMap; + try { + cfgMap = s4JClientConnInfo.getS4jConfObjMap(); + s4jConnFactory = new PulsarConnectionFactory(cfgMap); + + for (int i=0; i { + if (logger.isDebugEnabled()) { + logger.error("onException::Unexpected JMS error happened:" + e); + } + }); + + connLvlJmsContexts.put(connLvlJmsConnContextIdStr, jmsConnContext); + + if (logger.isDebugEnabled()) { + logger.debug("[Connection level JMSContext] {} -- {}", + Thread.currentThread().getName(), + jmsConnContext ); + } + } + } + catch (JMSRuntimeException e) { + if (logger.isDebugEnabled()) { + logger.debug("[ERROR] Unable to initialize JMS connection factory with the following configuration parameters: {}", s4JClientConnInfo.toString()); + } + throw new S4JAdapterUnexpectedException("Unable to initialize JMS connection factory with the following error message: " + e.getCause()); + } + catch (Exception e) { + e.printStackTrace(); + } + } + } + + public void shutdownSpace() { + long shutdownStartTimeMills = System.currentTimeMillis(); + + try { + waitUntilAllOpFinished(shutdownStartTimeMills); + + this.txnBatchTrackingCnt.remove(); + + for (S4JJMSContextWrapper s4JJMSContextWrapper : sessionLvlJmsContexts.values()) { + if (s4JJMSContextWrapper != null) { + if (s4JJMSContextWrapper.isTransactedMode()) { + s4JJMSContextWrapper.getJmsContext().rollback(); + } + s4JJMSContextWrapper.close(); + } + } + + for (JMSContext jmsContext : connLvlJmsContexts.values()) { + if (jmsContext != null) jmsContext.close(); + } + + s4jConnFactory.close(); + } + catch (Exception e) { + e.printStackTrace(); + throw new S4JAdapterUnexpectedException("Unexpected error when shutting down NB S4J space."); + } + } + + // When completing NB execution, don't shut down right away because otherwise, async operation processing may fail. + // Instead, shut down when either one of the following condition is satisfied + // 1) the total number of the received operation response is the same as the total number of operations being executed; + // 2) time has passed for 10 seconds + private void waitUntilAllOpFinished(long shutdownStartTimeMills) { + long totalCycleNum = getTotalCycleNum(); + long totalResponseCnt = 0; + long totalNullMsgCnt = 0; + long timeElapsedMills; + + boolean trackingMsgCnt = isTrackingMsgRecvCnt(); + boolean continueChk; + + do { + S4JAdapterUtil.pauseCurThreadExec(1); + + long curTimeMills = System.currentTimeMillis(); + timeElapsedMills = curTimeMills - shutdownStartTimeMills; + continueChk = (timeElapsedMills <= 10000); + + if (trackingMsgCnt) { + totalResponseCnt = this.getTotalOpResponseCnt(); + totalNullMsgCnt = this.getTotalNullMsgRecvdCnt(); + continueChk = continueChk && (totalResponseCnt < totalCycleNum); + } + + if (logger.isTraceEnabled()) { + logger.trace( + buildExecSummaryString(trackingMsgCnt, timeElapsedMills, totalResponseCnt, totalNullMsgCnt)); + } + } while (continueChk); + + logger.info( + buildExecSummaryString(trackingMsgCnt, timeElapsedMills, totalResponseCnt, totalNullMsgCnt)); + } + + private String buildExecSummaryString( + boolean trackingMsgCnt, + long timeElapsedMills, + long totalResponseCnt, + long totalNullMsgCnt) + { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder + .append("shutdownSpace::waitUntilAllOpFinished -- ") + .append("shutdown time elapsed: ").append(timeElapsedMills).append("ms; "); + + if (trackingMsgCnt) { + stringBuilder.append("response received: ").append(totalResponseCnt).append("; "); + stringBuilder.append("null msg received: ").append(totalNullMsgCnt).append("; "); + } + + return stringBuilder.toString(); + } + + public void processMsgAck(JMSContext jmsContext, Message message, float msgAckRatio, int slowAckInSec) throws JMSException { + int jmsSessionMode = jmsContext.getSessionMode(); + + if ((jmsSessionMode != Session.AUTO_ACKNOWLEDGE) && + (jmsSessionMode != Session.SESSION_TRANSACTED)) { + float rndVal = RandomUtils.nextFloat(0, 1); + if (rndVal < msgAckRatio) { + S4JAdapterUtil.pauseCurThreadExec(slowAckInSec); + message.acknowledge(); + } + } + } + + public String getConnLvlJmsContextIdentifier(int jmsConnSeqNum) { + return S4JAdapterUtil.buildCacheKey( + this.spaceName, + StringUtils.join("conn-", jmsConnSeqNum)); + } + + public String getSessionLvlJmsContextIdentifier(int jmsConnSeqNum, int jmsSessionSeqNum) { + return S4JAdapterUtil.buildCacheKey( + this.spaceName, + StringUtils.join("conn-", jmsConnSeqNum), + StringUtils.join("session-", jmsSessionSeqNum)); + } + + // Create JMSContext that represents a new JMS connection + public JMSContext getOrCreateConnLvlJMSContext( + PulsarConnectionFactory s4jConnFactory, + S4JClientConf s4JClientConf, + int sessionMode) + { + boolean useCredentialsEnable = S4JAdapterUtil.isUseCredentialsEnabled(s4JClientConf); + JMSContext jmsConnContext; + + if (!useCredentialsEnable) + jmsConnContext = s4jConnFactory.createContext(sessionMode); + else { + String userName = S4JAdapterUtil.getCredentialUserName(s4JClientConf); + String passWord = S4JAdapterUtil.getCredentialPassword(s4JClientConf); + + // Password must be in "token:" format + if (! StringUtils.startsWith(passWord, "token:")) { + throw new S4JAdapterInvalidParamException( + "When 'jms.useCredentialsFromCreateConnection' is enabled, " + + "the provided password must be in format 'token: "); + } + + jmsConnContext = s4jConnFactory.createContext(userName, passWord, sessionMode); + } + + return jmsConnContext; + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageConsumerOpDispenser.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageConsumerOpDispenser.java new file mode 100644 index 000000000..b71e9ad4a --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageConsumerOpDispenser.java @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.s4j.dispensers; + +import io.nosqlbench.adapter.s4j.S4JSpace; +import io.nosqlbench.adapter.s4j.ops.MessageConsumerOp; +import io.nosqlbench.adapter.s4j.util.S4JAdapterUtil; +import io.nosqlbench.adapter.s4j.util.S4JJMSContextWrapper; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.jms.*; +import java.util.HashMap; +import java.util.Map; +import java.util.function.LongFunction; + +public class MessageConsumerOpDispenser extends S4JBaseOpDispenser { + + private final static Logger logger = LogManager.getLogger("MessageConsumerOpDispenser"); + + // Doc-level parameter: blocking_msg_recv (default: false) + protected final boolean blockingMsgRecv; + // Doc-level parameter: shared_topic (default: false) + // - only applicable to Topic as the destination type + protected final boolean sharedTopic; + // Doc-level parameter: durable_topic (default: false) + // - only applicable to Topic as the destination type + protected final boolean durableTopic; + // default value: false + private final boolean noLocal; + // default value: 0 + // value <= 0 : no timeout + private final int readTimeout; + // default value: false + private final boolean recvNoWait; + // default value: 1.0 (all received messages are acknowledged) + // value must be between 0 and 1 (inclusive) + private final float msgAckRatio; + // default value: 0 + // value <= 0 : no slow message ack + private final int slowAckInSec; + private final LongFunction subNameStrFunc; + private final LongFunction localMsgSelectorFunc; + + // Generally the consumer related configurations can be set in the global "config.properties" file, + // which can be applied to many testing scenarios. + // Setting them here will allow scenario-specific customer configurations. At the moment, only the + // DLT related settings are supported + private final Map combinedConsumerConfigObjMap = new HashMap<>(); + + + public MessageConsumerOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + S4JSpace s4jSpace) { + super(adapter, op, tgtNameFunc, s4jSpace); + + this.blockingMsgRecv = + parsedOp.getStaticConfigOr(S4JAdapterUtil.DOC_LEVEL_PARAMS.BLOCKING_MSG_RECV.label, Boolean.FALSE); + this.sharedTopic = + parsedOp.getStaticConfigOr(S4JAdapterUtil.DOC_LEVEL_PARAMS.SHARED_TOPIC.label, Boolean.FALSE); + this.durableTopic = + parsedOp.getStaticConfigOr(S4JAdapterUtil.DOC_LEVEL_PARAMS.DURABLE_TOPIC.label, Boolean.FALSE); + this.noLocal = + parsedOp.getStaticConfigOr("no_local", Boolean.FALSE); + this.readTimeout = + parsedOp.getStaticConfigOr("read_timeout", Integer.valueOf(0)); + this.recvNoWait = + parsedOp.getStaticConfigOr("no_wait", Boolean.FALSE); + this.msgAckRatio = + parsedOp.getStaticConfigOr("msg_ack_ratio", Float.valueOf(1.0f)); + this.slowAckInSec = + parsedOp.getStaticConfigOr("slow_ack_in_sec", Integer.valueOf(0)); + this.subNameStrFunc = + lookupMandtoryStrOpValueFunc("subscription_name"); + this.localMsgSelectorFunc = + lookupOptionalStrOpValueFunc("msg_selector"); + + String[] stmtLvlConsumerConfKeyNameList = { + "consumer.ackTimeoutMillis", + "consumer.deadLetterPolicy", + "consumer.negativeAckRedeliveryBackoff", + "consumer.ackTimeoutRedeliveryBackoff"}; + HashMap stmtLvlConsumerConfRawMap = new HashMap<>(); + for (String confKey : stmtLvlConsumerConfKeyNameList ) { + String confVal = parsedOp.getStaticConfigOr(confKey, ""); + stmtLvlConsumerConfRawMap.put( + StringUtils.substringAfter(confKey, "consumer."), + confVal); + } + + this.combinedConsumerConfigObjMap.putAll( + s4jSpace.getS4JClientConf().mergeExtraConsumerConfig(stmtLvlConsumerConfRawMap)); + } + + @Override + public MessageConsumerOp apply(long cycle) { + S4JJMSContextWrapper s4JJMSContextWrapper = + getOrCreateS4jJmsContextWrapper(cycle, this.combinedConsumerConfigObjMap); + JMSContext jmsContext = s4JJMSContextWrapper.getJmsContext(); + boolean commitTransact = !super.commitTransaction(txnBatchNum, jmsContext.getSessionMode(), cycle); + + Destination destination; + try { + destination = getOrCreateJmsDestination( + s4JJMSContextWrapper, temporaryDest, destType, destNameStrFunc.apply(cycle)); + } + catch (JMSRuntimeException jmsRuntimeException) { + throw new RuntimeException("Unable to create the JMS destination!"); + } + + JMSConsumer jmsConsumer; + try { + jmsConsumer = getOrCreateJmsConsumer( + s4JJMSContextWrapper, + destination, + destType, + subNameStrFunc.apply(cycle), + localMsgSelectorFunc.apply(cycle), + msgAckRatio, + noLocal, + durableTopic, + sharedTopic, + asyncAPI, + slowAckInSec); + } + catch (JMSException jmsException) { + throw new RuntimeException("Unable to create the JMS consumer!"); + } + + return new MessageConsumerOp( + s4jAdapterMetrics, + s4jSpace, + jmsContext, + destination, + asyncAPI, + commitTransact, + jmsConsumer, + blockingMsgRecv, + msgAckRatio, + readTimeout, + recvNoWait, + slowAckInSec); + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageProducerOpDispenser.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageProducerOpDispenser.java new file mode 100644 index 000000000..8c2e5e1b7 --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageProducerOpDispenser.java @@ -0,0 +1,358 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.s4j.dispensers; + +import io.nosqlbench.adapter.s4j.S4JSpace; +import io.nosqlbench.adapter.s4j.exception.S4JAdapterInvalidParamException; +import io.nosqlbench.adapter.s4j.exception.S4JAdapterUnexpectedException; +import io.nosqlbench.adapter.s4j.ops.MessageProducerOp; +import io.nosqlbench.adapter.s4j.util.S4JAdapterUtil; +import io.nosqlbench.adapter.s4j.util.S4JJMSContextWrapper; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.jms.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.LongFunction; +public class MessageProducerOpDispenser extends S4JBaseOpDispenser { + + private final static Logger logger = LogManager.getLogger("MessageProducerOpDispenser"); + + public static final String MSG_HEADER_OP_PARAM = "msg_header"; + public static final String MSG_PROP_OP_PARAM = "msg_property"; + public static final String MSG_BODY_OP_PARAM = "msg_body"; + public static final String MSG_TYPE_OP_PARAM = "msg_type"; + + private final LongFunction msgHeaderRawJsonStrFunc; + private final LongFunction msgPropRawJsonStrFunc; + private final LongFunction msgBodyRawJsonStrFunc; + private final LongFunction msgTypeFunc; + + public MessageProducerOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + S4JSpace s4jSpace) { + super(adapter, op, tgtNameFunc, s4jSpace); + + this.msgHeaderRawJsonStrFunc = lookupOptionalStrOpValueFunc(MSG_HEADER_OP_PARAM); + this.msgPropRawJsonStrFunc = lookupOptionalStrOpValueFunc(MSG_PROP_OP_PARAM); + this.msgBodyRawJsonStrFunc = lookupMandtoryStrOpValueFunc(MSG_BODY_OP_PARAM); + this.msgTypeFunc = lookupMandtoryStrOpValueFunc(MSG_TYPE_OP_PARAM); + } + + private Message createAndSetMessagePayload( + S4JJMSContextWrapper s4JJMSContextWrapper, + String msgType, String msgBodyRawJsonStr) throws JMSException + { + Message message; + int messageSize = 0; + + JMSContext jmsContext = s4JJMSContextWrapper.getJmsContext(); + + if (StringUtils.equalsIgnoreCase(msgType, S4JAdapterUtil.JMS_MESSAGE_TYPES.TEXT.label)) { + message = jmsContext.createTextMessage(); + ((TextMessage) message).setText(msgBodyRawJsonStr); + messageSize = msgBodyRawJsonStr.length(); + } else if (StringUtils.equalsIgnoreCase(msgType, S4JAdapterUtil.JMS_MESSAGE_TYPES.MAP.label)) { + message = jmsContext.createMapMessage(); + + // The message body json string must be in the format of a collection of key/value pairs + // Otherwise, it is an error + Map jmsMsgBodyMap; + try { + jmsMsgBodyMap = S4JAdapterUtil.convertJsonToMap(msgBodyRawJsonStr); + } catch (Exception e) { + throw new RuntimeException("The specified message payload can't be converted to a map when requiring a 'Map' message type!"); + } + + for (String key : jmsMsgBodyMap.keySet()) { + String value = jmsMsgBodyMap.get(key); + ((MapMessage)message).setString(key, value); + messageSize += key.length(); + messageSize += value.length(); + } + } else if (StringUtils.equalsIgnoreCase(msgType, S4JAdapterUtil.JMS_MESSAGE_TYPES.STREAM.label)) { + message = jmsContext.createStreamMessage(); + + // The message body json string must be in the format of a list of objects + // Otherwise, it is an error + List jmsMsgBodyObjList; + try { + jmsMsgBodyObjList = S4JAdapterUtil.convertJsonToObjList(msgBodyRawJsonStr); + } catch (Exception e) { + throw new RuntimeException("The specified message payload can't be converted to a list of Objects when requiring a 'Stream' message type!"); + } + + for (Object obj : jmsMsgBodyObjList) { + ((StreamMessage)message).writeObject(obj); + messageSize += ((String)obj).length(); + } + } else if (StringUtils.equalsIgnoreCase(msgType, S4JAdapterUtil.JMS_MESSAGE_TYPES.OBJECT.label)) { + message = jmsContext.createObjectMessage(); + ((ObjectMessage) message).setObject(msgBodyRawJsonStr); + messageSize += msgBodyRawJsonStr.getBytes().length; + } + // default: BYTE message type + else { + message = jmsContext.createBytesMessage(); + byte[] msgBytePayload = msgBodyRawJsonStr.getBytes(); + ((BytesMessage)message).writeBytes(msgBytePayload); + messageSize += msgBytePayload.length; + } + + message.setStringProperty(S4JAdapterUtil.NB_MSG_SIZE_PROP, String.valueOf(messageSize)); + + return message; + } + + private Message updateMessageHeaders(S4JJMSContextWrapper s4JJMSContextWrapper, Message message, String msgType, String msgHeaderRawJsonStr) throws JMSException { + int messageSize = Integer.parseInt(message.getStringProperty(S4JAdapterUtil.NB_MSG_SIZE_PROP)); + + // Check if msgHeaderRawJsonStr is a valid JSON string with a collection of key/value pairs + // - if Yes, convert it to a map + // - otherwise, log an error message and ignore message headers without throwing a runtime exception + Map jmsMsgHeaders = new HashMap<>(); + if (!StringUtils.isBlank(msgHeaderRawJsonStr)) { + try { + jmsMsgHeaders = S4JAdapterUtil.convertJsonToMap(msgHeaderRawJsonStr); + } catch (Exception e) { + logger.warn( + "Error parsing message header JSON string {}, ignore message headers!", + msgHeaderRawJsonStr); + } + } + // make sure the actual message type is used + jmsMsgHeaders.put(S4JAdapterUtil.JMS_MSG_HEADER_STD.JMSType.label, msgType); + + Message outMessage = message; + for (String msgHeaderKey:jmsMsgHeaders.keySet()) { + // Ignore non-standard message headers + if (S4JAdapterUtil.isValidStdJmsMsgHeader(msgHeaderKey)) { + String value = jmsMsgHeaders.get(msgHeaderKey); + messageSize += msgHeaderKey.length(); + if (value != null) { + messageSize += value.length(); + } + + try { + if (StringUtils.equalsIgnoreCase(msgHeaderKey, S4JAdapterUtil.JMS_MSG_HEADER_STD.JMSType.label)) { + outMessage.setJMSType(msgType); + } else if (StringUtils.equalsIgnoreCase(msgHeaderKey, S4JAdapterUtil.JMS_MSG_HEADER_STD.JMSPriority.label)) { + if (value != null) outMessage.setJMSPriority(Integer.parseInt(value)); + } else if (StringUtils.equalsIgnoreCase(msgHeaderKey, S4JAdapterUtil.JMS_MSG_HEADER_STD.JMSDeliveryMode.label)) { + if (value != null) outMessage.setJMSDeliveryMode(Integer.parseInt(value)); + } else if (StringUtils.equalsIgnoreCase(msgHeaderKey, S4JAdapterUtil.JMS_MSG_HEADER_STD.JMSExpiration.label)) { + // TODO: convert from a Date/Time string to the required long value + if (value != null) outMessage.setJMSExpiration(Long.parseLong(value)); + } else if (StringUtils.equalsIgnoreCase(msgHeaderKey, S4JAdapterUtil.JMS_MSG_HEADER_STD.JMSCorrelationID.label)) { + if (value != null) outMessage.setJMSCorrelationID(value); + } else if (StringUtils.equalsIgnoreCase(msgHeaderKey, S4JAdapterUtil.JMS_MSG_HEADER_STD.JMSReplyTo.label)) { + // 'JMSReplyTo' value format: "[topic|queue]:" + if (value != null) { + String destType = StringUtils.substringBefore(value, ':'); + String destName = StringUtils.substringAfter(value, ':'); + outMessage.setJMSReplyTo(getOrCreateJmsDestination(s4JJMSContextWrapper,false, destType, destName)); + } + } + // Ignore these headers - handled by S4J API automatically + /* else if (StringUtils.equalsAnyIgnoreCase(msgHeaderKey, + S4JAdapterUtil.JMS_MSG_HEADER_STD.JMSDestination.label, + S4JAdapterUtil.JMS_MSG_HEADER_STD.JMSMessageID.label, + S4JAdapterUtil.JMS_MSG_HEADER_STD.JMSTimestamp.label, + S4JAdapterUtil.JMS_MSG_HEADER_STD.JMSRedelivered.label + )) { + }*/ + } catch (NumberFormatException nfe) { + logger.warn("Incorrect value format ('{}') for the message header field ('{}')!", + value, msgHeaderKey); + } + } + } + + outMessage.setStringProperty(S4JAdapterUtil.NB_MSG_SIZE_PROP, String.valueOf(messageSize)); + + return outMessage; + } + + private Message updateMessageProperties(Message message, String msgPropertyRawJsonStr) throws JMSException { + int messageSize = Integer.parseInt(message.getStringProperty(S4JAdapterUtil.NB_MSG_SIZE_PROP)); + + // Check if jmsMsgPropertyRawJsonStr is a valid JSON string with a collection of key/value pairs + // - if Yes, convert it to a map + // - otherwise, log an error message and ignore message headers without throwing a runtime exception + Map jmsMsgProperties = new HashMap<>(); + if (!StringUtils.isBlank(msgPropertyRawJsonStr)) { + try { + jmsMsgProperties = S4JAdapterUtil.convertJsonToMap(msgPropertyRawJsonStr); + } catch (Exception e) { + logger.warn( + "Error parsing message property JSON string {}, ignore message properties!", + msgPropertyRawJsonStr); + } + } + + // Each key in the property json file may include value type information, such as: + // - key(string): value + // The above format specifies a message property that has "key" as the property key + // and "value" as the property value; and the type of the property value is "string" + // + // If the value type is not specified, use "string" as the default value type. + for (Map.Entry entry : jmsMsgProperties.entrySet()) { + String rawKeyStr = entry.getKey(); + String value = entry.getValue(); + + if (! StringUtils.isAnyBlank(rawKeyStr, value)) { + String key = rawKeyStr; + String valueType = S4JAdapterUtil.JMS_MSG_PROP_TYPES.STRING.label; + + if (StringUtils.contains(rawKeyStr, '(')) { + key = StringUtils.substringBefore(rawKeyStr, "(").trim(); + valueType = StringUtils.substringAfter(rawKeyStr, "("); + valueType = StringUtils.substringBefore(valueType, ")").trim(); + } + + if (StringUtils.isBlank(valueType)) { + message.setStringProperty(entry.getKey(), value); + } + else { + if (StringUtils.equalsIgnoreCase(valueType, S4JAdapterUtil.JMS_MSG_PROP_TYPES.SHORT.label)) + message.setShortProperty(key, NumberUtils.toShort(value)); + else if (StringUtils.equalsIgnoreCase(valueType, S4JAdapterUtil.JMS_MSG_PROP_TYPES.INT.label)) + message.setIntProperty(key, NumberUtils.toInt(value)); + else if (StringUtils.equalsIgnoreCase(valueType, S4JAdapterUtil.JMS_MSG_PROP_TYPES.LONG.label)) + message.setLongProperty(key, NumberUtils.toLong(value)); + else if (StringUtils.equalsIgnoreCase(valueType, S4JAdapterUtil.JMS_MSG_PROP_TYPES.FLOAT.label)) + message.setFloatProperty(key, NumberUtils.toFloat(value)); + else if (StringUtils.equalsIgnoreCase(valueType, S4JAdapterUtil.JMS_MSG_PROP_TYPES.DOUBLE.label)) + message.setDoubleProperty(key, NumberUtils.toDouble(value)); + else if (StringUtils.equalsIgnoreCase(valueType, S4JAdapterUtil.JMS_MSG_PROP_TYPES.BOOLEAN.label)) + message.setBooleanProperty(key, BooleanUtils.toBoolean(value)); + else if (StringUtils.equalsIgnoreCase(valueType, S4JAdapterUtil.JMS_MSG_PROP_TYPES.STRING.label)) + message.setStringProperty(key, value); + else if (StringUtils.equalsIgnoreCase(valueType, S4JAdapterUtil.JMS_MSG_PROP_TYPES.BYTE.label)) + message.setByteProperty(key, NumberUtils.toByte(value)); + else + throw new S4JAdapterInvalidParamException( + "Unsupported JMS message property value type (\"" + valueType + "\"). " + + "Value types are: \"" + S4JAdapterUtil.getValidJmsMsgPropTypeList() + "\""); + } + + messageSize += key.length(); + messageSize += value.length(); + } + } + + message.setStringProperty(S4JAdapterUtil.NB_MSG_SIZE_PROP, String.valueOf(messageSize)); + + return message; + } + + @Override + public MessageProducerOp apply(long cycle) { + String destName = destNameStrFunc.apply(cycle); + String jmsMsgHeaderRawJsonStr = msgHeaderRawJsonStrFunc.apply(cycle); + String jmsMsgPropertyRawJsonStr = msgPropRawJsonStrFunc.apply(cycle); + String jmsMsgBodyRawJsonStr = msgBodyRawJsonStrFunc.apply(cycle); + + if (StringUtils.isBlank(jmsMsgBodyRawJsonStr)) { + throw new S4JAdapterInvalidParamException("Message payload must be specified and can't be empty!"); + } + + S4JJMSContextWrapper s4JJMSContextWrapper = getOrCreateS4jJmsContextWrapper(cycle); + JMSContext jmsContext = s4JJMSContextWrapper.getJmsContext(); + boolean commitTransaction = !super.commitTransaction(txnBatchNum, jmsContext.getSessionMode(), cycle); + + Destination destination; + try { + destination = getOrCreateJmsDestination(s4JJMSContextWrapper, temporaryDest, destType, destName); + } + catch (JMSRuntimeException jmsRuntimeException) { + throw new S4JAdapterUnexpectedException("Unable to create the JMS destination!"); + } + + JMSProducer producer; + try { + producer = getOrCreateJmsProducer(s4JJMSContextWrapper, asyncAPI); + } + catch (JMSException jmsException) { + throw new S4JAdapterUnexpectedException("Unable to create the JMS producer!"); + } + + // Get the right JMS message type + String jmsMsgType = msgTypeFunc.apply(cycle); + if (! S4JAdapterUtil.isValidJmsMessageType(jmsMsgType) ) { + logger.warn( + "The specified JMS message type {} is not valid, use the default TextMessage type!", + jmsMsgType); + jmsMsgType = S4JAdapterUtil.JMS_MESSAGE_TYPES.TEXT.label; + } + + + ///////////// + // Set proper message payload based on the message type and the specified input + // ----------------------- + // + Message message; + try { + message = createAndSetMessagePayload(s4JJMSContextWrapper, jmsMsgType, jmsMsgBodyRawJsonStr); + } + catch (JMSException jmsException) { + throw new RuntimeException("Failed to set create a JMS message and set its payload!"); + } + + ///////////// + // Set standard message headers + // ----------------------- + // + try { + message = updateMessageHeaders(s4JJMSContextWrapper, message, jmsMsgType, jmsMsgHeaderRawJsonStr); + } + catch (JMSException jmsException) { + throw new S4JAdapterUnexpectedException("Failed to set create a JMS message and set its payload!"); + } + + ///////////// + // Set defined JMS message properties and other custom properties + // ----------------------- + // + try { + message = updateMessageProperties(message, jmsMsgPropertyRawJsonStr); + // for testing purpose + message.setLongProperty(S4JAdapterUtil.NB_MSG_SEQ_PROP, cycle); + } + catch (JMSException jmsException) { + throw new S4JAdapterUnexpectedException("Failed to set JMS message properties!"); + } + + return new MessageProducerOp( + s4jAdapterMetrics, + s4jSpace, + jmsContext, + destination, + asyncAPI, + commitTransaction, + producer, + message); + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/S4JBaseOpDispenser.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/S4JBaseOpDispenser.java new file mode 100644 index 000000000..f49fda8aa --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/S4JBaseOpDispenser.java @@ -0,0 +1,395 @@ +package io.nosqlbench.adapter.s4j.dispensers; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import com.datastax.oss.pulsar.jms.PulsarConnectionFactory; +import com.datastax.oss.pulsar.jms.PulsarJMSContext; +import io.nosqlbench.adapter.s4j.S4JSpace; +import io.nosqlbench.adapter.s4j.ops.S4JOp; +import io.nosqlbench.adapter.s4j.util.*; +import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.jms.*; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.LongFunction; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public abstract class S4JBaseOpDispenser extends BaseOpDispenser { + + private final static Logger logger = LogManager.getLogger("PulsarBaseOpDispenser"); + + protected final ParsedOp parsedOp; + protected final S4JSpace s4jSpace; + protected final S4JAdapterMetrics s4jAdapterMetrics; + + private final ConcurrentHashMap connLvlJmsContexts = new ConcurrentHashMap<>(); + private final ConcurrentHashMap sessionLvlJmsContexts = new ConcurrentHashMap<>(); + protected final ConcurrentHashMap jmsDestinations = new ConcurrentHashMap<>(); + protected final ConcurrentHashMap jmsProducers = new ConcurrentHashMap<>(); + protected final ConcurrentHashMap jmsConsumers = new ConcurrentHashMap<>(); + + // Doc-level parameter: temporary_dest (default: false) + protected final boolean temporaryDest; + // Doc-level parameter: dest_type (default: Topic) + protected final String destType; + // Doc-level parameter: async_api (default: true) + protected final boolean asyncAPI; + // Doc-level parameter: txn_batch_num (default: 0) + // - value <=0 : no transaction + protected final int txnBatchNum; + + protected final LongFunction destNameStrFunc; + + protected final int totalThreadNum; + protected final long totalCycleNum; + + public S4JBaseOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction destNameStrFunc, + S4JSpace s4jSpace) { + + super(adapter, op); + + this.parsedOp = op; + this.s4jSpace = s4jSpace; + this.connLvlJmsContexts.putAll(s4jSpace.getConnLvlJmsContexts()); + this.sessionLvlJmsContexts.putAll(s4jSpace.getSessionLvlJmsContexts()); + + String defaultMetricsPrefix = getDefaultMetricsPrefix(this.parsedOp); + this.s4jAdapterMetrics = new S4JAdapterMetrics(defaultMetricsPrefix); + s4jAdapterMetrics.initS4JAdapterInstrumentation(); + + this.destNameStrFunc = destNameStrFunc; + this.temporaryDest = + parsedOp.getStaticConfigOr(S4JAdapterUtil.DOC_LEVEL_PARAMS.TEMP_DEST.label, Boolean.FALSE); + this.destType = + parsedOp.getStaticConfig(S4JAdapterUtil.DOC_LEVEL_PARAMS.DEST_TYPE.label, String.class); + this.asyncAPI = + parsedOp.getStaticConfigOr(S4JAdapterUtil.DOC_LEVEL_PARAMS.ASYNC_API.label, Boolean.TRUE); + this.txnBatchNum = + parsedOp.getStaticConfigOr(S4JAdapterUtil.DOC_LEVEL_PARAMS.TXN_BATCH_NUM.label, Integer.valueOf(0)); + + this.totalThreadNum = NumberUtils.toInt(parsedOp.getStaticValue("threads")); + this.totalCycleNum = NumberUtils.toLong(parsedOp.getStaticValue("cycles")); + s4jSpace.setTotalCycleNum(totalCycleNum); + } + + public S4JSpace getS4jSpace() { return s4jSpace; } + public S4JAdapterMetrics getS4jAdapterMetrics() { return s4jAdapterMetrics; } + + protected LongFunction lookupStaticBoolConfigValueFunc(String paramName, boolean defaultValue) { + LongFunction booleanLongFunction; + booleanLongFunction = (l) -> parsedOp.getOptionalStaticConfig(paramName, String.class) + .filter(Predicate.not(String::isEmpty)) + .map(value -> BooleanUtils.toBoolean(value)) + .orElse(defaultValue); + logger.info("{}: {}", paramName, booleanLongFunction.apply(0)); + return booleanLongFunction; + } + + protected LongFunction> lookupStaticStrSetOpValueFunc(String paramName) { + LongFunction> setStringLongFunction; + setStringLongFunction = (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) + .filter(Predicate.not(String::isEmpty)) + .map(value -> { + Set set = new HashSet<>(); + + if (StringUtils.contains(value,',')) { + set = Arrays.stream(value.split(",")) + .map(String::trim) + .filter(Predicate.not(String::isEmpty)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + return set; + }).orElse(Collections.emptySet()); + logger.info("{}: {}", paramName, setStringLongFunction.apply(0)); + return setStringLongFunction; + } + + // If the corresponding Op parameter is not provided, use the specified default value + protected LongFunction lookupStaticIntOpValueFunc(String paramName, int defaultValue) { + LongFunction integerLongFunction; + integerLongFunction = (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) + .filter(Predicate.not(String::isEmpty)) + .map(value -> NumberUtils.toInt(value)) + .map(value -> { + if (value < 0) return 0; + else return value; + }).orElse(defaultValue); + logger.info("{}: {}", paramName, integerLongFunction.apply(0)); + return integerLongFunction; + } + + // If the corresponding Op parameter is not provided, use the specified default value + protected LongFunction lookupOptionalStrOpValueFunc(String paramName, String defaultValue) { + LongFunction stringLongFunction; + stringLongFunction = parsedOp.getAsOptionalFunction(paramName, String.class) + .orElse((l) -> defaultValue); + logger.info("{}: {}", paramName, stringLongFunction.apply(0)); + + return stringLongFunction; + } + protected LongFunction lookupOptionalStrOpValueFunc(String paramName) { + return lookupOptionalStrOpValueFunc(paramName, ""); + } + + // Mandatory Op parameter. Throw an error if not specified or having empty value + protected LongFunction lookupMandtoryStrOpValueFunc(String paramName) { + LongFunction stringLongFunction; + stringLongFunction = parsedOp.getAsRequiredFunction(paramName, String.class); + logger.info("{}: {}", paramName, stringLongFunction.apply(0)); + + return stringLongFunction; + } + + // Get the next JMSContext Wrapper in the following approach + // - The JMSContext wrapper pool has the following sequence (assuming 3 [c]onnections and 2 [s]essions per connection): + // c0s0, c0s1, c1s0, c1s1, c2s0, c2s1 + // - When getting the next JMSContext wrapper, always get from the next connection, starting from the first session + // When reaching the end of connection, move back to the first connection, but get the next session. + // e.g. first: c0s0 (0) + // next: c1s0 (1) + // next: c2s0 (2) + // next: c0s1 (3) + // next: c1s1 (4) + // next: c2s1 (5) + // next: c0s0 (6) <-- repeat the pattern + // next: c1s0 (7) + // next: c2s0 (8) + // next: c0s1 (9) + // ... ... + public S4JJMSContextWrapper getOrCreateS4jJmsContextWrapper( + long curCycle, + Map overrideS4jConfMap) + { + int totalConnNum = s4jSpace.getMaxNumConn(); + int totalSessionPerConnNum = s4jSpace.getMaxNumSessionPerConn(); + + int connSeqNum = (int) curCycle % totalConnNum; + int sessionSeqNum = ( (int)(curCycle / totalConnNum) ) % totalSessionPerConnNum; + + String jmsConnContextIdStr = s4jSpace.getConnLvlJmsContextIdentifier(connSeqNum); + JMSContext connLvlJmsContext = connLvlJmsContexts.get(jmsConnContextIdStr); + // Connection level JMSContext objects should be already created during the initialization phase + assert (connLvlJmsContext != null); + + String jmsSessionContextIdStr = s4jSpace.getSessionLvlJmsContextIdentifier(connSeqNum, sessionSeqNum); + S4JJMSContextWrapper jmsContextWrapper = sessionLvlJmsContexts.get(jmsSessionContextIdStr); + + if (jmsContextWrapper == null) { + JMSContext jmsContext = null; + + if (overrideS4jConfMap == null || overrideS4jConfMap.isEmpty()) { + jmsContext = connLvlJmsContext.createContext(connLvlJmsContext.getSessionMode()); + } else { + jmsContext = ((PulsarJMSContext) connLvlJmsContext).createContext( + connLvlJmsContext.getSessionMode(), overrideS4jConfMap); + } + + jmsContextWrapper = new S4JJMSContextWrapper(jmsSessionContextIdStr, jmsContext); + sessionLvlJmsContexts.put(jmsSessionContextIdStr, jmsContextWrapper); + + if (logger.isDebugEnabled()) { + logger.debug("[Session level JMSContext] {} -- {}", + Thread.currentThread().getName(), + jmsContextWrapper); + } + + } + return jmsContextWrapper; + } + + public S4JJMSContextWrapper getOrCreateS4jJmsContextWrapper(long curCycle) { + return getOrCreateS4jJmsContextWrapper(curCycle, null); + } + + /** + * If the JMS destination that corresponds to a topic exists, reuse it; Otherwise, create it + */ + public Destination getOrCreateJmsDestination( + S4JJMSContextWrapper s4JJMSContextWrapper, + boolean tempDest, + String destType, + String destName) throws JMSRuntimeException + { + String jmsContextIdStr = s4JJMSContextWrapper.getJmsContextIdentifer(); + JMSContext jmsContext = s4JJMSContextWrapper.getJmsContext(); + + // Regular, non-temporary destination + if (!tempDest) { + String destinationCacheKey = S4JAdapterUtil.buildCacheKey(jmsContextIdStr, destType, destName); + Destination destination = jmsDestinations.get(destinationCacheKey); + + if (destination == null) { + if (StringUtils.equalsIgnoreCase(destType, S4JAdapterUtil.JMS_DEST_TYPES.QUEUE.label)) { + destination = jmsContext.createQueue(destName); + } else { + destination = jmsContext.createTopic(destName); + } + + jmsDestinations.put(destinationCacheKey, destination); + } + + return destination; + } + // Temporary destination + else { + if (StringUtils.equalsIgnoreCase(destType, S4JAdapterUtil.JMS_DEST_TYPES.QUEUE.label)) { + return jmsContext.createTemporaryQueue(); + } else { + return jmsContext.createTemporaryTopic(); + } + } + } + + // Get simplified NB thread name + private String getSimplifiedNBThreadName(String fullThreadName) { + assert (StringUtils.isNotBlank(fullThreadName)); + + if (StringUtils.contains(fullThreadName, '/')) + return StringUtils.substringAfterLast(fullThreadName, "/"); + else + return fullThreadName; + } + + + /** + * If the JMS producer that corresponds to a destination exists, reuse it; Otherwise, create it + */ + public JMSProducer getOrCreateJmsProducer( + S4JJMSContextWrapper s4JJMSContextWrapper, + boolean asyncApi) throws JMSException + { + JMSContext jmsContext = s4JJMSContextWrapper.getJmsContext(); + String producerCacheKey = S4JAdapterUtil.buildCacheKey( + getSimplifiedNBThreadName(Thread.currentThread().getName()), "producer"); + JMSProducer jmsProducer = jmsProducers.get(producerCacheKey); + + if (jmsProducer == null) { + jmsProducer = jmsContext.createProducer(); + + if (asyncApi) { + jmsProducer.setAsync(new S4JCompletionListener(s4jSpace, this)); + } + + if (logger.isDebugEnabled()) { + logger.debug("Producer created: {} -- {} -- {}", + producerCacheKey, jmsProducer, s4JJMSContextWrapper); + } + + jmsProducers.put(producerCacheKey, jmsProducer); + } + + return jmsProducer; + } + + /** + * If the JMS consumer that corresponds to a destination(, subscription, message selector) exists, reuse it; Otherwise, create it + */ + public JMSConsumer getOrCreateJmsConsumer( + S4JJMSContextWrapper s4JJMSContextWrapper, + Destination destination, + String destType, + String subName, + String msgSelector, + float msgAckRatio, + boolean nonLocal, + boolean durable, + boolean shared, + boolean asyncApi, + int slowAckInSec) throws JMSException + { + JMSContext jmsContext = s4JJMSContextWrapper.getJmsContext(); + boolean isTopic = StringUtils.equalsIgnoreCase(destType, S4JAdapterUtil.JMS_DEST_TYPES.TOPIC.label); + String consumerCacheKey = S4JAdapterUtil.buildCacheKey( + getSimplifiedNBThreadName(Thread.currentThread().getName()), "consumer"); + + JMSConsumer jmsConsumer = jmsConsumers.get(consumerCacheKey); + if (jmsConsumer == null) { + if (isTopic) { + if (!durable && !shared) + jmsConsumer = jmsContext.createConsumer(destination, msgSelector, nonLocal); + else { + if (StringUtils.isBlank(subName)) { + throw new RuntimeException("Subscription name is required for receiving messages from a durable or shared topic!"); + } + + if (durable && !shared) + jmsConsumer = jmsContext.createDurableConsumer((Topic) destination, subName, msgSelector, nonLocal); + else if (!durable) + jmsConsumer = jmsContext.createSharedConsumer((Topic) destination, subName, msgSelector); + else + jmsConsumer = jmsContext.createSharedDurableConsumer((Topic) destination, subName, msgSelector); + } + } + else { + jmsConsumer = jmsContext.createConsumer(destination, msgSelector, nonLocal); + } + + if (asyncApi) { + jmsConsumer.setMessageListener( + new S4JMessageListener(jmsContext, s4jSpace, this, msgAckRatio, slowAckInSec)); + } + + if (logger.isDebugEnabled()) { + logger.debug("Consumer created: {} -- {} -- {}", + consumerCacheKey, jmsConsumer, s4JJMSContextWrapper); + } + + jmsConsumers.put(consumerCacheKey, jmsConsumer); + } + + return jmsConsumer; + } + + protected boolean commitTransaction(int txnBatchNum, int jmsSessionMode, long curCycleNum) { + // Whether to commit the transaction which happens when: + // - session mode is equal to "SESSION_TRANSACTED" + // - "txn_batch_num" has been reached since last reset + boolean commitTransaction = ( (Session.SESSION_TRANSACTED == jmsSessionMode) && (txnBatchNum > 0) ); + if (commitTransaction) { + int txnBatchTackingCnt = s4jSpace.getTxnBatchTrackingCnt(); + + if ( ( (txnBatchTackingCnt > 0) && ((txnBatchTackingCnt % txnBatchNum) == 0) ) || + ( curCycleNum >= (totalCycleNum - 1) ) ) { + if (logger.isDebugEnabled()) { + logger.debug("Commit transaction ({}, {}, {})", + txnBatchTackingCnt, + s4jSpace.getTotalOpResponseCnt(), curCycleNum); + } + } + else { + commitTransaction = false; + } + + s4jSpace.incTxnBatchTrackingCnt(); + } + + return !commitTransaction; + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterAsyncOperationFailedException.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterAsyncOperationFailedException.java new file mode 100644 index 000000000..70d2f7dec --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterAsyncOperationFailedException.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.s4j.exception; + +public class S4JAdapterAsyncOperationFailedException extends RuntimeException { + + public S4JAdapterAsyncOperationFailedException(Throwable t) { + super(t); + printStackTrace(); + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterInvalidParamException.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterInvalidParamException.java new file mode 100644 index 000000000..0b4a35bc8 --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterInvalidParamException.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.s4j.exception; + +public class S4JAdapterInvalidParamException extends RuntimeException { + + public S4JAdapterInvalidParamException(String paramName, String errDesc) { + super("Invalid setting for parameter (" + paramName + "): " + errDesc); + } + + public S4JAdapterInvalidParamException(String fullErrDesc) { + super(fullErrDesc); + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterUnexpectedException.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterUnexpectedException.java new file mode 100644 index 000000000..f96ce2edb --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterUnexpectedException.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.s4j.exception; + +public class S4JAdapterUnexpectedException extends RuntimeException { + + public S4JAdapterUnexpectedException(String message) { + super(message); + printStackTrace(); + } + public S4JAdapterUnexpectedException(Exception e) { + super(e); + printStackTrace(); + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterUnsupportedOpException.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterUnsupportedOpException.java new file mode 100644 index 000000000..7a86d66f2 --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/exception/S4JAdapterUnsupportedOpException.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.s4j.exception; + +public class S4JAdapterUnsupportedOpException extends RuntimeException { + + public S4JAdapterUnsupportedOpException(String pulsarOpType) { + super("Unsupported Pulsar adapter operation type: \"" + pulsarOpType + "\""); + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/MessageConsumerOp.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/MessageConsumerOp.java new file mode 100644 index 000000000..cdc911aa0 --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/MessageConsumerOp.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.s4j.ops; + +import com.codahale.metrics.Timer; +import io.nosqlbench.adapter.s4j.S4JSpace; +import io.nosqlbench.adapter.s4j.exception.S4JAdapterAsyncOperationFailedException; +import io.nosqlbench.adapter.s4j.exception.S4JAdapterUnexpectedException; +import io.nosqlbench.adapter.s4j.util.S4JAdapterMetrics; +import io.nosqlbench.adapter.s4j.util.S4JAdapterUtil; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.shade.org.apache.avro.AvroRuntimeException; + +import javax.jms.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class MessageConsumerOp extends S4JOp { + + private final static Logger logger = LogManager.getLogger(MessageConsumerOp.class); + + private final JMSConsumer jmsConsumer; + private final boolean blockingMsgRecv; + private final float msgAckRatio; + private final long msgReadTimeout; + private final boolean recvNoWait; + private final int slowInSec; + + public MessageConsumerOp(S4JAdapterMetrics s4jAdapterMetrics, + S4JSpace s4jSpace, + JMSContext jmsContext, + Destination destination, + boolean asyncApi, + boolean commitTransact, + JMSConsumer jmsConsumer, + boolean blockingMsgRecv, + float msgAckRatio, + long readTimeout, + boolean recvNoWait, + int slowInSec) + { + super(s4jAdapterMetrics, s4jSpace, jmsContext, destination, asyncApi, commitTransact); + + this.jmsConsumer = jmsConsumer; + this.blockingMsgRecv = blockingMsgRecv; + this.msgAckRatio = msgAckRatio; + this.msgReadTimeout = readTimeout; + this.recvNoWait = recvNoWait; + this.slowInSec = slowInSec; + } + + @Override + public Object apply(long value) { + long timeElapsedMills = System.currentTimeMillis() - s4jOpStartTimeMills; + + // If maximum S4J operation duration is specified, only receive messages + // before the maximum duration threshold is reached. Otherwise, this is + // just no-op. + if ( (maxS4jOpDurationInSec == 0) || (timeElapsedMills <= (maxS4jOpDurationInSec*1000)) ) { + + // Please see S4JSpace::getOrCreateJmsConsumer() for async processing + if (!asyncApi) { + Message recvdMsg; + + try { + // blocking message receiving only applies to synchronous API + if (blockingMsgRecv) { + recvdMsg = jmsConsumer.receive(); + } else if (recvNoWait) { + recvdMsg = jmsConsumer.receiveNoWait(); + } else { + // timeout value 0 means to wait forever + recvdMsg = jmsConsumer.receive(msgReadTimeout); + } + if (this.commitTransact) jmsContext.commit(); + + if (recvdMsg != null) { + s4jSpace.processMsgAck(jmsContext, recvdMsg, msgAckRatio, slowInSec); + + byte[] recvdMsgBody = recvdMsg.getBody(byte[].class); + int messageSize = recvdMsgBody.length; + + messageSizeHistogram.update(messageSize); + + if (logger.isDebugEnabled()) { + // for testing purpose + String myMsgSeq = recvdMsg.getStringProperty(S4JAdapterUtil.NB_MSG_SEQ_PROP); + logger.debug("Sync message receive successful - message ID {} ({}) " + , recvdMsg.getJMSMessageID(), myMsgSeq); + } + + if (s4jSpace.isTrackingMsgRecvCnt()) { + s4jSpace.incTotalOpResponseCnt(); + } + } else { + if (s4jSpace.isTrackingMsgRecvCnt()) { + s4jSpace.incTotalNullMsgRecvdCnt(); + } + } + } catch (JMSException | JMSRuntimeException e) { + S4JAdapterUtil.processMsgErrorHandling( + e, + s4jSpace.isStrictMsgErrorHandling(), + "Unexpected errors when sync receiving a JMS message."); + } + } + } + else { + if (logger.isTraceEnabled()) { + logger.trace("NB cycle number {} is no-op (maxS4jOpDurationInSec: {}, timeElapsedMills: {})", + value, maxS4jOpDurationInSec, timeElapsedMills); + } + } + + + return null; + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/MessageProducerOp.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/MessageProducerOp.java new file mode 100644 index 000000000..2624c36cf --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/MessageProducerOp.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.s4j.ops; + + +import com.codahale.metrics.Histogram; +import io.nosqlbench.adapter.s4j.S4JSpace; +import io.nosqlbench.adapter.s4j.util.S4JAdapterMetrics; +import io.nosqlbench.adapter.s4j.util.S4JAdapterUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +import javax.jms.*; +import java.util.HashMap; +import java.util.Map; + + +public class MessageProducerOp extends S4JOp { + + private final static Logger logger = LogManager.getLogger("MessageProducerOp"); + + private final JMSProducer jmsProducer; + private final Message message; + + public MessageProducerOp(S4JAdapterMetrics s4jAdapterMetrics, + S4JSpace s4jSpace, + JMSContext jmsContext, + Destination destination, + boolean asyncApi, + boolean commitTransact, + JMSProducer jmsProducer, + Message message) { + super(s4jAdapterMetrics, s4jSpace, jmsContext, destination, asyncApi, commitTransact); + + this.jmsProducer = jmsProducer; + this.message = message; + } + + @Override + public Object apply(long value) { + + long timeElapsedMills = System.currentTimeMillis() - s4jOpStartTimeMills; + + // If maximum S4J operation duration is specified, only publish messages + // before the maximum duration threshold is reached. Otherwise, this is + // just no-op. + if ( (maxS4jOpDurationInSec == 0) || (timeElapsedMills <= (maxS4jOpDurationInSec*1000)) ) { + try { + jmsProducer.send(destination, message); + if (this.commitTransact) { + jmsContext.commit(); + } + + int msgSize = message.getIntProperty(S4JAdapterUtil.NB_MSG_SIZE_PROP); + messageSizeHistogram.update(msgSize); + + // Please see s4JSpace::getOrCreateJmsProducer() for async processing + if (!asyncApi) { + if (logger.isDebugEnabled()) { + // for testing purpose + String myMsgSeq = message.getStringProperty(S4JAdapterUtil.NB_MSG_SEQ_PROP); + logger.debug("Sync message sending is successful - message ID {} ({}) " + , message.getJMSMessageID(), myMsgSeq); + } + + if (s4jSpace.isTrackingMsgRecvCnt()) { + s4jSpace.incTotalOpResponseCnt(); + } + } + } catch (JMSException | JMSRuntimeException e) { + S4JAdapterUtil.processMsgErrorHandling( + e, + s4jSpace.isStrictMsgErrorHandling(), + "Unexpected errors when sync sending a JMS message."); + } + } + else { + if (logger.isTraceEnabled()) { + logger.trace("NB cycle number {} is no-op (maxS4jOpDurationInSec: {}, timeElapsedMills: {})", + value, maxS4jOpDurationInSec, timeElapsedMills); + } + } + + return null; + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/S4JOp.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/S4JOp.java new file mode 100644 index 000000000..4fee61dab --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/ops/S4JOp.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.s4j.ops; + +import com.codahale.metrics.Histogram; +import io.nosqlbench.adapter.s4j.S4JSpace; +import io.nosqlbench.adapter.s4j.util.S4JAdapterMetrics; +import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp; + +import javax.jms.Destination; +import javax.jms.JMSContext; + +public abstract class S4JOp implements CycleOp { + protected S4JAdapterMetrics s4jAdapterMetrics; + protected final S4JSpace s4jSpace; + protected final JMSContext jmsContext; + protected final Destination destination; + protected final boolean asyncApi; + protected final boolean commitTransact; + protected final long s4jOpStartTimeMills; + protected final long maxS4jOpDurationInSec; + protected final Histogram messageSizeHistogram; + + + public S4JOp( + S4JAdapterMetrics s4jAdapterMetrics, + S4JSpace s4jSpace, + JMSContext jmsContext, + Destination destination, + boolean asyncApi, + boolean commitTransact) + { + this.s4jAdapterMetrics = s4jAdapterMetrics; + this.s4jSpace = s4jSpace; + this.jmsContext = jmsContext; + this.destination = destination; + this.asyncApi = asyncApi; + this.commitTransact = commitTransact; + this.s4jOpStartTimeMills = s4jSpace.getS4JActivityStartTimeMills(); + this.maxS4jOpDurationInSec = s4jSpace.getMaxS4JOpTimeInSec(); + this.messageSizeHistogram = s4jAdapterMetrics.getMessagesizeHistogram(); + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JAdapterMetrics.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JAdapterMetrics.java new file mode 100644 index 000000000..2903353a3 --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JAdapterMetrics.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.s4j.util; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Timer; +import io.nosqlbench.api.config.NBNamedElement; +import io.nosqlbench.api.engine.metrics.ActivityMetrics; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class S4JAdapterMetrics implements NBNamedElement { + + private final static Logger logger = LogManager.getLogger("S4JAdapterMetrics"); + + private final String defaultAdapterMetricsPrefix; + + private Histogram messageSizeHistogram; + private Timer bindTimer; + private Timer executeTimer; + + public S4JAdapterMetrics(String defaultMetricsPrefix) { + this.defaultAdapterMetricsPrefix = defaultMetricsPrefix; + } + + @Override + public String getName() { + return "S4JAdapterMetrics"; + } + + public void initS4JAdapterInstrumentation() { + // Histogram metrics + this.messageSizeHistogram = + ActivityMetrics.histogram( + this, + defaultAdapterMetricsPrefix + "message_size", + ActivityMetrics.DEFAULT_HDRDIGITS); + + // Timer metrics + this.bindTimer = + ActivityMetrics.timer( + this, + defaultAdapterMetricsPrefix + "bind", + ActivityMetrics.DEFAULT_HDRDIGITS); + this.executeTimer = + ActivityMetrics.timer( + this, + defaultAdapterMetricsPrefix + "execute", + ActivityMetrics.DEFAULT_HDRDIGITS); + } + + public Timer getBindTimer() { return bindTimer; } + public Timer getExecuteTimer() { return executeTimer; } + public Histogram getMessagesizeHistogram() { return messageSizeHistogram; } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JAdapterUtil.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JAdapterUtil.java new file mode 100644 index 000000000..8906b2565 --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JAdapterUtil.java @@ -0,0 +1,326 @@ +package io.nosqlbench.adapter.s4j.util; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.datastax.oss.pulsar.jms.PulsarJMSConstants; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.nosqlbench.adapter.s4j.S4JOpType; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.jms.*; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class S4JAdapterUtil { + + private final static Logger logger = LogManager.getLogger(S4JAdapterUtil.class); + + /////// + // Valid document level parameters for JMS NB yaml file + public final static String JMS_SPEC_VER_12 = "1.2"; + public final static String JMS_SPEC_VER_20 = "2.0"; + public enum DOC_LEVEL_PARAMS { + + // Temporary destination + TEMP_DEST("temporary_dest"), + // JMS destination type - topic or queue + // String value + // - valid values: see JMS_DEST_TYPES + DEST_TYPE("dest_type"), + // JMS destination name + // String value + DEST_NAME("dest_name"), + // Asynchronous message processing + ASYNC_API("async_api"), + // Transaction batch size + // - Only relevant when session mode is SESSION_TRANSACTED + TXN_BATCH_NUM("txn_batch_num"), + // Whether to use blocking message receiving as the default behavior + BLOCKING_MSG_RECV("blocking_msg_recv"), + // Whether the destination is a shared topic + SHARED_TOPIC("shared_topic"), + // Whether the destination is a durable topic + DURABLE_TOPIC("durable_topic"); + + public final String label; + + DOC_LEVEL_PARAMS(String label) { + this.label = label; + } + } + public static boolean isValidDocLevelParam(String param) { + return Arrays.stream(DOC_LEVEL_PARAMS.values()).anyMatch(t -> t.label.equals(param)); + } + public static String getValidDocLevelParamList() { + return Arrays.stream(DOC_LEVEL_PARAMS.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + // JMS Destination Types + public enum JMS_DEST_TYPES { + QUEUE("queue"), + TOPIC("topic"); + + public final String label; + JMS_DEST_TYPES(String label) { + this.label = label; + } + } + public static boolean isValidJmsDestType(String type) { + return Arrays.stream(JMS_DEST_TYPES.values()).anyMatch(t -> t.label.equals(type)); + } + public static String getValidJmsDestTypeList() { + return Arrays.stream(JMS_DEST_TYPES.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + // Standard JMS message headers (by JMS specification) + public enum JMS_MSG_HEADER_STD { + JMSDestination("JMSDestination"), + JMSDeliveryMode("JMSDeliveryMode"), + JMSMessageID("JMSMessageID"), + JMSTimestamp("JMSTimestamp"), + JMSRedelivered("JMSRedelivered"), + JMSExpiration("JMSExpiration"), + JMSCorrelationID("JMSCorrelationID"), + JMSType("JMSType"), + JMSReplyTo("JMSReplyTo"), + JMSPriority("JMSPriority"); + + public final String label; + JMS_MSG_HEADER_STD(String label) { + this.label = label; + } + } + public static boolean isValidStdJmsMsgHeader(String header) { + return Arrays.stream(JMS_MSG_HEADER_STD.values()).anyMatch(t -> t.label.equals(header)); + } + public static String getValidStdJmsMsgHeaderList() { + return Arrays.stream(JMS_MSG_HEADER_STD.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + // JMS defined message properties (by JMS specification) + public enum JMS_DEFINED_MSG_PROPERTY { + JMSDestination("JMSDestination"), + JMSDeliveryMode("JMSDeliveryMode"), + JMSMessageID("JMSMessageID"), + JMSTimestamp("JMSTimestamp"), + JMSRedelivered("JMSRedelivered"), + JMSExpiration("JMSExpiration"), + JMSCorrelationID("JMSCorrelationID"), + JMSType("JMSType"), + JMSReplyTo("JMSReplyTo"), + JMSPriority("JMSPriority"); + + public final String label; + JMS_DEFINED_MSG_PROPERTY(String label) { + this.label = label; + } + } + public static boolean isValidJmsDfndMsgProp(String property) { + return Arrays.stream(JMS_DEFINED_MSG_PROPERTY.values()).anyMatch(t -> t.label.equals(property)); + } + public static String getValidJmsDfndMsgPropList() { + return Arrays.stream(JMS_DEFINED_MSG_PROPERTY.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + public final static String NB_MSG_SEQ_PROP = "NBMsgSeqProp"; + public final static String NB_MSG_SIZE_PROP = "NBMsgSize"; + + // JMS Destination Types + public enum JMS_SESSION_MODES { + AUTO_ACK("auto_ack"), + CLIENT_ACK("client_ack"), + DUPS_OK_ACK("dups_ok_ack"), + INDIVIDUAL_ACK("individual_ack"), + TRANSACT("transact_ack"); + + public final String label; + JMS_SESSION_MODES(String label) { + this.label = label; + } + } + public static boolean isValidJmsSessionMode(String mode) { + return Arrays.stream(JMS_SESSION_MODES.values()).anyMatch(t -> t.label.equals(mode)); + } + public static String getValidJmsSessionModeList() { + return Arrays.stream(JMS_SESSION_MODES.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + // JMS Message Types + public enum JMS_MESSAGE_TYPES { + TEXT("text"), + BYTE("byte"), + MAP("map"), + STREAM("stream"), + OBJECT("object"); + + public final String label; + JMS_MESSAGE_TYPES(String label) { + this.label = label; + } + } + public static boolean isValidJmsMessageType(String type) { + return Arrays.stream(JMS_MESSAGE_TYPES.values()).anyMatch(t -> t.label.equals(type)); + } + public static String getValidJmsMessageTypeList() { + return Arrays.stream(JMS_MESSAGE_TYPES.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + // JMS Message Types + public enum JMS_MSG_PROP_TYPES { + SHORT("short"), + INT("int"), + LONG("long"), + FLOAT("float"), + DOUBLE("double"), + STRING("string"), + BOOLEAN("boolean"), + BYTE("byte"); + + public final String label; + JMS_MSG_PROP_TYPES(String label) { + this.label = label; + } + } + public static boolean isValidJmsMsgPropType(String type) { + return Arrays.stream(JMS_MSG_PROP_TYPES.values()).anyMatch(t -> t.label.equals(type)); + } + public static String getValidJmsMsgPropTypeList() { + return Arrays.stream(JMS_MSG_PROP_TYPES.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + /////// + // Convert JSON string to a key/value map + public static Map convertJsonToMap(String jsonStr) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(jsonStr, new TypeReference>(){}); + } + + /////// + // Convert JSON string to a list of objects + public static List convertJsonToObjList(String jsonStr) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + return Arrays.asList(mapper.readValue(jsonStr, Object[].class)); + } + + /////// + // Get the destination name from the Destination object + public static String getDestinationName(Destination destination, String destType) throws JMSException { + String destName; + + boolean isTopic = StringUtils.equalsIgnoreCase(destType, JMS_DEST_TYPES.TOPIC.label); + + if (isTopic) + destName = ((Topic) destination).getTopicName(); + else + destName = ((Queue) destination).getQueueName(); + + return destName; + } + + /////// + public static int getSessionModeFromStr(String sessionModeStr) { + // default ack mode: auto_ack + int sessionMode = -1; + + if (StringUtils.isBlank(sessionModeStr)) + sessionMode = JMSContext.AUTO_ACKNOWLEDGE; + else if (StringUtils.equalsIgnoreCase(sessionModeStr, JMS_SESSION_MODES.AUTO_ACK.label)) + sessionMode = JMSContext.AUTO_ACKNOWLEDGE; + else if (StringUtils.equalsIgnoreCase(sessionModeStr, JMS_SESSION_MODES.CLIENT_ACK.label)) + sessionMode = JMSContext.CLIENT_ACKNOWLEDGE; + else if (StringUtils.equalsIgnoreCase(sessionModeStr, JMS_SESSION_MODES.DUPS_OK_ACK.label)) + sessionMode = JMSContext.DUPS_OK_ACKNOWLEDGE; + else if (StringUtils.equalsIgnoreCase(sessionModeStr, JMS_SESSION_MODES.TRANSACT.label)) + sessionMode = JMSContext.SESSION_TRANSACTED; + else if (StringUtils.equalsIgnoreCase(sessionModeStr, JMS_SESSION_MODES.INDIVIDUAL_ACK.label)) + sessionMode = PulsarJMSConstants.INDIVIDUAL_ACKNOWLEDGE; + else { + if (logger.isDebugEnabled()) { + logger.debug("Invalid session mode string \"{}\". Valid values are: {}. Use the default \"auto_ack\" mode!" + ,sessionModeStr, getValidJmsSessionModeList()); + sessionMode = JMSContext.AUTO_ACKNOWLEDGE; + } + } + + return sessionMode; + } + + public static boolean isUseCredentialsEnabled(S4JClientConf s4JClientConf) { + assert (s4JClientConf != null); + + boolean enabled = false; + Map s4jConfMap = s4JClientConf.getS4jConfObjMap(); + + if (s4jConfMap.containsKey("jms.useCredentialsFromCreateConnection")) { + enabled = BooleanUtils.toBoolean(s4jConfMap.get("jms.useCredentialsFromCreateConnection").toString()); + } + return enabled; + } + + public static String getCredentialUserName(S4JClientConf s4JClientConf) { + return "dummy"; + } + + public static String getCredentialPassword(S4JClientConf s4JClientConf) { + Map s4jConfMap = s4JClientConf.getS4jConfObjMap(); + if (s4jConfMap.containsKey("authParams")) + return s4jConfMap.get("authParams").toString(); + else + return ""; + } + + /////// + // Calculate a unique cache key from a series of input parameters + public static String buildCacheKey(String... keyParts) { + return String.join("::", keyParts); + } + + + /////// + // Pause the execution of the current thread + public static void pauseCurThreadExec(int pauseInSec) { + if (pauseInSec > 0) { + try { + Thread.sleep(pauseInSec * 1000); + } + catch (InterruptedException ie) { + ie.printStackTrace(); + } + } + } + + /////// + // Error handling for message processing + public static void processMsgErrorHandling(Exception exception, boolean strictErrorHandling, String errorMsg) { + exception.printStackTrace(); + + if (strictErrorHandling) { + throw new RuntimeException(errorMsg + " [ " + exception.getMessage() + " ]"); + } + else { + S4JAdapterUtil.pauseCurThreadExec(1); + } + } +} + diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JClientConf.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JClientConf.java new file mode 100644 index 000000000..e2a62c5a1 --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JClientConf.java @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.s4j.util; + +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.FileBasedConfiguration; +import org.apache.commons.configuration2.PropertiesConfiguration; +import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder; +import org.apache.commons.configuration2.builder.fluent.Parameters; +import org.apache.commons.configuration2.ex.ConfigurationException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +public class S4JClientConf { + private final static Logger logger = LogManager.getLogger(S4JClientConf.class); + + public static final String CLIENT_CONF_PREFIX = "client"; + public static final String PRODUCER_CONF_PREFIX = "producer"; + public static final String CONSUMER_CONF_PREFIX = "consumer"; + public static final String JMS_CONF_PREFIX = "jms"; + + + // "Raw" map is what is read from the config properties file + // "Tgt" map is what is really needed in the Pulsar producer/consumer API + private Map clientConfMapRaw = new HashMap<>(); + private Map producerConfMapRaw = new HashMap<>(); + private Map consumerConfMapRaw = new HashMap<>(); + private Map jmsConfMapRaw = new HashMap<>(); + private Map miscConfMapRaw = new HashMap<>(); + + private final Map s4jConfMapTgt = new HashMap<>(); + private Map clientConfMapTgt = new HashMap<>(); + private Map producerConfMapTgt = new HashMap<>(); + + private Map consumerConfMapTgt = new HashMap<>(); + private Map jmsConfMapTgt = new HashMap<>(); + private Map miscConfMapTgt = new HashMap<>(); + + + + public S4JClientConf(String webSvcUrl, String pulsarSvcUrl, String s4jConfFileName) { + + ////////////////// + // Read related Pulsar client configuration settings from a file + readRawConfFromFile(s4jConfFileName); + + + ////////////////// + // Ignores the following Pulsar client/producer/consumer configurations since + // they're either not supported in the S4J API or the property must be specified + // as the NB CLI parameter or the NB yaml file parameter. + + // <<< https://pulsar.apache.org/docs/reference-configuration/#client >>> + // pulsar client config + // * webServiceUrl + // * brokerServiceUrl + clientConfMapRaw.put("brokerServiceUrl", pulsarSvcUrl); + clientConfMapRaw.put("webServiceUrl", webSvcUrl); + + + // <<< https://pulsar.apache.org/docs/client-libraries-java/#configure-producer >>> + // producer config + // * topicName + producerConfMapRaw.remove("topicName"); + + // <<< https://pulsar.apache.org/docs/client-libraries-java/#configure-consumer >>> + // consumer config + // * topicNames + // * topicsPattern + // * subscriptionName + // * subscriptionType + consumerConfMapRaw.remove("topicNames"); + consumerConfMapRaw.remove("topicPattern"); + consumerConfMapRaw.remove("subscriptionName"); + consumerConfMapRaw.remove("subscriptionType"); + consumerConfMapRaw.remove("subscriptionInitialPosition"); + consumerConfMapRaw.remove("regexSubscriptionMode"); + + + ////////////////// + // Convert the raw configuration map () to the required map () + clientConfMapTgt.putAll(S4JClientConfConverter.convertRawClientConf(clientConfMapRaw)); + producerConfMapTgt.putAll(S4JClientConfConverter.convertRawProducerConf(producerConfMapRaw)); + consumerConfMapTgt.putAll(S4JClientConfConverter.convertRawConsumerConf(consumerConfMapRaw)); + jmsConfMapTgt.putAll(S4JClientConfConverter.convertRawJmsConf(jmsConfMapRaw)); + miscConfMapTgt.putAll(S4JClientConfConverter.convertRawMiscConf(miscConfMapRaw)); + + s4jConfMapTgt.putAll(clientConfMapTgt); + s4jConfMapTgt.put("producerConfig", producerConfMapTgt); + s4jConfMapTgt.put("consumerConfig", consumerConfMapTgt); + s4jConfMapTgt.putAll(jmsConfMapTgt); + s4jConfMapTgt.putAll(miscConfMapTgt); + } + + public void readRawConfFromFile(String fileName) { + File file = new File(fileName); + + try { + String canonicalFilePath = file.getCanonicalPath(); + + Parameters params = new Parameters(); + + FileBasedConfigurationBuilder builder = + new FileBasedConfigurationBuilder(PropertiesConfiguration.class) + .configure(params.properties() + .setFileName(fileName)); + + Configuration config = builder.getConfiguration(); + + for (Iterator it = config.getKeys(); it.hasNext(); ) { + String confKey = it.next(); + String confVal = config.getProperty(confKey).toString(); + + if (!StringUtils.isBlank(confVal)) { + // Get client connection specific configuration settings, removing "client." prefix + if (StringUtils.startsWith(confKey, CLIENT_CONF_PREFIX)) { + clientConfMapRaw.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), confVal); + } + // Get producer specific configuration settings, removing "producer." prefix + else if (StringUtils.startsWith(confKey, PRODUCER_CONF_PREFIX)) { + producerConfMapRaw.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), confVal); + } + // Get consumer specific configuration settings, removing "consumer." prefix + else if (StringUtils.startsWith(confKey, CONSUMER_CONF_PREFIX)) { + consumerConfMapRaw.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), confVal); + } + // Get JMS specific configuration settings, keeping "jms." prefix + else if (StringUtils.startsWith(confKey, JMS_CONF_PREFIX)) { + jmsConfMapRaw.put(confKey, confVal); + } + // For all other configuration settings (not having any of the above prefixes), keep as is + else { + miscConfMapRaw.put(confKey, confVal); + } + } + } + } catch (IOException ioe) { + logger.error("Can't read the specified config properties file: " + fileName); + ioe.printStackTrace(); + } catch (ConfigurationException cex) { + logger.error("Error loading configuration items from the specified config properties file: " + fileName + ":" + cex.getMessage()); + cex.printStackTrace(); + } + } + + public Map getS4jConfObjMap() { return this.s4jConfMapTgt; } + public Map getS4jConfMapObj_client() { return this.clientConfMapTgt; } + public Map getS4jConfMapObj_producer() { return this.producerConfMapTgt; } + public Map getS4jConfMapObj_consumer() { return this.consumerConfMapTgt; } + public Map getS4jConfMapObj_jms() { return this.jmsConfMapTgt; } + public Map getS4jConfMapObj_misc() { return this.miscConfMapTgt; } + + private Map mergeConfigObjMaps( + Map origConfigObjMap, + Map extraConfigObjMap ) + { + Map newConfigObjMap = new HashMap<>(); + + // If there are the same settings in both "orig" and "extra" maps, + // the one in the "extra" map will take over + newConfigObjMap.putAll(origConfigObjMap); + newConfigObjMap.putAll(extraConfigObjMap); + + return newConfigObjMap; + } + public Map mergeExtraConsumerConfig( + Map extraConsumerConfigRaw) + { + if ( (extraConsumerConfigRaw == null) || (extraConsumerConfigRaw.isEmpty()) ) { + return getS4jConfObjMap(); + } + else { + Map origConsumerConfigObjMap = getS4jConfMapObj_consumer(); + Map extraConsumerConfigObjMap = + S4JClientConfConverter.convertRawConsumerConf(extraConsumerConfigRaw); + Map mergedConsumerConfigObjMap = + mergeConfigObjMaps(origConsumerConfigObjMap, extraConsumerConfigObjMap); + + Map mergedS4JConfObjMap = getS4jConfObjMap(); + mergedS4JConfObjMap.put("consumerConfig", mergedConsumerConfigObjMap); + + return mergedS4JConfObjMap; + } + } + + public String toString() { + return new ToStringBuilder(this). + append("effectiveS4jConfMap", s4jConfMapTgt). + toString(); + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JClientConfConverter.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JClientConfConverter.java new file mode 100644 index 000000000..9ba86bfb3 --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JClientConfConverter.java @@ -0,0 +1,429 @@ +package io.nosqlbench.adapter.s4j.util; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.datastax.oss.pulsar.jms.shaded.org.apache.pulsar.client.api.CompressionType; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.nosqlbench.adapter.s4j.exception.S4JAdapterInvalidParamException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This class is used to convert the configuration items in its raw + * format (as provided in the property file) to the format needed by + * the S4J driver + */ +public class S4JClientConfConverter { + + public static Map convertRawClientConf(Map pulsarClientConfMapRaw) { + Map s4jClientConfObjMap = new HashMap<>(); + s4jClientConfObjMap.putAll(pulsarClientConfMapRaw); + + /** + * No special handling for non-primitive types + */ + + return s4jClientConfObjMap; + } + + // <<< https://pulsar.apache.org/docs/client-libraries-java/#configure-producer >>> + private final static Map validStdProducerConfKeyTypeMap = Map.ofEntries( + Map.entry("topicName", "String"), + Map.entry("producerName","String"), + Map.entry("sendTimeoutMs","long"), + Map.entry("blockIfQueueFull","boolean"), + Map.entry("maxPendingMessages","int"), + Map.entry("maxPendingMessagesAcrossPartitions","int"), + Map.entry("messageRoutingMode","MessageRoutingMode"), + Map.entry("hashingScheme","HashingScheme"), + Map.entry("cryptoFailureAction","ProducerCryptoFailureAction"), + Map.entry("batchingMaxPublishDelayMicros","long"), + Map.entry("batchingMaxMessages","int"), + Map.entry("batchingEnabled","boolean"), + Map.entry("chunkingEnabled","boolean"), + Map.entry("compressionType","CompressionType"), + Map.entry("initialSubscriptionName","string") + ); + public static Map convertRawProducerConf(Map pulsarProducerConfMapRaw) { + Map s4jProducerConfObjMap = new HashMap<>(); + setConfObjMapForPrimitives(s4jProducerConfObjMap, pulsarProducerConfMapRaw, validStdProducerConfKeyTypeMap); + + /** + * Non-primitive type processing for Pulsar producer configuration items + */ + // "compressionType" has value type "CompressionType" + // - expecting the following values: 'LZ4', 'ZLIB', 'ZSTD', 'SNAPPY' + String confKeyName = "compressionType"; + String confVal = pulsarProducerConfMapRaw.get(confKeyName); + String expectedVal = "(LZ4|ZLIB|ZSTD|SNAPPY)"; + + if (StringUtils.isNotBlank(confVal)) { + if (StringUtils.equalsAnyIgnoreCase(confVal, "LZ4", "ZLIB", "ZSTD", "SNAPPY")) { + CompressionType compressionType = CompressionType.NONE; + + switch (StringUtils.upperCase(confVal)) { + case "LZ4": + compressionType = CompressionType.LZ4; + case "ZLIB": + compressionType = CompressionType.ZLIB; + case "ZSTD": + compressionType = CompressionType.ZSTD; + case "SNAPPY": + compressionType = CompressionType.SNAPPY; + } + + s4jProducerConfObjMap.put(confKeyName, compressionType); + } else { + throw new S4JAdapterInvalidParamException( + getInvalidConfValStr(confKeyName, confVal, "producer", expectedVal)); + } + } + + // TODO: Skip the following Pulsar configuration items for now because they're not really + // needed in the NB S4J testing at the moment. Add support for them when needed. + // * messageRoutingMode + // * hashingScheme + // * cryptoFailureAction + + return s4jProducerConfObjMap; + } + + // https://pulsar.apache.org/docs/client-libraries-java/#configure-consumer + private final static Map validStdConsumerConfKeyTypeMap = Map.ofEntries( + Map.entry("topicNames", "Set"), + Map.entry("topicsPattern","Pattern"), + Map.entry("subscriptionName","String"), + Map.entry("subscriptionType","SubscriptionType"), + Map.entry("receiverQueueSize","int"), + Map.entry("acknowledgementsGroupTimeMicros","long"), + Map.entry("negativeAckRedeliveryDelayMicros","long"), + Map.entry("maxTotalReceiverQueueSizeAcrossPartitions","int"), + Map.entry("consumerName","String"), + Map.entry("ackTimeoutMillis","long"), + Map.entry("tickDurationMillis","long"), + Map.entry("priorityLevel","int"), + Map.entry("cryptoFailureAction","ConsumerCryptoFailureAction"), + Map.entry("properties","SortedMap"), + Map.entry("readCompacted","boolean"), + Map.entry("subscriptionInitialPosition", "SubscriptionInitialPosition"), + Map.entry("patternAutoDiscoveryPeriod", "int"), + Map.entry("regexSubscriptionMode", "RegexSubscriptionMode"), + Map.entry("deadLetterPolicy", "DeadLetterPolicy"), + Map.entry("autoUpdatePartitions", "boolean"), + Map.entry("replicateSubscriptionState", "boolean"), + Map.entry("negativeAckRedeliveryBackoff", "RedeliveryBackoff"), + Map.entry("ackTimeoutRedeliveryBackoff", "RedeliveryBackoff"), + Map.entry("autoAckOldestChunkedMessageOnQueueFull", "boolean"), + Map.entry("maxPendingChunkedMessage", "int"), + Map.entry("expireTimeOfIncompleteChunkedMessageMillis", "long") + ); + public static Map convertRawConsumerConf(Map pulsarConsumerConfMapRaw) { + Map s4jConsumerConfObjMap = new HashMap<>(); + setConfObjMapForPrimitives(s4jConsumerConfObjMap, pulsarConsumerConfMapRaw, validStdConsumerConfKeyTypeMap); + + /** + * Non-primitive type processing for Pulsar consumer configuration items + */ + // The following non-primitive type configuration items are already excluded + // and don't need to be processed. + // * topicNames + // * topicPattern + // * subscriptionName + // * subscriptionType + // * subscriptionInitialPosition + // * regexSubscriptionMode + + // "properties" has value type "SortedMap" + // - expecting the value string has the format: a JSON string that includes a set of key/value pairs + String confKeyName = "properties"; + String confVal = pulsarConsumerConfMapRaw.get(confKeyName); + String expectedVal = "{\"property1\":\"value1\", \"property2\":\"value2\"}, ..."; + + ObjectMapper mapper = new ObjectMapper(); + + if (StringUtils.isNotBlank(confVal)) { + try { + Map consumerProperties = mapper.readValue(confVal, Map.class); + + // Empty map value is considered as no value + if (!consumerProperties.isEmpty()) { + s4jConsumerConfObjMap.put(confKeyName, consumerProperties); + } + + } catch (Exception e) { + throw new S4JAdapterInvalidParamException( + getInvalidConfValStr(confKeyName, confVal, "consumer", expectedVal)); + } + } + + // "deadLetterPolicy" + // - expecting the value is a JSON string has the format: + // {"maxRedeliverCount":"","deadLetterTopic":"","initialSubscriptionName":""} + confKeyName = "deadLetterPolicy"; + confVal = pulsarConsumerConfMapRaw.get(confKeyName); + expectedVal = "{" + + "\"maxRedeliverCount\":\"\"," + + "\"deadLetterTopic\":\"\"," + + "\"initialSubscriptionName\":\"\"}"; + + if (StringUtils.isNotBlank(confVal)) { + try { + Map dlqPolicyMap = mapper.readValue(confVal, Map.class); + + // Empty map value is considered as no value + if (!dlqPolicyMap.isEmpty()) { + boolean valid = true; + + // The JSON key must be one of "maxRedeliverCount", "deadLetterTopic", "initialSubscriptionName" + for (String key : dlqPolicyMap.keySet()) { + if (!StringUtils.equalsAnyIgnoreCase(key, + "maxRedeliverCount", "deadLetterTopic", "initialSubscriptionName")) { + valid = false; + break; + } + } + + // DLQ.maxRedeliverCount is mandatory + if (valid && !dlqPolicyMap.containsKey("maxRedeliverCount")) { + valid = false; + } + + String maxRedeliverCountStr = dlqPolicyMap.get("maxRedeliverCount"); + if (!NumberUtils.isCreatable(maxRedeliverCountStr)) { + valid = false; + } + + if (valid) { + // In S4J driver, DLQ setting is done via a Map + // <<< https://docs.datastax.com/en/fast-pulsar-jms/docs/1.1/pulsar-jms-implementation.html#dead-letter-policy >>> + + s4jConsumerConfObjMap.put(confKeyName, dlqPolicyMap); + } else { + throw new S4JAdapterInvalidParamException( + getInvalidConfValStr(confKeyName, confVal, "consumer", expectedVal)); + } + } + } catch (Exception e) { + throw new S4JAdapterInvalidParamException( + getInvalidConfValStr(confKeyName, confVal, "consumer", expectedVal)); + } + } + + // "negativeAckRedeliveryBackoff" or "ackTimeoutRedeliveryBackoff" + // - expecting the value is a JSON string has the format: + // {"minDelayMs":"", "maxDelayMs":"", "multiplier":""} + String[] redeliveryBackoffConfigSet = {"negativeAckRedeliveryBackoff", "ackTimeoutRedeliveryBackoff"}; + expectedVal = "{" + + "\"minDelayMs\":\"\"," + + "\"maxDelayMs\":\"\"," + + "\"multiplier\":\"\"}"; + + for (String confKey : redeliveryBackoffConfigSet) { + confVal = pulsarConsumerConfMapRaw.get(confKey); + + if (StringUtils.isNotBlank(confVal)) { + try { + Map redliveryBackoffMap = mapper.readValue(confVal, Map.class); + + // Empty map value is considered as no value + if (! redliveryBackoffMap.isEmpty()) { + boolean valid = true; + + // The JSON key must be one of "maxRedeliverCount", "deadLetterTopic", "initialSubscriptionName" + for (String key : redliveryBackoffMap.keySet()) { + if (!StringUtils.equalsAnyIgnoreCase(key, + "minDelayMs", "maxDelayMs", "multiplier")) { + valid = false; + break; + } + } + + String minDelayMsStr = redliveryBackoffMap.get("minDelayMs"); + String maxDelayMsStr = redliveryBackoffMap.get("maxDelayMs"); + String multiplierStr = redliveryBackoffMap.get("multiplier"); + + if ((StringUtils.isNotBlank(minDelayMsStr) && !NumberUtils.isCreatable(minDelayMsStr)) || + (StringUtils.isNotBlank(maxDelayMsStr) && !NumberUtils.isCreatable(maxDelayMsStr)) || + (StringUtils.isNotBlank(multiplierStr) && !NumberUtils.isCreatable(multiplierStr))) { + valid = false; + } + + if (valid) { + // In S4J driver, AckTimeOut and Negative TimeOut is done via a Map + // <<< https://docs.datastax.com/en/fast-pulsar-jms/docs/1.1/pulsar-jms-implementation.html#ack-timeout >>> + // <<< https://docs.datastax.com/en/fast-pulsar-jms/docs/1.1/pulsar-jms-implementation.html#negative-ack >>> + + s4jConsumerConfObjMap.put(confKey, redliveryBackoffMap); + + } else { + throw new S4JAdapterInvalidParamException( + getInvalidConfValStr(confKey, confVal, "consumer", expectedVal)); + } + } + + } catch (Exception e) { + throw new S4JAdapterInvalidParamException( + getInvalidConfValStr(confKey, confVal, "consumer", expectedVal)); + } + } + } + + + // TODO: Skip the following Pulsar configuration items for now because they're not really + // needed in the NB S4J testing right now. Add the support for them when needed. + // * cryptoFailureAction + + return s4jConsumerConfObjMap; + } + + // https://docs.datastax.com/en/fast-pulsar-jms/docs/1.1/pulsar-jms-reference.html#_configuration_options + private final static Map validS4jJmsConfKeyTypeMap = Map.ofEntries( + Map.entry("jms.acknowledgeRejectedMessages", "boolean"), + Map.entry("jms.clientId","String"), + Map.entry("jms.emulateTransactions","boolean"), + Map.entry("jms.enableClientSideEmulation","boolean"), + Map.entry("jms.forceDeleteTemporaryDestinations","boolean"), + Map.entry("jms.precreateQueueSubscription","boolean"), + Map.entry("jms.queueSubscriptionName","String"), + Map.entry("jms.systemNamespace","String"), + Map.entry("jms.topicSharedSubscriptionType","String"), + Map.entry("jms.useCredentialsFromCreateConnection","boolean"), + Map.entry("jms.useExclusiveSubscriptionsForSimpleConsumers","long"), + Map.entry("jms.usePulsarAdmin","boolean"), + Map.entry("jms.useServerSideFiltering","boolean"), + Map.entry("jms.waitForServerStartupTimeout","int"), + Map.entry("jms.transactionsStickyPartitions", "boolean") + ); + public static Map convertRawJmsConf(Map s4jJmsConfMapRaw) { + Map s4jJmsConfObjMap = new HashMap<>(); + setConfObjMapForPrimitives(s4jJmsConfObjMap, s4jJmsConfMapRaw, validS4jJmsConfKeyTypeMap); + + /** + * Non-primitive type processing for Pulsar client configuration items + */ + // None + + return s4jJmsConfObjMap; + } + + // https://docs.datastax.com/en/fast-pulsar-jms/docs/1.1/pulsar-jms-reference.html#_configuration_options + private final static Map validS4jMiscConfKeyTypeMap = Map.ofEntries( + Map.entry("brokerServiceUrl","String"), + Map.entry("webServiceUrl","String"), + Map.entry("ackTimeout", "long"), + Map.entry("ackTimeoutMillis","long"), + Map.entry("enableTransaction","boolean"), + Map.entry("consumerConfig","Map"), + Map.entry("producerConfig","Map") + ); + public static Map convertRawMiscConf(Map s4jMiscConfMapRaw) { + Map s4jMiscConfObjMap = new HashMap<>(); + setConfObjMapForPrimitives(s4jMiscConfObjMap, s4jMiscConfMapRaw, validS4jMiscConfKeyTypeMap); + + /** + * Non-primitive type processing for Pulsar client configuration items + */ + // Only the following 2 non-primitive type settings will be set explicitly + // * producerConfig + // * consumerConfig + + return s4jMiscConfObjMap; + } + + + // Utility function + // - get configuration key names by the value type + private static List getConfKeyNameByValueType(Map confKeyTypeMap, String tgtValType) { + ArrayList confKeyNames = new ArrayList<>(); + + for (Map.Entry entry: confKeyTypeMap.entrySet()) { + if (StringUtils.equalsIgnoreCase(entry.getValue().toString(), tgtValType)) { + confKeyNames.add(entry.getKey().toString()); + } + } + + return confKeyNames; + } + + // Conversion from Map to Map for configuration items with primitive + // value types + private static void setConfObjMapForPrimitives( + Map tgtConfObjMap, + Map srcConfMapRaw, + Map validConfKeyTypeMap) + { + List confKeyList = new ArrayList<>(); + + // All configuration items with "String" as the value type + confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "String"); + for (String confKey : confKeyList) { + if (srcConfMapRaw.containsKey(confKey)) { + String confVal = srcConfMapRaw.get(confKey); + if (StringUtils.isNotBlank(confVal)) { + tgtConfObjMap.put(confKey, confVal); + } + } + } + + // All configuration items with "long" as the value type + confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "long"); + for (String confKey : confKeyList) { + if (srcConfMapRaw.containsKey(confKey)) { + String confVal = srcConfMapRaw.get(confKey); + if (StringUtils.isNotBlank(confVal)) { + tgtConfObjMap.put(confKey, Long.valueOf(confVal)); + } + } + } + + // All configuration items with "int" as the value type + confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "int"); + for (String confKey : confKeyList) { + if (srcConfMapRaw.containsKey(confKey)) { + String confVal = srcConfMapRaw.get(confKey); + if (StringUtils.isNotBlank(confVal)) { + tgtConfObjMap.put(confKey, Integer.valueOf(confVal)); + } + } + } + + // All configuration items with "boolean" as the value type + confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "boolean"); + for (String confKey : confKeyList) { + if (srcConfMapRaw.containsKey(confKey)) { + String confVal = srcConfMapRaw.get(confKey); + if (StringUtils.isNotBlank(confVal)) { + tgtConfObjMap.put(confKey, Boolean.valueOf(confVal)); + } + } + } + + // TODO: So far the above primitive types should be good enough. + // Add support for other types when needed + } + + private static String getInvalidConfValStr(String confKey, String confVal, String configCategory, String expectedVal) { + return "Incorrect value \"" + confVal + "\" for Pulsar " + configCategory + + " configuration item of \"" + confKey + "\". Expecting the following value (format): " + expectedVal; + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JCompletionListener.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JCompletionListener.java new file mode 100644 index 000000000..fcc185869 --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JCompletionListener.java @@ -0,0 +1,86 @@ +package io.nosqlbench.adapter.s4j.util; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import io.nosqlbench.adapter.s4j.S4JSpace; +import io.nosqlbench.adapter.s4j.dispensers.S4JBaseOpDispenser; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.jms.CompletionListener; +import javax.jms.JMSException; +import javax.jms.Message; + +/** + * Used for async message production + */ +public class S4JCompletionListener implements CompletionListener { + + private final static Logger logger = LogManager.getLogger(S4JCompletionListener.class); + + private final S4JSpace s4JSpace; + private final S4JBaseOpDispenser s4jBaseOpDispenser; + + public S4JCompletionListener(S4JSpace s4JSpace, S4JBaseOpDispenser s4jBaseOpDispenser) { + assert (s4JSpace != null); + assert (s4jBaseOpDispenser != null); + this.s4JSpace = s4JSpace; + this.s4jBaseOpDispenser = s4jBaseOpDispenser; + } + + @Override + public void onCompletion(Message message) { + try { + if (logger.isTraceEnabled()) { + // for testing purpose + String myMsgSeq = message.getStringProperty(S4JAdapterUtil.NB_MSG_SEQ_PROP); + logger.trace("onCompletion::Async message send successful - message ID {} ({}) " + , message.getJMSMessageID(), myMsgSeq); + } + + if (s4JSpace.isTrackingMsgRecvCnt() ) { + long totalResponseCnt = s4JSpace.incTotalOpResponseCnt(); + if (logger.isTraceEnabled()) { + logger.trace("... async op response received so far: {}", totalResponseCnt); + } + } + } + catch (JMSException e) { + S4JAdapterUtil.processMsgErrorHandling( + e, + s4JSpace.isStrictMsgErrorHandling(), + "Unexpected errors when async sending a JMS message."); + } + } + + @Override + public void onException(Message message, Exception e) { + try { + if (logger.isDebugEnabled()) { + // for testing purpose + String myMsgSeq = message.getStringProperty(S4JAdapterUtil.NB_MSG_SEQ_PROP); + + logger.debug("onException::Async message send failed - message ID {} ({}) " + , message.getJMSMessageID(), myMsgSeq); + } + } + catch (JMSException jmsException) { + logger.warn("onException::Unexpected error: " + jmsException.getMessage()); + } + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JJMSContextWrapper.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JJMSContextWrapper.java new file mode 100644 index 000000000..f22b6d78b --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JJMSContextWrapper.java @@ -0,0 +1,52 @@ +package io.nosqlbench.adapter.s4j.util; + +import org.apache.commons.lang3.builder.ToStringBuilder; + +import javax.jms.JMSContext; +import javax.jms.Session; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +public class S4JJMSContextWrapper { + private final String jmsContextIdentifer; + private final JMSContext jmsContext; + private final int jmsSessionMode; + + public S4JJMSContextWrapper(String identifer, JMSContext jmsContext) { + this.jmsContextIdentifer = identifer; + this.jmsContext = jmsContext; + this.jmsSessionMode = jmsContext.getSessionMode(); + } + + public int getJmsSessionMode() { return jmsSessionMode; } + public boolean isTransactedMode() { return Session.SESSION_TRANSACTED == this.getJmsSessionMode(); } + public String getJmsContextIdentifer() { return jmsContextIdentifer; } + public JMSContext getJmsContext() { return jmsContext; } + + public void close() { + if (jmsContext != null) { + jmsContext.close(); + } + } + + public String toString() { + return new ToStringBuilder(this). + append("jmsContextIdentifer", jmsContextIdentifer). + append("jmsContext", jmsContext.toString()). + toString(); + } +} diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JMessageListener.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JMessageListener.java new file mode 100644 index 000000000..e7e9c0a5e --- /dev/null +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/util/S4JMessageListener.java @@ -0,0 +1,97 @@ +package io.nosqlbench.adapter.s4j.util; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.codahale.metrics.Histogram; +import io.nosqlbench.adapter.s4j.S4JSpace; +import io.nosqlbench.adapter.s4j.dispensers.S4JBaseOpDispenser; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.jms.JMSContext; +import javax.jms.JMSException; +import javax.jms.Message; +import javax.jms.MessageListener; + +/** + * Used for async message consumption + */ +public class S4JMessageListener implements MessageListener { + + private final static Logger logger = LogManager.getLogger(S4JMessageListener.class); + + private final float msgAckRatio; + private final int slowAckInSec; + private final JMSContext jmsContext; + private final S4JSpace s4jSpace; + private final S4JBaseOpDispenser s4jBaseOpDispenser; + + public S4JMessageListener( + JMSContext jmsContext, + S4JSpace s4jSpace, + S4JBaseOpDispenser s4jBaseOpDispenser, + float msgAckRatio, + int slowAckInSec) + { + assert (jmsContext != null); + assert (s4jSpace != null); + assert (s4jBaseOpDispenser != null); + + this.jmsContext = jmsContext; + this.s4jSpace = s4jSpace; + this.s4jBaseOpDispenser = s4jBaseOpDispenser; + this.msgAckRatio = msgAckRatio; + this.slowAckInSec = slowAckInSec; + } + + @Override + public void onMessage(Message message) { + try { + if (message != null) { + s4jSpace.processMsgAck(jmsContext, message, msgAckRatio, slowAckInSec); + + int msgSize = message.getIntProperty(S4JAdapterUtil.NB_MSG_SIZE_PROP); + S4JAdapterMetrics s4JAdapterMetrics = s4jBaseOpDispenser.getS4jAdapterMetrics(); + Histogram messageSizeHistogram = s4JAdapterMetrics.getMessagesizeHistogram(); + messageSizeHistogram.update(msgSize); + + if (logger.isTraceEnabled()) { + // for testing purpose + String myMsgSeq = message.getStringProperty(S4JAdapterUtil.NB_MSG_SEQ_PROP); + logger.trace("onMessage::Async message receive successful - message ID {} ({}) " + , message.getJMSMessageID(), myMsgSeq); + } + + if (s4jSpace.isTrackingMsgRecvCnt()) { + s4jSpace.incTotalOpResponseCnt(); + } + } + else { + if (s4jSpace.isTrackingMsgRecvCnt()) { + s4jSpace.incTotalNullMsgRecvdCnt(); + } + } + } + catch (JMSException e) { + S4JAdapterUtil.processMsgErrorHandling( + e, + s4jSpace.isStrictMsgErrorHandling(), + "Unexpected errors when async receiving a JMS message."); + } + } +} diff --git a/nb5/pom.xml b/nb5/pom.xml index f801d40cd..340360d07 100644 --- a/nb5/pom.xml +++ b/nb5/pom.xml @@ -94,6 +94,12 @@ 4.17.31-SNAPSHOT + + io.nosqlbench + adapter-s4j + 4.17.31-SNAPSHOT + + diff --git a/pom.xml b/pom.xml index 7eac71c2c..098185117 100644 --- a/pom.xml +++ b/pom.xml @@ -62,6 +62,7 @@ adapter-dynamodb adapter-mongodb adapter-pulsar + adapter-s4j From dc74283952694a2bb9b63e4b9a69fbedea387c0c Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 6 Dec 2022 20:34:02 -0600 Subject: [PATCH 28/45] nosqlbench-824 Add function to provide clustered values with monotonic stepping --- .../to_long/TriangularStepFunction.java | 96 ++++++++++++ .../to_long/TriangularStepFunctionTest.java | 146 ++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java create mode 100644 virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunctionTest.java diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java new file mode 100644 index 000000000..6368fe169 --- /dev/null +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.virtdata.library.basics.shared.from_long.to_long; + +import io.nosqlbench.virtdata.api.annotations.Example; +import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; + +import java.util.function.LongUnaryOperator; + +/** + *

Compute a value which increases monotonically with respect to the cycle value. + * All values for f(X+(m>=0)) will be equal or greater than f(X). In effect, this + * means that with a sequence of monotonic inputs, the results will be monotonic as + * well as clustered. The values will approximate input/average, but will vary in frequency + * around a simple binomial distribution.

+ * + *

The practical effect of this is to be able to compute a sequence of values + * over inputs which can act as foreign keys, but which are effectively ordered.

+ * + *

Call for Ideas

+ *

Due to the complexity of generalizing this as a pure function over other distributions, + * this is the only function of this type for now. If you are interested in this problem + * domain and have some suggestions for how to extend it to other distributions, please + * join the project or let us know.

+ */ +@ThreadSafeMapper +public class TriangularStepFunction implements LongUnaryOperator { + + private final Hash hasher = new Hash(); + private final long median; + private final LongUnaryOperator sizer; + + private final long variance; + + + @Example({"TriangularStepFunction(100,20)","Create a sequence of values where the average and median is 100, but the range of values is between 90 and 120."}) + @Example({"TriangularStepFunction(80,10)","Create a sequence of values where the average and median is 80, but the range of values is between 70 and 90."}) + TriangularStepFunction(long average, long variance) { + if (variance < 0 || variance > average) { + throw new RuntimeException( + "The median must be non-negative, and the variance must be less than the median. " + + "You provided median=" + average + ", variance=" + variance + "." + ); + } + this.median = average; + this.variance = variance; + this.sizer = new HashRange(average-variance,average+variance); + } + + TriangularStepFunction(long average) { + this(average, average/2); +// if (maxOffset>=avgsize) { +// throw new RuntimeException("max offset " + maxOffset + " has to be less than avg size " + avgsize); +// } + } + + @Override + public long applyAsLong(long operand) { + // window number + long count = operand / median; + // offset within window + long offset = operand % median; + // base of window + long base = operand - offset; + // variate up to window size + long variance = sizer.applyAsLong(base); + // variate offset from start of window + long slice = base + variance; + // select current or next window + long result = ((slice)>operand) ? count : count + 1; + return result; + } + + public long inlined(long operand) { + return (operand < operand - operand % median + sizer.applyAsLong(operand - operand % median)) ? operand / median : operand / median + 1; + } + + @Override + public String toString() { + return this.getClass().getSimpleName()+"{median="+median+",variance="+variance+"}"; + } +} diff --git a/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunctionTest.java b/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunctionTest.java new file mode 100644 index 000000000..4df0c7b90 --- /dev/null +++ b/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunctionTest.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.virtdata.library.basics.shared.from_long.to_long; + +import org.junit.jupiter.api.Test; + +import java.security.InvalidParameterException; +import java.util.Arrays; +import java.util.LongSummaryStatistics; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TriangularStepFunctionTest { + + private static final int LABEL=0; + private static final int FREQUENCY=1; + + @Test + public void testExample1() { + TriangularStepFunction e1 = new TriangularStepFunction(100, 20); + int[] runLengths = this.rleStatsFor(e1, 0, 10000); + System.out.println(Arrays.toString(runLengths)); + assertThat(IntStream.of(runLengths).min().orElseThrow()).isEqualTo(80L); + assertThat(IntStream.of(runLengths).max().orElseThrow()).isEqualTo(120L); + } + + @Test + public void testExample2() { + TriangularStepFunction e1 = new TriangularStepFunction(80, 10); + int[] runLengths = this.rleStatsFor(e1, 0, 10000); + System.out.println(Arrays.toString(runLengths)); + assertThat(IntStream.of(runLengths).min().orElseThrow()).isEqualTo(70L); + assertThat(IntStream.of(runLengths).max().orElseThrow()).isEqualTo(90L); + } + + @Test + public void testStepSlice() { + int avgsize=10; + TriangularStepFunction f = new TriangularStepFunction(avgsize); + int[] ary = new int[avgsize*2]; + long current=0L; + int count=0; + for (int i = 0; i < 10000; i++) { + long result = f.applyAsLong(i); + if (result==current) { + count++; + } else { + ary[count]++; + current=result; + count=0; + } + } + + for (int i = 0; i < ary.length; i++) { + System.out.println("bucket " + i + ", count " + ary[i]); + } + } + + + @Test + public void testIncrementalVariance() { + TriangularStepFunction f = new TriangularStepFunction(100, 0); + assertThat(f.applyAsLong(0L)).isEqualTo(0L); + assertThat(f.applyAsLong(1L)).isEqualTo(0L); + assertThat(f.applyAsLong(99L)).isEqualTo(0L); + assertThat(f.applyAsLong(100L)).isEqualTo(1L); + } + + @Test + public void testVariance() { + long first=0; + TriangularStepFunction f = new TriangularStepFunction(100,1); + var rlestats = rleStatsFor(f, 0, 100000); + LongSummaryStatistics stats99to101 = statsForRle((int) f.applyAsLong(first),rlestats); + assertThat(stats99to101.getMin()).isEqualTo(99L); + assertThat(stats99to101.getMax()).isEqualTo(101L); + + int[][] histo = histoFor(rlestats); + LongSummaryStatistics histoStats = new LongSummaryStatistics(); + for (int[] ints : histo) { + histoStats.accept(ints[LABEL]); + } + assertThat(histoStats.getAverage()).isEqualTo(100); + } + + private int[] rleStatsFor(TriangularStepFunction f, long firstTrialIncl, long lastTrialExcl) { + long firstBucket = f.applyAsLong(firstTrialIncl); + long lastBucket = f.applyAsLong(lastTrialExcl); + if (firstBucket>Integer.MAX_VALUE||lastBucket>Integer.MAX_VALUE) { + throw new InvalidParameterException("can't fit result data into range of ints from long [" + firstBucket + ","+lastBucket+"]"); + } + int base = (int) firstBucket; + int[] counts = new int[(((int) lastBucket-(int)firstBucket))+1]; + for (long trial=firstTrialIncl; trial < lastTrialExcl; trial++) { + long result = f.applyAsLong(trial); + counts[(int)(result-base)]++; + } + // remove last partial, as only the front initial partial is compensated + counts= Arrays.copyOfRange(counts,0,counts.length-1); + return counts; + } + + private int[][] histoFor(int[] counts) { + var minval = IntStream.of(counts).min().orElseThrow(); + var maxval = IntStream.of(counts).max().orElseThrow(); + + int[][] histo = new int[(maxval-minval)+1][2]; + for (int i = 0; i <= histo[LABEL].length; i++) { + histo[i][LABEL]=i+minval; + } + + for (int count : counts) { + System.out.println(count); + histo[count-minval][FREQUENCY]++; + } + return histo; + } + + private LongSummaryStatistics statsForRle(int base, int[] counts) { + LongSummaryStatistics stats = new LongSummaryStatistics(); + for (int element = 0; element < counts.length; element++) { + int count = counts[element]; + if (count==0) { + continue; + } + stats.accept(count); + } + return stats; + + } +} From 97cf9a32cb9429909c9015a61a449679693a284e Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 6 Dec 2022 20:34:31 -0600 Subject: [PATCH 29/45] add ad-hoc doc to show publishing flow --- devdocs/sketches/docs-publishing-flow.svg | 566 ++++++++++++++++++++++ 1 file changed, 566 insertions(+) create mode 100644 devdocs/sketches/docs-publishing-flow.svg diff --git a/devdocs/sketches/docs-publishing-flow.svg b/devdocs/sketches/docs-publishing-flow.svg new file mode 100644 index 000000000..a16d11f52 --- /dev/null +++ b/devdocs/sketches/docs-publishing-flow.svg @@ -0,0 +1,566 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + NB + + + + RELEASEDNB-DOCS + + + + NB-BUILD-DOCS + + + + non-releasebuild + + + + releasebuild + + + + + + + + + + + + + + + edit embededdocs and code + edit docscontent + publish docscontent to maindocs site + publish embeddeddocs content to maindocs site + publish embeddeddocs content to previewdocs site + + + From 3677a60ec846e8a4d43726777d84f7a11764dc1f Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 7 Dec 2022 13:36:24 -0600 Subject: [PATCH 30/45] nosqlbench-830 limit spin-looping with cqld4 retryreplace option. --- .../cqld4/Cqld4CqlReboundStatement.java | 4 +- .../ExceededRetryReplaceException.java | 45 +++++++++++++++++++ .../opdispensers/Cqld4BaseOpDispenser.java | 7 +++ .../Cqld4PreparedStmtDispenser.java | 1 + .../opdispensers/Cqld4RawStmtDispenser.java | 3 +- .../Cqld4SimpleCqlStmtDispenser.java | 3 +- .../cqld4/optypes/Cqld4CqlBatchStatement.java | 4 +- .../adapter/cqld4/optypes/Cqld4CqlOp.java | 21 ++++++++- .../optypes/Cqld4CqlPreparedStatement.java | 4 +- .../optypes/Cqld4CqlSimpleStatement.java | 4 +- adapter-cqld4/src/main/resources/cqld4.md | 5 +++ 11 files changed, 89 insertions(+), 12 deletions(-) create mode 100644 adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/exceptions/ExceededRetryReplaceException.java diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4CqlReboundStatement.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4CqlReboundStatement.java index 0ac61009c..e2fb9e1d9 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4CqlReboundStatement.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4CqlReboundStatement.java @@ -24,8 +24,8 @@ import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlOp; public class Cqld4CqlReboundStatement extends Cqld4CqlOp { private final BoundStatement stmt; - public Cqld4CqlReboundStatement(CqlSession session, int maxpages, boolean retryreplace, BoundStatement rebound, RSProcessors processors) { - super(session,maxpages,retryreplace,processors); + public Cqld4CqlReboundStatement(CqlSession session, int maxpages, boolean retryreplace, int maxlwtretries, int lwtRetryCount, BoundStatement rebound, RSProcessors processors) { + super(session,maxpages,retryreplace,maxlwtretries,lwtRetryCount, processors); this.stmt = rebound; } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/exceptions/ExceededRetryReplaceException.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/exceptions/ExceededRetryReplaceException.java new file mode 100644 index 000000000..3d7cb3b4b --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/exceptions/ExceededRetryReplaceException.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.cqld4.exceptions; + + +import com.datastax.oss.driver.api.core.cql.ResultSet; + +/** + * This is a synthetic error generated by the cql driver in NoSQLBench when + * the retryreplace option is used but the number of LWT round-trips from the driver + * is excessive. The number of LWT round trips allowed is controlled by the + * maxlwtretries op field. + */ +public class ExceededRetryReplaceException extends CqlGenericCycleException { + + private final ResultSet resultSet; + private final String queryString; + private final int retries; + + public ExceededRetryReplaceException(ResultSet resultSet, String queryString, int retries) { + super("After " + retries + " retires using the retryreplace option, Operation was not applied:" + queryString); + this.retries = retries; + this.resultSet = resultSet; + this.queryString = queryString; + } + + public ResultSet getResultSet() { + return resultSet; + } + public String getQueryString() { return queryString; } +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java index 34200479e..540b34dc1 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java @@ -45,12 +45,14 @@ public abstract class Cqld4BaseOpDispenser extends BaseOpDispenser sessionFunc; private final boolean isRetryReplace; + private final int maxLwtRetries; public Cqld4BaseOpDispenser(DriverAdapter adapter, LongFunction sessionFunc, ParsedOp op) { super(adapter, op); this.sessionFunc = sessionFunc; this.maxpages = op.getStaticConfigOr("maxpages", 1); this.isRetryReplace = op.getStaticConfigOr("retryreplace", false); + this.maxLwtRetries = op.getStaticConfigOr("maxlwtretries", 1); } public int getMaxPages() { @@ -61,6 +63,11 @@ public abstract class Cqld4BaseOpDispenser extends BaseOpDispenser getSessionFunc() { return sessionFunc; } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java index 497b67b99..83503c9c8 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java @@ -89,6 +89,7 @@ public class Cqld4PreparedStmtDispenser extends Cqld4BaseOpDispenser { boundStatement, getMaxPages(), isRetryReplace(), + getMaxLwtRetries(), processors ); } catch (Exception exception) { diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4RawStmtDispenser.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4RawStmtDispenser.java index fe986415c..16b999e60 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4RawStmtDispenser.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4RawStmtDispenser.java @@ -49,7 +49,8 @@ public class Cqld4RawStmtDispenser extends Cqld4BaseOpDispenser { getSessionFunc().apply(value), (SimpleStatement) stmtFunc.apply(value), getMaxPages(), - isRetryReplace() + isRetryReplace(), + getMaxLwtRetries() ); } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4SimpleCqlStmtDispenser.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4SimpleCqlStmtDispenser.java index a3e85cac5..f58ce6dc3 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4SimpleCqlStmtDispenser.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4SimpleCqlStmtDispenser.java @@ -46,7 +46,8 @@ public class Cqld4SimpleCqlStmtDispenser extends Cqld4BaseOpDispenser { getSessionFunc().apply(value), (SimpleStatement) stmtFunc.apply(value), getMaxPages(), - isRetryReplace() + isRetryReplace(), + getMaxLwtRetries() ); } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlBatchStatement.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlBatchStatement.java index bf8a64c3b..6828a70ec 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlBatchStatement.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlBatchStatement.java @@ -24,8 +24,8 @@ public class Cqld4CqlBatchStatement extends Cqld4CqlOp { private final BatchStatement stmt; - public Cqld4CqlBatchStatement(CqlSession session, BatchStatement stmt, int maxpages, boolean retryreplace) { - super(session,maxpages,retryreplace,new RSProcessors()); + public Cqld4CqlBatchStatement(CqlSession session, BatchStatement stmt, int maxpages, int maxlwtretries, boolean retryreplace) { + super(session,maxpages,retryreplace,maxlwtretries,new RSProcessors()); this.stmt = stmt; } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java index e72c835c3..ec9a147b6 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java @@ -23,6 +23,7 @@ import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.core.cql.Statement; import io.nosqlbench.adapter.cqld4.*; import io.nosqlbench.adapter.cqld4.exceptions.ChangeUnappliedCycleException; +import io.nosqlbench.adapter.cqld4.exceptions.ExceededRetryReplaceException; import io.nosqlbench.adapter.cqld4.exceptions.UndefinedResultSetException; import io.nosqlbench.adapter.cqld4.exceptions.UnexpectedPagingException; import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.*; @@ -46,15 +47,27 @@ public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, private final CqlSession session; private final int maxpages; private final boolean retryreplace; + private final int maxLwtRetries; + private int retryReplaceCount =0; private ResultSet rs; private Cqld4CqlOp nextOp; private final RSProcessors processors; - public Cqld4CqlOp(CqlSession session, int maxpages, boolean retryreplace, RSProcessors processors) { + public Cqld4CqlOp(CqlSession session, int maxpages, boolean retryreplace, int maxLwtRetries, RSProcessors processors) { this.session = session; this.maxpages = maxpages; this.retryreplace = retryreplace; + this.maxLwtRetries =maxLwtRetries; + this.processors = processors; + } + + protected Cqld4CqlOp(CqlSession session, int maxpages, boolean retryreplace, int maxLwtRetries, int retryRplaceCount, RSProcessors processors) { + this.session = session; + this.maxpages = maxpages; + this.retryreplace = retryreplace; + this.maxLwtRetries =maxLwtRetries; + this.retryReplaceCount=retryRplaceCount; this.processors = processors; } @@ -69,6 +82,10 @@ public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, if (!retryreplace) { throw new ChangeUnappliedCycleException(rs, getQueryString()); } else { + retryReplaceCount++; + if (retryReplaceCount >maxLwtRetries) { + throw new ExceededRetryReplaceException(rs,getQueryString(), retryReplaceCount); + } Row one = rs.one(); processors.buffer(one); totalRows++; @@ -119,7 +136,7 @@ public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, private Cqld4CqlOp rebindLwt(Statement stmt, Row row) { BoundStatement rebound = LWTRebinder.rebindUnappliedStatement(stmt, row); - return new Cqld4CqlReboundStatement(session, maxpages, retryreplace, rebound, processors); + return new Cqld4CqlReboundStatement(session, maxpages, retryreplace, maxLwtRetries, retryReplaceCount, rebound, processors); } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java index 4c8720196..938475f7d 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java @@ -24,8 +24,8 @@ public class Cqld4CqlPreparedStatement extends Cqld4CqlOp { private final BoundStatement stmt; - public Cqld4CqlPreparedStatement(CqlSession session, BoundStatement stmt, int maxpages, boolean retryreplace, RSProcessors processors) { - super(session,maxpages,retryreplace,processors); + public Cqld4CqlPreparedStatement(CqlSession session, BoundStatement stmt, int maxpages, boolean retryreplace, int maxLwtRetries, RSProcessors processors) { + super(session,maxpages,retryreplace,maxLwtRetries,processors); this.stmt = stmt; } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java index 19f1a6936..3fb6a9578 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java @@ -23,8 +23,8 @@ import io.nosqlbench.adapter.cqld4.RSProcessors; public class Cqld4CqlSimpleStatement extends Cqld4CqlOp { private final SimpleStatement stmt; - public Cqld4CqlSimpleStatement(CqlSession session, SimpleStatement stmt, int maxpages, boolean retryreplace) { - super(session, maxpages,retryreplace, new RSProcessors()); + public Cqld4CqlSimpleStatement(CqlSession session, SimpleStatement stmt, int maxpages, boolean retryreplace, int maxlwtretries) { + super(session, maxpages,retryreplace, maxlwtretries, new RSProcessors()); this.stmt = stmt; } diff --git a/adapter-cqld4/src/main/resources/cqld4.md b/adapter-cqld4/src/main/resources/cqld4.md index cb83996ce..473969a14 100644 --- a/adapter-cqld4/src/main/resources/cqld4.md +++ b/adapter-cqld4/src/main/resources/cqld4.md @@ -198,6 +198,11 @@ params: # match the preconditions) in order to test LWT performance. retryreplace: true + # Set the number of retries allowed by the retryreplace option. This is set + # to 1 conservatively, as with the maxpages setting. This means that you will + # see an error if the first LWT retry after an unapplied change was not successful. + maxlwtretries: 1 + ## The following options are meant for advanced testing scenarios only, ## and are not generally meant to be used in typical application-level, ## data mode, performance or scale testing. These expose properties From d90731ec7abef823966fdc431e089a883e8b4bd2 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 7 Dec 2022 13:41:20 -0600 Subject: [PATCH 31/45] typo fix --- .../adapter/cqld4/exceptions/ExceededRetryReplaceException.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/exceptions/ExceededRetryReplaceException.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/exceptions/ExceededRetryReplaceException.java index 3d7cb3b4b..a5e5f1f46 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/exceptions/ExceededRetryReplaceException.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/exceptions/ExceededRetryReplaceException.java @@ -32,7 +32,7 @@ public class ExceededRetryReplaceException extends CqlGenericCycleException { private final int retries; public ExceededRetryReplaceException(ResultSet resultSet, String queryString, int retries) { - super("After " + retries + " retires using the retryreplace option, Operation was not applied:" + queryString); + super("After " + retries + " retries using the retryreplace option, Operation was not applied:" + queryString); this.retries = retries; this.resultSet = resultSet; this.queryString = queryString; From 7737ffc0f98c1643f058f7a8a7aa8567d0fd023b Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 7 Dec 2022 14:30:04 -0600 Subject: [PATCH 32/45] nosqlbench-828 nosqlbench-824 --- .../to_long/TriangularStepFunction.java | 9 +------ .../to_long/TriangularStepFunctionTest.java | 25 ------------------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java index 6368fe169..266ab3732 100644 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java @@ -47,7 +47,7 @@ public class TriangularStepFunction implements LongUnaryOperator { private final long variance; - @Example({"TriangularStepFunction(100,20)","Create a sequence of values where the average and median is 100, but the range of values is between 90 and 120."}) + @Example({"TriangularStepFunction(100,20)","Create a sequence of values where the average and median is 100, but the range of values is between 80 and 120."}) @Example({"TriangularStepFunction(80,10)","Create a sequence of values where the average and median is 80, but the range of values is between 70 and 90."}) TriangularStepFunction(long average, long variance) { if (variance < 0 || variance > average) { @@ -63,9 +63,6 @@ public class TriangularStepFunction implements LongUnaryOperator { TriangularStepFunction(long average) { this(average, average/2); -// if (maxOffset>=avgsize) { -// throw new RuntimeException("max offset " + maxOffset + " has to be less than avg size " + avgsize); -// } } @Override @@ -85,10 +82,6 @@ public class TriangularStepFunction implements LongUnaryOperator { return result; } - public long inlined(long operand) { - return (operand < operand - operand % median + sizer.applyAsLong(operand - operand % median)) ? operand / median : operand / median + 1; - } - @Override public String toString() { return this.getClass().getSimpleName()+"{median="+median+",variance="+variance+"}"; diff --git a/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunctionTest.java b/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunctionTest.java index 4df0c7b90..a835fa3d5 100644 --- a/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunctionTest.java +++ b/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunctionTest.java @@ -48,30 +48,6 @@ public class TriangularStepFunctionTest { assertThat(IntStream.of(runLengths).max().orElseThrow()).isEqualTo(90L); } - @Test - public void testStepSlice() { - int avgsize=10; - TriangularStepFunction f = new TriangularStepFunction(avgsize); - int[] ary = new int[avgsize*2]; - long current=0L; - int count=0; - for (int i = 0; i < 10000; i++) { - long result = f.applyAsLong(i); - if (result==current) { - count++; - } else { - ary[count]++; - current=result; - count=0; - } - } - - for (int i = 0; i < ary.length; i++) { - System.out.println("bucket " + i + ", count " + ary[i]); - } - } - - @Test public void testIncrementalVariance() { TriangularStepFunction f = new TriangularStepFunction(100, 0); @@ -141,6 +117,5 @@ public class TriangularStepFunctionTest { stats.accept(count); } return stats; - } } From 15f15ebc9065d2144f3d467b65338c0621d9d0ba Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 7 Dec 2022 14:32:35 -0600 Subject: [PATCH 33/45] nosqlbench-828 nosqlbench-824 --- .../to_long/TriangularStepFunction.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java index 266ab3732..758fddf69 100644 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java @@ -41,22 +41,22 @@ import java.util.function.LongUnaryOperator; public class TriangularStepFunction implements LongUnaryOperator { private final Hash hasher = new Hash(); - private final long median; + private final long average; private final LongUnaryOperator sizer; private final long variance; - @Example({"TriangularStepFunction(100,20)","Create a sequence of values where the average and median is 100, but the range of values is between 80 and 120."}) - @Example({"TriangularStepFunction(80,10)","Create a sequence of values where the average and median is 80, but the range of values is between 70 and 90."}) + @Example({"TriangularStepFunction(100,20)","Create a sequence of values where the average is 100, but the range of values is between 80 and 120."}) + @Example({"TriangularStepFunction(80,10)","Create a sequence of values where the average is 80, but the range of values is between 70 and 90."}) TriangularStepFunction(long average, long variance) { - if (variance < 0 || variance > average) { + if (average<=0 || variance < 0 || variance > average) { throw new RuntimeException( - "The median must be non-negative, and the variance must be less than the median. " + - "You provided median=" + average + ", variance=" + variance + "." + "The average must be non-negative, and the variance must be less than the average. " + + "You provided average=" + average + ", variance=" + variance + "." ); } - this.median = average; + this.average = average; this.variance = variance; this.sizer = new HashRange(average-variance,average+variance); } @@ -68,9 +68,9 @@ public class TriangularStepFunction implements LongUnaryOperator { @Override public long applyAsLong(long operand) { // window number - long count = operand / median; + long count = operand / average; // offset within window - long offset = operand % median; + long offset = operand % average; // base of window long base = operand - offset; // variate up to window size @@ -84,6 +84,6 @@ public class TriangularStepFunction implements LongUnaryOperator { @Override public String toString() { - return this.getClass().getSimpleName()+"{median="+median+",variance="+variance+"}"; + return this.getClass().getSimpleName()+"{average="+ average +",variance="+variance+"}"; } } From a1fbb6dccaf53ced46c9ab104fe1cd899622b003 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 7 Dec 2022 14:33:46 -0600 Subject: [PATCH 34/45] use average instead of median, add safety check for average==0 --- .../basics/shared/from_long/to_long/TriangularStepFunction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java index 758fddf69..9d7d8f8ae 100644 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java @@ -52,7 +52,7 @@ public class TriangularStepFunction implements LongUnaryOperator { TriangularStepFunction(long average, long variance) { if (average<=0 || variance < 0 || variance > average) { throw new RuntimeException( - "The average must be non-negative, and the variance must be less than the average. " + + "The average must be positive, variance non-negative and the variance must be less than the average. " + "You provided average=" + average + ", variance=" + variance + "." ); } From 6fbbfb5d0f9b403926754f8acad2a207cbcd8f89 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 7 Dec 2022 15:01:11 -0600 Subject: [PATCH 35/45] diagram updates --- devdocs/sketches/docs-publishing-flow.svg | 345 ++++++++++++++++------ 1 file changed, 261 insertions(+), 84 deletions(-) diff --git a/devdocs/sketches/docs-publishing-flow.svg b/devdocs/sketches/docs-publishing-flow.svg index a16d11f52..ac6f1f7b5 100644 --- a/devdocs/sketches/docs-publishing-flow.svg +++ b/devdocs/sketches/docs-publishing-flow.svg @@ -186,6 +186,90 @@ d="M 0,0 5,-5 -12.5,0 5,5 Z" id="path10457-28" /> + + + + + + + + + + + + + + + + + + + transform="translate(-101.6,28.574999)"> NB + style="text-align:center;text-anchor:middle;stroke-width:0.264583" + x="163.23721" + y="-35.909264">NBmain branch + transform="translate(171.45,17.4625)"> + transform="translate(36.515457,20.6375)"> + transform="translate(36.152018,80.16875)"> build + transform="translate(155.575,-112.7125)"> + cx="56.960236" + cy="65.919228" + r="2.7336953" /> + style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="m 57.15,68.2625 -10e-7,6.35 -3.174999,6.35" + id="path11421" + sodipodi:nodetypes="ccc" /> + style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="m 57.149999,74.6125 3.175001,6.35" + id="path11423" + sodipodi:nodetypes="cc" /> + style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:none;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;-inkscape-stroke:none;stop-color:#000000;stop-opacity:1" + d="m 53.975,71.4375 h 6.35" + id="path11425" + sodipodi:nodetypes="cc" /> edit embedededit embededdocs and code + x="127.40736" + y="-35.554535" + id="tspan13944">docs, javadoc,and code edit docscontent publish docspublish all docscontent to maincontent to maindocs site publish embeddedpublish embeddeddocs content to maindocs content to maindocs site publish embeddedpublish embeddeddocs content to previewdocs content to previewdocs site + on push to mainand change toRELEASENOTES.md + Initially, this doc sitewill be checkpointed manually into the main docs site + on push to mainand NO change toRELEASENOTES.md Date: Wed, 7 Dec 2022 15:07:34 -0600 Subject: [PATCH 36/45] fix camel case across all affected method signatures --- .../cqld4/Cqld4CqlReboundStatement.java | 4 ++-- .../cqld4/optypes/Cqld4CqlBatchStatement.java | 4 ++-- .../adapter/cqld4/optypes/Cqld4CqlOp.java | 24 +++++++++---------- .../optypes/Cqld4CqlPreparedStatement.java | 4 ++-- .../optypes/Cqld4CqlSimpleStatement.java | 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4CqlReboundStatement.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4CqlReboundStatement.java index e2fb9e1d9..6cf024b36 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4CqlReboundStatement.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4CqlReboundStatement.java @@ -24,8 +24,8 @@ import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlOp; public class Cqld4CqlReboundStatement extends Cqld4CqlOp { private final BoundStatement stmt; - public Cqld4CqlReboundStatement(CqlSession session, int maxpages, boolean retryreplace, int maxlwtretries, int lwtRetryCount, BoundStatement rebound, RSProcessors processors) { - super(session,maxpages,retryreplace,maxlwtretries,lwtRetryCount, processors); + public Cqld4CqlReboundStatement(CqlSession session, int maxPages, boolean retryReplace, int maxLwtRetries, int lwtRetryCount, BoundStatement rebound, RSProcessors processors) { + super(session,maxPages,retryReplace,maxLwtRetries,lwtRetryCount, processors); this.stmt = rebound; } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlBatchStatement.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlBatchStatement.java index 6828a70ec..d122aa2eb 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlBatchStatement.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlBatchStatement.java @@ -24,8 +24,8 @@ public class Cqld4CqlBatchStatement extends Cqld4CqlOp { private final BatchStatement stmt; - public Cqld4CqlBatchStatement(CqlSession session, BatchStatement stmt, int maxpages, int maxlwtretries, boolean retryreplace) { - super(session,maxpages,retryreplace,maxlwtretries,new RSProcessors()); + public Cqld4CqlBatchStatement(CqlSession session, BatchStatement stmt, int maxPage, int maxLwtRetries, boolean retryReplace) { + super(session,maxPage,retryReplace,maxLwtRetries,new RSProcessors()); this.stmt = stmt; } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java index ec9a147b6..67931b83a 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java @@ -45,8 +45,8 @@ import java.util.Map; public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, OpGenerator, OpResultSize { private final CqlSession session; - private final int maxpages; - private final boolean retryreplace; + private final int maxPages; + private final boolean retryReplace; private final int maxLwtRetries; private int retryReplaceCount =0; @@ -54,18 +54,18 @@ public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, private Cqld4CqlOp nextOp; private final RSProcessors processors; - public Cqld4CqlOp(CqlSession session, int maxpages, boolean retryreplace, int maxLwtRetries, RSProcessors processors) { + public Cqld4CqlOp(CqlSession session, int maxPages, boolean retryReplace, int maxLwtRetries, RSProcessors processors) { this.session = session; - this.maxpages = maxpages; - this.retryreplace = retryreplace; + this.maxPages = maxPages; + this.retryReplace = retryReplace; this.maxLwtRetries =maxLwtRetries; this.processors = processors; } - protected Cqld4CqlOp(CqlSession session, int maxpages, boolean retryreplace, int maxLwtRetries, int retryRplaceCount, RSProcessors processors) { + protected Cqld4CqlOp(CqlSession session, int maxPages, boolean retryReplace, int maxLwtRetries, int retryRplaceCount, RSProcessors processors) { this.session = session; - this.maxpages = maxpages; - this.retryreplace = retryreplace; + this.maxPages = maxPages; + this.retryReplace = retryReplace; this.maxLwtRetries =maxLwtRetries; this.retryReplaceCount=retryRplaceCount; this.processors = processors; @@ -79,7 +79,7 @@ public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, int totalRows = 0; if (!rs.wasApplied()) { - if (!retryreplace) { + if (!retryReplace) { throw new ChangeUnappliedCycleException(rs, getQueryString()); } else { retryReplaceCount++; @@ -103,8 +103,8 @@ public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, Row row = reader.next(); processors.buffer(row); } - if (pages++ > maxpages) { - throw new UnexpectedPagingException(rs, getQueryString(), pages, maxpages, stmt.getPageSize()); + if (pages++ > maxPages) { + throw new UnexpectedPagingException(rs, getQueryString(), pages, maxPages, stmt.getPageSize()); } if (rs.isFullyFetched()) { break; @@ -136,7 +136,7 @@ public abstract class Cqld4CqlOp implements CycleOp, VariableCapture, private Cqld4CqlOp rebindLwt(Statement stmt, Row row) { BoundStatement rebound = LWTRebinder.rebindUnappliedStatement(stmt, row); - return new Cqld4CqlReboundStatement(session, maxpages, retryreplace, maxLwtRetries, retryReplaceCount, rebound, processors); + return new Cqld4CqlReboundStatement(session, maxPages, retryReplace, maxLwtRetries, retryReplaceCount, rebound, processors); } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java index 938475f7d..9e6dfcad9 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlPreparedStatement.java @@ -24,8 +24,8 @@ public class Cqld4CqlPreparedStatement extends Cqld4CqlOp { private final BoundStatement stmt; - public Cqld4CqlPreparedStatement(CqlSession session, BoundStatement stmt, int maxpages, boolean retryreplace, int maxLwtRetries, RSProcessors processors) { - super(session,maxpages,retryreplace,maxLwtRetries,processors); + public Cqld4CqlPreparedStatement(CqlSession session, BoundStatement stmt, int maxPages, boolean retryReplace, int maxLwtRetries, RSProcessors processors) { + super(session,maxPages,retryReplace,maxLwtRetries,processors); this.stmt = stmt; } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java index 3fb6a9578..0f119c91d 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlSimpleStatement.java @@ -23,8 +23,8 @@ import io.nosqlbench.adapter.cqld4.RSProcessors; public class Cqld4CqlSimpleStatement extends Cqld4CqlOp { private final SimpleStatement stmt; - public Cqld4CqlSimpleStatement(CqlSession session, SimpleStatement stmt, int maxpages, boolean retryreplace, int maxlwtretries) { - super(session, maxpages,retryreplace, maxlwtretries, new RSProcessors()); + public Cqld4CqlSimpleStatement(CqlSession session, SimpleStatement stmt, int maxPages, boolean retryReplace, int maxLwtRetries) { + super(session, maxPages,retryReplace, maxLwtRetries, new RSProcessors()); this.stmt = stmt; } From c4c13bbadd02b30ead68d8356f84cedfe6ea65c7 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 7 Dec 2022 16:30:18 -0600 Subject: [PATCH 37/45] minor features release for blocked users --- RELEASENOTES.md | 215 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 203 insertions(+), 12 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index e2e130ab7..ff78ef769 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,12 +1,203 @@ -- 114aea71b (HEAD -> main) Merge branch 'main' of github.com:nosqlbench/nosqlbench -- 0fd85c09b (origin/main) Merge pull request #728 from nosqlbench/feature/mongodb_baselines2_workloads -- 99f0226fc Merge pull request #729 from nosqlbench/snyk-upgrade-2d988862477c7e76f15ff8b65bcdc3a4 -- d22413259 Merge pull request #727 from nosqlbench/snyk-upgrade-b28610be666fa7e80936b0c2f87df569 -- d49a5087d improved build diagnostics -- e594aab92 (origin/snyk-upgrade-2d988862477c7e76f15ff8b65bcdc3a4) fix: upgrade com.datastax.oss:pulsar-jms from 2.4.4 to 2.4.9 -- 64990c412 (origin/feature/mongodb_baselines2_workloads) Initial working draft of MongoDB timeseries -- 7ebb16a06 (origin/snyk-upgrade-b28610be666fa7e80936b0c2f87df569) fix: upgrade org.postgresql:postgresql from 42.4.2 to 42.5.0 -- 68cdd075b new function, security updates, actions fix -- ec8e6ee71 Merge branch 'main' of github.com:nosqlbench/nosqlbench -- a63fa951c actions fix release -- 0f6bce0b0 fix typo in docker push logic +- d0f25f5c6 (HEAD -> main, origin/main) Merge pull request #831 from nosqlbench/nosqlbench-830 +- 03a3afd86 Merge pull request #828 from nosqlbench/nosqlbench-824 +- 2b63b8e64 (origin/nosqlbench-830, nosqlbench-830) fix camel case across all affected method signatures +- 6fbbfb5d0 (origin/nosqlbench-824, nosqlbench-824) diagram updates +- a1fbb6dcc use average instead of median, add safety check for average==0 +- 15f15ebc9 nosqlbench-828 nosqlbench-824 +- 7737ffc0f nosqlbench-828 nosqlbench-824 +- d90731ec7 typo fix +- b33e93eae Merge pull request #827 from nosqlbench/snyk-upgrade-5c7e9a58ec726ce1a5540b1007997a68 +- 3677a60ec nosqlbench-830 limit spin-looping with cqld4 retryreplace option. +- 97cf9a32c add ad-hoc doc to show publishing flow +- dc7428395 nosqlbench-824 Add function to provide clustered values with monotonic stepping +- eaa8c5671 Merge pull request #826 from nosqlbench/snyk-upgrade-62cc0785172d699541ae5c0e9e487268 +- a1e7eb198 Merge pull request #825 from nosqlbench/snyk-upgrade-32a05328633bd6d7b922681265c560ee +- 14df638f7 Merge pull request #823 from nosqlbench/snyk-upgrade-d560c5ed862bae1f5c8558515d8e4571 +- cde915914 Merge pull request #818 from nosqlbench/snyk-upgrade-e5f0f6d488b383219dd2ca00632f1dde +- b13c899e8 Merge pull request #815 from nosqlbench/javadoc_improvements +- f8623f3b7 (origin/snyk-upgrade-5c7e9a58ec726ce1a5540b1007997a68) fix: upgrade io.swagger.core.v3:swagger-models from 2.2.4 to 2.2.6 +- 76f5507e4 (origin/snyk-upgrade-62cc0785172d699541ae5c0e9e487268) fix: upgrade org.apache.commons:commons-compress from 1.21 to 1.22 +- 150793cda (origin/snyk-upgrade-32a05328633bd6d7b922681265c560ee) fix: upgrade org.glassfish.jersey.media:jersey-media-json-jackson from 3.0.8 to 3.1.0 +- 82bfa4485 (origin/snyk-upgrade-d560c5ed862bae1f5c8558515d8e4571) fix: upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.340 to 1.12.341 +- 2ca4320c2 Merge pull request #819 from nosqlbench/snyk-upgrade-cab2f09370a21c3a871a0379d2ad3792 +- 569d35f2b Merge pull request #820 from nosqlbench/snyk-upgrade-a30d7c35198a17d29ecdca13feba6579 +- 5595f5083 Merge pull request #821 from nosqlbench/snyk-upgrade-f3ed4565eab47a4f529c1dfe1685b618 +- 8eb934c56 Merge pull request #822 from nosqlbench/snyk-upgrade-292f7f0db6ee968fd7ce222173de764c +- 141bfe088 fix: upgrade org.graalvm.js:js-scriptengine from 21.3.3.1 to 21.3.4 +- fd4673f15 fix: upgrade io.dropwizard.metrics:metrics-graphite from 4.2.10 to 4.2.12 +- 3030e58df fix: upgrade org.graalvm.tools:profiler from 20.3.6.1 to 20.3.8 +- 1e6367ecb fix: upgrade org.graalvm.js:js from 21.3.3.1 to 21.3.4 +- 52ef69373 (origin/snyk-upgrade-e5f0f6d488b383219dd2ca00632f1dde) fix: upgrade io.netty:netty-handler from 4.1.84.Final to 4.1.85.Final +- 726028059 (origin/javadoc_improvements, javadoc_improvements) update docs for op types +- b4b09048f update docs for op types +- 98493cfcb Merge pull request #812 from nosqlbench/snyk-upgrade-1754fe16038ae415d7733b7b3fdfb0ad +- 842427e4f Merge pull request #813 from nosqlbench/snyk-upgrade-ca34705c36a67d06ea66413eae078875 +- b06996319 Merge pull request #814 from nosqlbench/snyk-upgrade-a37d7e3729b4fe18258665c47ebd0cf4 +- b3e20c16e Merge pull request #809 from nosqlbench/nosqlbench-808 +- 6ad405d9f fix: upgrade org.glassfish.jersey.core:jersey-server from 3.0.8 to 3.1.0 +- db6594d41 Merge pull request #811 from nosqlbench/snyk-upgrade-8275b1467b7f05117180335fa4d5f944 +- b6cc8fcf0 Merge pull request #810 from nosqlbench/snyk-upgrade-342e4814a0e11c6d2f22b112ba735c12 +- d88359a6a fix: upgrade org.snakeyaml:snakeyaml-engine from 2.4 to 2.5 +- f413a5af7 fix: upgrade org.codehaus.groovy:groovy from 3.0.12 to 3.0.13 +- 34b51b198 fix: upgrade com.datastax.oss:java-driver-query-builder from 4.14.1 to 4.15.0 +- c89339839 fix: upgrade joda-time:joda-time from 2.11.2 to 2.12.1 +- 9bca349e4 (origin/nosqlbench-808, nosqlbench-808) use correct value in exception handlers +- bf518c375 Merge pull request #804 from nosqlbench/snyk-upgrade-4a7ee2762e595acb421d164dc872273b +- b16de9edb Merge pull request #806 from nosqlbench/snyk-fix-d203bc55bd9076b01b05bf8a45a65046 +- 7c8727e15 Merge pull request #805 from nosqlbench/snyk-upgrade-18902492bf09462887646247db53d19b +- 017fcf8ff Merge pull request #803 from nosqlbench/snyk-upgrade-12d61683e1022753a4e624d6d62d5a19 +- 1f96e625d Merge pull request #802 from nosqlbench/snyk-upgrade-cf1da0c80b868193683f0a03748b3835 +- ce353d899 Merge pull request #801 from nosqlbench/snyk-upgrade-399641fb454b01fc3feead152acf52d5 +- 74ae911e5 Merge pull request #800 from nosqlbench/dependabot/maven/driver-cockroachdb/org.postgresql-postgresql-42.5.1 +- 82c253acb (origin/snyk-fix-d203bc55bd9076b01b05bf8a45a65046) fix: driver-cockroachdb/pom.xml to reduce vulnerabilities +- fec156a25 (origin/snyk-upgrade-18902492bf09462887646247db53d19b) fix: upgrade io.netty:netty-handler from 4.1.82.Final to 4.1.84.Final +- d622670f7 (origin/snyk-upgrade-4a7ee2762e595acb421d164dc872273b) fix: upgrade org.graalvm.sdk:graal-sdk from 21.3.3.1 to 21.3.4 +- ec858f69d (origin/snyk-upgrade-12d61683e1022753a4e624d6d62d5a19) fix: upgrade info.picocli:picocli from 4.6.3 to 4.7.0 +- 96be7811a (origin/snyk-upgrade-cf1da0c80b868193683f0a03748b3835) fix: upgrade com.google.code.gson:gson from 2.9.1 to 2.10 +- 64f11bed1 (origin/snyk-upgrade-399641fb454b01fc3feead152acf52d5) fix: upgrade com.github.oshi:oshi-core-java11 from 6.2.2 to 6.3.1 +- 702692058 (origin/dependabot/maven/driver-cockroachdb/org.postgresql-postgresql-42.5.1) build(deps): bump postgresql in /driver-cockroachdb +- f39ba9331 fix: upgrade io.swagger.core.v3:swagger-models from 2.2.3 to 2.2.4 (#799) +- 9afe376b4 fix: upgrade io.swagger.parser.v3:swagger-parser from 2.1.4 to 2.1.7 (#798) +- 79a2d0977 fix: upgrade org.apache.pulsar:pulsar-client from 2.10.1 to 2.10.2 (#796) +- 523f500cc Merge pull request #773 from nosqlbench/snyk-fix-8aa6ab863bb5432def89d1b31da171c7 +- 26ee71883 Merge pull request #772 from nosqlbench/snyk-fix-ab49d52aab0b82f5b5b84c1e7389d4b1 +- 8b87b10ca Merge branch 'main' into snyk-fix-ab49d52aab0b82f5b5b84c1e7389d4b1 +- 00e0596fd Merge pull request #771 from nosqlbench/snyk-fix-ac633ddba1da2c7ec65dff2ae08fa725 +- 1e0ac38b4 Merge branch 'main' into snyk-fix-ac633ddba1da2c7ec65dff2ae08fa725 +- f9adeff1d Merge pull request #774 from nosqlbench/snyk-fix-b2c236df12318590f34b590edea9705e +- 5ffda7d1e Merge branch 'main' into snyk-fix-b2c236df12318590f34b590edea9705e +- e5f03b91a Merge pull request #775 from nosqlbench/snyk-fix-8c31cd4a0b1166676ed0fe1e7cf9a216 +- c7e5e7291 Merge branch 'main' into snyk-fix-8c31cd4a0b1166676ed0fe1e7cf9a216 +- 3ceddb631 Merge pull request #776 from nosqlbench/snyk-fix-6ba0467ba1a5842fab939a7c4f8b6a14 +- e39954ccc Merge branch 'main' into snyk-fix-6ba0467ba1a5842fab939a7c4f8b6a14 +- 56470956b Merge pull request #778 from nosqlbench/snyk-fix-62d2529ba6efa1321216d8e295c48e5d +- 59b430b12 Merge branch 'main' into snyk-fix-62d2529ba6efa1321216d8e295c48e5d +- 4c764250d Merge pull request #794 from nosqlbench/snyk-fix-2ac9869bf1d47aa3f30a82f3655c5432 +- 7491370ba Merge branch 'main' into snyk-fix-2ac9869bf1d47aa3f30a82f3655c5432 +- 47db96c06 Merge pull request #795 from nosqlbench/snyk-fix-3f81768022b768b87ace72058200f7ac +- fa528eebe Merge pull request #779 from nosqlbench/shutdown_hooks +- 875e2d8f9 (origin/shutdown_hooks, shutdown_hooks) temporarily disable test for close exception handler +- 901034875 fix for second noisy shutdown test +- 2db6c6cf6 Update codeql-analysis.yml +- 18ba2fdaf Merge branch 'main' into shutdown_hooks +- 8602d5b53 Merge pull request #793 from yabinmeng/main +- 3b616d7d2 (origin/snyk-fix-3f81768022b768b87ace72058200f7ac) fix: adapter-dynamodb/pom.xml to reduce vulnerabilities +- eaefd9a1b Merge pull request #781 from nosqlbench/780-sporadic-buildtest-failures-gh-actions +- 69617042b (origin/780-sporadic-buildtest-failures-gh-actions, 780-sporadic-buildtest-failures-gh-actions) minor fixes on PR +- 28f561aed direct work-around for flaky test, simple improvements to logging, some improvements to consistent view of status, misc cleanups +- 0193ed355 (origin/snyk-fix-2ac9869bf1d47aa3f30a82f3655c5432) fix: adapter-dynamodb/pom.xml to reduce vulnerabilities +- c077218e8 Fix example yaml file producer and consumer parameter names +- 16417f37b additional diagnostics for spurious test +- f7c9e50f3 Add README file and adjust sample yaml files +- eb0013f16 logging updates for build sanity checking +- bc4773269 build workflow should upload logfiles for success() or failure() of previous jobs, but not when canceled +- 40c968234 Merge branch '780-sporadic-buildtest-failures-gh-actions' of github.com:nosqlbench/nosqlbench into 780-sporadic-buildtest-failures-gh-actions +- dcf7baf86 Dead code utest resurrection and timing w/ gh actions Cyclerate=10 to test github actions part2 Debug and exception handling Detailed error handler logging System out diagnostics Capture step included Try-catch diagnostics. sysout cleanup; general cleanup +- 20ed4950d removing broken actions plugin on main +- 2831d8c3f Merge branch 'main' of github.com:nosqlbench/nosqlbench +- 3e8153dd8 Merge pull request #792 from yabinmeng/main +- b8684107b memoize retries in hot code path +- ebb6f2c58 include space usage in diag for testing +- 48192cbde close native driver connections on on activity+space shutdown +- 4471d90cc implement closeable spaces via decorator interface +- bd8aff908 Added support for 1) subscription initial position and 2) regex subscription mode +- 35eec4d81 Dead code utest resurrection and timing w/ gh actions Cyclerate=10 to test github actions part2 Debug and exception handling Detailed error handler logging System out diagnostics Capture step included Try-catch diagnostics. sysout cleanup; general cleanup +- e80e9926c Merge pull request #791 from yabinmeng/main +- 0de710288 Add support for 1) producer message compression 2) consumer DLT/negAck/ackTimeout policy +- 12f8697e0 Complete the first draft of NB5 Pulsar adapter code, except the per-thread rate limiter. +- e128d2867 Merge branch 'nosqlbench:main' into main +- 0c71696b1 Completed NB4 Pulsar driver code migration to NB5. But there are still a few lingering issues due to the difference of how NB4 vs NB5 driver/adapter interacts with the NB engine. +- 33e330dc3 [Snyk] Security upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.325 to 1.12.330 (#768) +- 2158dfa33 fix: upgrade io.netty:netty-handler from 4.1.81.Final to 4.1.82.Final (#767) +- 5fc22bbaa fix: upgrade com.google.code.gson:gson from 2.9.0 to 2.9.1 (#766) +- 6999a675f [Snyk] Upgrade io.dropwizard.metrics:metrics-core from 4.2.10 to 4.2.12 (#765) +- ca6f2b052 Add PulsarAdapter specific metrics +- 8b76d1fcf add shutdown hooks for adapters and spaces +- 8c3d744a0 Merge pull request #764 from nosqlbench/snyk-upgrade-6693c12bca4fafc634662d8962afcb9a +- eb1de8f1d Timing changes for github actions to properly perform build/test step. +- a05c511de fix: upgrade org.eclipse.jetty:jetty-server from 11.0.11 to 11.0.12 (#763) +- c309d200a - Delete unsed code "driver-jms" - NB5 Pulsar driver Admin functionality in place - create/delete tenant, namespace, and topic +- 2b9452fde Buildfixes, trying to unblock current builds (#777) +- fe268d7f8 (origin/snyk-fix-62d2529ba6efa1321216d8e295c48e5d) fix: nb-api/pom.xml to reduce vulnerabilities +- d47462929 (origin/buildfixes, buildfixes) remove blocking issues github actions plugin, since it is broken +- 4dc830aa1 improve wording around op template resolution +- b2f74d329 make example/test script more robust for meager github actions environment +- cbb01fb7f (origin/snyk-fix-6ba0467ba1a5842fab939a7c4f8b6a14) fix: adapter-dynamodb/pom.xml to reduce vulnerabilities +- e50098800 (origin/snyk-fix-8c31cd4a0b1166676ed0fe1e7cf9a216) fix: nb-api/pom.xml to reduce vulnerabilities +- 422849209 (origin/snyk-fix-b2c236df12318590f34b590edea9705e) fix: adapter-dynamodb/pom.xml to reduce vulnerabilities +- c5b632518 (origin/snyk-fix-8aa6ab863bb5432def89d1b31da171c7) fix: docsys/pom.xml to reduce vulnerabilities +- f8af301fb (origin/snyk-fix-ab49d52aab0b82f5b5b84c1e7389d4b1) fix: nb-api/pom.xml to reduce vulnerabilities +- a5974919a (origin/snyk-fix-ac633ddba1da2c7ec65dff2ae08fa725) fix: adapter-dynamodb/pom.xml to reduce vulnerabilities +- 7c1fb1963 Merge pull request #770 from nosqlbench/ms-mongodb-keyvalue-schema-fix +- 98c7b1a97 Merge pull request #769 from nosqlbench/base64 +- 45ee75fef bump min_version +- 8f32c9496 Updated MongoDB key-value collection to have a 'value' field +- 9a92129a8 (origin/base64) base64 +- bb27a6f7f (origin/snyk-upgrade-6693c12bca4fafc634662d8962afcb9a) fix: upgrade io.swagger.core.v3:swagger-models from 2.2.2 to 2.2.3 +- 0e3893190 (origin/jeffb/nb736) Merge pull request #757 from nosqlbench/snyk-upgrade-de963ecc07996edad35bfd93b9db28c2 +- 92e41e7f7 Merge pull request #758 from nosqlbench/snyk-upgrade-38e72059fc3a8a2099fed05a4e8be9b0 +- 64093b2ab Merge pull request #759 from nosqlbench/snyk-upgrade-5f4f139c9c2fc23d4a2117f5bda8f31e +- 6d25c03ff Merge pull request #760 from nosqlbench/snyk-upgrade-c29375a9a40341a121dfc342ee787201 +- a92897b1d Merge pull request #761 from nosqlbench/snyk-upgrade-c51251ba7d6c428fd60b9aa362f5d4b4 +- 927f4e2c6 fix: upgrade org.eclipse.jetty:jetty-servlets from 11.0.11 to 11.0.12 +- f64a25280 fix: upgrade org.eclipse.jetty:jetty-servlet from 11.0.11 to 11.0.12 +- 74db1d97e fix: upgrade org.eclipse.jetty:jetty-rewrite from 11.0.11 to 11.0.12 +- 02257d41b fix: upgrade org.yaml:snakeyaml from 1.32 to 1.33 +- ed1365773 fix: upgrade joda-time:joda-time from 2.11.1 to 2.11.2 +- dae02bb60 Merge pull request #746 from nosqlbench/snyk-fix-c33fb16683078e999e1d6cab62020656 +- 90bf9dd47 Merge pull request #748 from nosqlbench/snyk-fix-4981c736a854543f414acc756aa6773d +- 2cbb844db Merge pull request #756 from nosqlbench/dependabot/maven/mvn-defaults/org.apache.commons-commons-text-1.10.0 +- 702187794 Merge pull request #744 from nosqlbench/snyk-fix-26f3021e2a4eb5d42b86ca025e907044 +- def46982f Merge branch 'main' into snyk-fix-26f3021e2a4eb5d42b86ca025e907044 +- 2292c6a21 Merge branch 'main' into snyk-fix-4981c736a854543f414acc756aa6773d +- 0021e1cf5 Merge pull request #740 from nosqlbench/snyk-upgrade-e3eae2348ed5c3a9c5a4df3da5434294 +- 2874bdd6a Merge branch 'main' into snyk-fix-c33fb16683078e999e1d6cab62020656 +- 5fb062c1a Merge pull request #749 from nosqlbench/snyk-fix-01ab345df3b2b9725654404c28451b69 +- f684760ee Merge branch 'main' into snyk-fix-01ab345df3b2b9725654404c28451b69 +- 25fe0a541 Merge pull request #739 from nosqlbench/snyk-upgrade-61e7fcecc7f6c453bf8ee04e1799fe04 +- 0cb654259 Merge pull request #738 from nosqlbench/snyk-upgrade-de204ed7275df7f0d05e2783982c6ac7 +- 6bcc1cfd2 build(deps): bump commons-text from 1.9 to 1.10.0 in /mvn-defaults +- 974b29f11 Merge pull request #742 from nosqlbench/snyk-upgrade-dc69205a67c70990049a72696a0c367b +- 88987fb04 Merge pull request #743 from nosqlbench/snyk-upgrade-ac680967c12a71fb0828a5fb85bd8259 +- 879a8778f Merge pull request #745 from nosqlbench/snyk-fix-6accd862435675058d9b3c98aefe4f4a +- af6380f70 Merge pull request #747 from nosqlbench/snyk-fix-fc4ff10e48c0a29879ed09f2c567dd08 +- 47489b075 Merge pull request #750 from nosqlbench/snyk-fix-30fed3e9e7bf54aa1715f6cb0cd3bd93 +- bc1039a11 Merge pull request #751 from nosqlbench/snyk-fix-eeda5403343ff40d314d648afcfdf408 +- 256d7b721 Merge pull request #752 from nosqlbench/snyk-fix-b9ee34aa87226cd6400a08d1f6f78f84 +- eb3e57041 Merge pull request #754 from nosqlbench/snyk-fix-5ff726edd68fbb5f2553166c8f1878d6 +- b478a3766 Merge pull request #755 from nosqlbench/snyk-fix-fec660b2a5471e3538d424b7a8a66438 +- f875f2c7f fix: nb-api/pom.xml to reduce vulnerabilities +- e90580de2 Merge pull request #753 from nosqlbench/ms-trivial-mongodb-workload-fix +- c35ec732e fix: adapter-dynamodb/pom.xml to reduce vulnerabilities +- 60bd26104 Minor update to timeseries clustering index name +- 3496f965c fix: mvn-defaults/pom.xml to reduce vulnerabilities +- e3187bc1e fix: mvn-defaults/pom.xml to reduce vulnerabilities +- 75aa1e45c fix: mvn-defaults/pom.xml to reduce vulnerabilities +- ad369143d fix: adapter-dynamodb/pom.xml to reduce vulnerabilities +- 9b279895e fix: nb-api/pom.xml to reduce vulnerabilities +- 49309872c fix: engine-rest/pom.xml to reduce vulnerabilities +- 35d57fca5 fix: nb-api/pom.xml & mvn-defaults/pom.xml to reduce vulnerabilities +- 5cb5bd5b0 fix: mvn-defaults/pom.xml to reduce vulnerabilities +- 8a3720792 fix: adapter-dynamodb/pom.xml to reduce vulnerabilities +- 7e5bd8f7f fix: upgrade com.datastax.oss:java-driver-core from 4.14.1 to 4.15.0 +- cb3191a1a fix: upgrade com.datastax.oss:pulsar-jms from 2.4.9 to 2.4.11 +- 706880c01 fix: upgrade org.apache.logging.log4j:log4j-core from 2.18.0 to 2.19.0 +- dc4c92eca fix: upgrade org.apache.logging.log4j:log4j-api from 2.18.0 to 2.19.0 +- e7a92d68d fix: upgrade org.apache.logging.log4j:log4j-jcl from 2.18.0 to 2.19.0 +- 87c3904f9 Merge pull request #730 from nosqlbench/snyk-upgrade-497803c521a3e4b5bf1f8320886e4693 +- c769c59af Merge pull request #731 from nosqlbench/snyk-upgrade-4de9e27b335c4ba704d719c6b8208fc9 +- ee4243af7 Merge pull request #732 from nosqlbench/snyk-upgrade-b07f4c6dc36ef0b84aac016e42bc0ac0 +- 59a0b3fbf Merge pull request #733 from nosqlbench/snyk-upgrade-9d3ff80bbe2914cac41eefa7bf016fc4 +- a67e9604b Merge pull request #734 from nosqlbench/snyk-upgrade-6f292d686b89f8516e45eb37ec3e77bb +- 1929bc87c Merge pull request #735 from nosqlbench/snyk-fix-d6c9e33a2e55402294f57e1ce7c62a6e +- ab7487e98 Merge pull request #737 from jeffbanks/jeffb/count-warn-nb561 +- 9bc3a2fd1 Notify user when using error=count vs. counter. +- b1d5411af Updated min_version to be 4.17.30 +- 5abef6db3 Updated min_version to be 4.17.30 +- 84b3b5e25 Updated min_version to be 4.17.30 +- a60c41256 (origin/snyk-fix-d6c9e33a2e55402294f57e1ce7c62a6e) fix: docsys/pom.xml to reduce vulnerabilities +- 35d5c0cee (origin/snyk-upgrade-6f292d686b89f8516e45eb37ec3e77bb) fix: upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.269 to 1.12.300 +- 9d3310413 (origin/snyk-upgrade-9d3ff80bbe2914cac41eefa7bf016fc4) fix: upgrade org.antlr:antlr4-runtime from 4.10.1 to 4.11.1 +- c41b5990d (origin/snyk-upgrade-b07f4c6dc36ef0b84aac016e42bc0ac0) fix: upgrade com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider from 2.13.3 to 2.13.4 +- e0fb11da4 (origin/snyk-upgrade-4de9e27b335c4ba704d719c6b8208fc9) fix: upgrade io.netty:netty-handler from 4.1.80.Final to 4.1.81.Final +- c6b327161 (origin/snyk-upgrade-497803c521a3e4b5bf1f8320886e4693) fix: upgrade com.amazonaws:aws-java-sdk-dynamodb from 1.12.262 to 1.12.299 From fb131e9b0977a8dc112f05829f2138afb772bdfa Mon Sep 17 00:00:00 2001 From: nb-droid Date: Wed, 7 Dec 2022 23:30:02 +0000 Subject: [PATCH 38/45] [maven-release-plugin] prepare release nosqlbench-4.17.31 --- adapter-cqld4/pom.xml | 4 ++-- adapter-diag/pom.xml | 6 +++--- adapter-dynamodb/pom.xml | 4 ++-- adapter-http/pom.xml | 4 ++-- adapter-mongodb/pom.xml | 4 ++-- adapter-pulsar/pom.xml | 6 +++--- adapter-stdout/pom.xml | 6 +++--- adapter-tcp/pom.xml | 4 ++-- adapters-api/pom.xml | 8 ++++---- docsys/pom.xml | 6 +++--- engine-api/pom.xml | 12 ++++++------ engine-cli/pom.xml | 6 +++--- engine-clients/pom.xml | 4 ++-- engine-core/pom.xml | 6 +++--- engine-docker/pom.xml | 4 ++-- engine-docs/pom.xml | 4 ++-- engine-extensions/pom.xml | 4 ++-- engine-rest/pom.xml | 4 ++-- mvn-defaults/pom.xml | 4 ++-- nb-annotations/pom.xml | 2 +- nb-api/pom.xml | 4 ++-- nb-spectest/pom.xml | 2 +- nb5/pom.xml | 22 +++++++++++----------- nbr-examples/pom.xml | 6 +++--- nbr/pom.xml | 14 +++++++------- pom.xml | 4 ++-- virtdata-api/pom.xml | 6 +++--- virtdata-lang/pom.xml | 2 +- virtdata-lib-basics/pom.xml | 4 ++-- virtdata-lib-curves4/pom.xml | 6 +++--- virtdata-lib-random/pom.xml | 6 +++--- virtdata-lib-realer/pom.xml | 4 ++-- virtdata-realdata/pom.xml | 4 ++-- virtdata-userlibs/pom.xml | 16 ++++++++-------- 34 files changed, 101 insertions(+), 101 deletions(-) diff --git a/adapter-cqld4/pom.xml b/adapter-cqld4/pom.xml index 4e8c8d41a..9988fb06c 100644 --- a/adapter-cqld4/pom.xml +++ b/adapter-cqld4/pom.xml @@ -20,7 +20,7 @@ io.nosqlbench mvn-defaults - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -38,7 +38,7 @@ io.nosqlbench adapters-api - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/adapter-diag/pom.xml b/adapter-diag/pom.xml index 302538264..0811ac0e5 100644 --- a/adapter-diag/pom.xml +++ b/adapter-diag/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -37,13 +37,13 @@ io.nosqlbench nb-annotations - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench engine-api - 4.17.31-SNAPSHOT + 4.17.31 compile diff --git a/adapter-dynamodb/pom.xml b/adapter-dynamodb/pom.xml index 5d6d2d48e..c12b98c50 100644 --- a/adapter-dynamodb/pom.xml +++ b/adapter-dynamodb/pom.xml @@ -20,7 +20,7 @@ io.nosqlbench mvn-defaults - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -39,7 +39,7 @@ io.nosqlbench adapters-api - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/adapter-http/pom.xml b/adapter-http/pom.xml index b6ccd52eb..847bb3835 100644 --- a/adapter-http/pom.xml +++ b/adapter-http/pom.xml @@ -20,7 +20,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -38,7 +38,7 @@ io.nosqlbench engine-api - 4.17.31-SNAPSHOT + 4.17.31 compile diff --git a/adapter-mongodb/pom.xml b/adapter-mongodb/pom.xml index e60032009..fb4c143d1 100644 --- a/adapter-mongodb/pom.xml +++ b/adapter-mongodb/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -36,7 +36,7 @@ io.nosqlbench engine-api - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/adapter-pulsar/pom.xml b/adapter-pulsar/pom.xml index 2428c54fe..891fe9d9a 100644 --- a/adapter-pulsar/pom.xml +++ b/adapter-pulsar/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -41,13 +41,13 @@ io.nosqlbench engine-api - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench adapters-api - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/adapter-stdout/pom.xml b/adapter-stdout/pom.xml index 285f86353..95e747c1c 100644 --- a/adapter-stdout/pom.xml +++ b/adapter-stdout/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -38,14 +38,14 @@ io.nosqlbench nb-annotations - 4.17.31-SNAPSHOT + 4.17.31 compile io.nosqlbench adapters-api - 4.17.31-SNAPSHOT + 4.17.31 compile diff --git a/adapter-tcp/pom.xml b/adapter-tcp/pom.xml index 6204a442b..5c1029def 100644 --- a/adapter-tcp/pom.xml +++ b/adapter-tcp/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -40,7 +40,7 @@ io.nosqlbench engine-api - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/adapters-api/pom.xml b/adapters-api/pom.xml index ea91c96ca..39beee7a0 100644 --- a/adapters-api/pom.xml +++ b/adapters-api/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -39,19 +39,19 @@ io.nosqlbench nb-spectest - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench nb-api - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench virtdata-userlibs - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/docsys/pom.xml b/docsys/pom.xml index 5735bd869..aa8d7a3cc 100644 --- a/docsys/pom.xml +++ b/docsys/pom.xml @@ -28,7 +28,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -37,7 +37,7 @@ io.nosqlbench nb-api - 4.17.31-SNAPSHOT + 4.17.31 @@ -133,7 +133,7 @@ io.nosqlbench virtdata-api - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/engine-api/pom.xml b/engine-api/pom.xml index 9db3bb359..ef7547237 100644 --- a/engine-api/pom.xml +++ b/engine-api/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -39,31 +39,31 @@ io.nosqlbench nb-api - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench adapters-api - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench nb-spectest - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench nb-annotations - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench virtdata-userlibs - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/engine-cli/pom.xml b/engine-cli/pom.xml index abccb1e38..f9e2bf0e4 100644 --- a/engine-cli/pom.xml +++ b/engine-cli/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -23,13 +23,13 @@ io.nosqlbench engine-core - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench engine-docker - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/engine-clients/pom.xml b/engine-clients/pom.xml index cc761c986..6a3581c4e 100644 --- a/engine-clients/pom.xml +++ b/engine-clients/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -21,7 +21,7 @@ io.nosqlbench engine-api - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/engine-core/pom.xml b/engine-core/pom.xml index 20caa2538..00e2b0077 100644 --- a/engine-core/pom.xml +++ b/engine-core/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -38,7 +38,7 @@ io.nosqlbench engine-api - 4.17.31-SNAPSHOT + 4.17.31 @@ -89,7 +89,7 @@ io.nosqlbench engine-clients - 4.17.31-SNAPSHOT + 4.17.31 compile diff --git a/engine-docker/pom.xml b/engine-docker/pom.xml index bd03836dc..88d25d2ae 100644 --- a/engine-docker/pom.xml +++ b/engine-docker/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -56,7 +56,7 @@ io.nosqlbench engine-api - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/engine-docs/pom.xml b/engine-docs/pom.xml index 8c5838d89..7310fc54d 100644 --- a/engine-docs/pom.xml +++ b/engine-docs/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -28,7 +28,7 @@ io.nosqlbench docsys - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/engine-extensions/pom.xml b/engine-extensions/pom.xml index 9a700fb61..376fc0f04 100644 --- a/engine-extensions/pom.xml +++ b/engine-extensions/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -22,7 +22,7 @@ io.nosqlbench engine-api - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/engine-rest/pom.xml b/engine-rest/pom.xml index 58df73fa6..0591cdaee 100644 --- a/engine-rest/pom.xml +++ b/engine-rest/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -35,7 +35,7 @@ io.nosqlbench engine-cli - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index a20b9ac09..d891e7c7f 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -19,7 +19,7 @@ io.nosqlbench mvn-defaults - 4.17.31-SNAPSHOT + 4.17.31 pom @@ -89,7 +89,7 @@ scm:git:https://github.com/nosqlbench/nosqlbench.git - HEAD + nosqlbench-4.17.31 diff --git a/nb-annotations/pom.xml b/nb-annotations/pom.xml index 5447fe35e..0af134a6a 100644 --- a/nb-annotations/pom.xml +++ b/nb-annotations/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults diff --git a/nb-api/pom.xml b/nb-api/pom.xml index e26f8e869..1472b30fe 100644 --- a/nb-api/pom.xml +++ b/nb-api/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -58,7 +58,7 @@ io.nosqlbench nb-annotations - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/nb-spectest/pom.xml b/nb-spectest/pom.xml index 274b325f7..3a6029bbc 100644 --- a/nb-spectest/pom.xml +++ b/nb-spectest/pom.xml @@ -20,7 +20,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults diff --git a/nb5/pom.xml b/nb5/pom.xml index f801d40cd..4fef97492 100644 --- a/nb5/pom.xml +++ b/nb5/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -40,7 +40,7 @@ io.nosqlbench nbr - 4.17.31-SNAPSHOT + 4.17.31 @@ -49,49 +49,49 @@ io.nosqlbench adapter-tcp - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench adapter-mongodb - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench adapter-stdout - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench adapter-diag - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench adapter-dynamodb - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench adapter-cqld4 - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench adapter-http - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench adapter-pulsar - 4.17.31-SNAPSHOT + 4.17.31 @@ -166,7 +166,7 @@ io.nosqlbench adapter-mongodb - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/nbr-examples/pom.xml b/nbr-examples/pom.xml index 6e8e73730..fea18e7b3 100644 --- a/nbr-examples/pom.xml +++ b/nbr-examples/pom.xml @@ -20,7 +20,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -39,13 +39,13 @@ io.nosqlbench nbr - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench adapter-diag - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/nbr/pom.xml b/nbr/pom.xml index 9dca9b6cf..8e2be4939 100644 --- a/nbr/pom.xml +++ b/nbr/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -40,37 +40,37 @@ io.nosqlbench engine-rest - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench engine-cli - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench engine-docs - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench engine-core - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench engine-extensions - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench adapter-diag - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/pom.xml b/pom.xml index 7eac71c2c..5276d67f4 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 mvn-defaults @@ -188,7 +188,7 @@ scm:git:https://github.com/nosqlbench/nosqlbench.git scm:git:git@github.com:nosqlbench/nosqlbench.git - HEAD + nosqlbench-4.17.31 diff --git a/virtdata-api/pom.xml b/virtdata-api/pom.xml index 4124f7e4b..62cb79ef9 100644 --- a/virtdata-api/pom.xml +++ b/virtdata-api/pom.xml @@ -7,7 +7,7 @@ io.nosqlbench mvn-defaults - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -23,14 +23,14 @@ io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 nb-api io.nosqlbench virtdata-lang - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/virtdata-lang/pom.xml b/virtdata-lang/pom.xml index 3fbb276f7..0fd7b2e89 100644 --- a/virtdata-lang/pom.xml +++ b/virtdata-lang/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults diff --git a/virtdata-lib-basics/pom.xml b/virtdata-lib-basics/pom.xml index a84409b13..e1434146b 100644 --- a/virtdata-lib-basics/pom.xml +++ b/virtdata-lib-basics/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -20,7 +20,7 @@ io.nosqlbench virtdata-api - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/virtdata-lib-curves4/pom.xml b/virtdata-lib-curves4/pom.xml index 8dcdd827d..801106ac8 100644 --- a/virtdata-lib-curves4/pom.xml +++ b/virtdata-lib-curves4/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -22,13 +22,13 @@ io.nosqlbench virtdata-api - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench virtdata-lib-basics - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/virtdata-lib-random/pom.xml b/virtdata-lib-random/pom.xml index 3ae330a56..3e1c0c61a 100644 --- a/virtdata-lib-random/pom.xml +++ b/virtdata-lib-random/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -20,13 +20,13 @@ io.nosqlbench virtdata-api - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench virtdata-lib-basics - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/virtdata-lib-realer/pom.xml b/virtdata-lib-realer/pom.xml index c72acb3ad..361fabc0d 100644 --- a/virtdata-lib-realer/pom.xml +++ b/virtdata-lib-realer/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -20,7 +20,7 @@ io.nosqlbench virtdata-lib-basics - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/virtdata-realdata/pom.xml b/virtdata-realdata/pom.xml index d16316af5..dab911084 100644 --- a/virtdata-realdata/pom.xml +++ b/virtdata-realdata/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -18,7 +18,7 @@ io.nosqlbench virtdata-api - 4.17.31-SNAPSHOT + 4.17.31 diff --git a/virtdata-userlibs/pom.xml b/virtdata-userlibs/pom.xml index 9ae56cc89..ccc25bf19 100644 --- a/virtdata-userlibs/pom.xml +++ b/virtdata-userlibs/pom.xml @@ -20,7 +20,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.31 ../mvn-defaults @@ -34,42 +34,42 @@ io.nosqlbench virtdata-realdata - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench virtdata-lib-realer - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench virtdata-api - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench virtdata-lib-random - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench virtdata-lib-basics - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench virtdata-lib-curves4 - 4.17.31-SNAPSHOT + 4.17.31 io.nosqlbench docsys - 4.17.31-SNAPSHOT + 4.17.31 compile From 02522a468e82ee9c4f6a98fc3f391e249d0609d8 Mon Sep 17 00:00:00 2001 From: nb-droid Date: Wed, 7 Dec 2022 23:30:05 +0000 Subject: [PATCH 39/45] [maven-release-plugin] prepare for next development iteration --- adapter-cqld4/pom.xml | 4 ++-- adapter-diag/pom.xml | 6 +++--- adapter-dynamodb/pom.xml | 4 ++-- adapter-http/pom.xml | 4 ++-- adapter-mongodb/pom.xml | 4 ++-- adapter-pulsar/pom.xml | 6 +++--- adapter-stdout/pom.xml | 6 +++--- adapter-tcp/pom.xml | 4 ++-- adapters-api/pom.xml | 8 ++++---- docsys/pom.xml | 6 +++--- engine-api/pom.xml | 12 ++++++------ engine-cli/pom.xml | 6 +++--- engine-clients/pom.xml | 4 ++-- engine-core/pom.xml | 6 +++--- engine-docker/pom.xml | 4 ++-- engine-docs/pom.xml | 4 ++-- engine-extensions/pom.xml | 4 ++-- engine-rest/pom.xml | 4 ++-- mvn-defaults/pom.xml | 4 ++-- nb-annotations/pom.xml | 2 +- nb-api/pom.xml | 4 ++-- nb-spectest/pom.xml | 2 +- nb5/pom.xml | 22 +++++++++++----------- nbr-examples/pom.xml | 6 +++--- nbr/pom.xml | 14 +++++++------- pom.xml | 4 ++-- virtdata-api/pom.xml | 6 +++--- virtdata-lang/pom.xml | 2 +- virtdata-lib-basics/pom.xml | 4 ++-- virtdata-lib-curves4/pom.xml | 6 +++--- virtdata-lib-random/pom.xml | 6 +++--- virtdata-lib-realer/pom.xml | 4 ++-- virtdata-realdata/pom.xml | 4 ++-- virtdata-userlibs/pom.xml | 16 ++++++++-------- 34 files changed, 101 insertions(+), 101 deletions(-) diff --git a/adapter-cqld4/pom.xml b/adapter-cqld4/pom.xml index 9988fb06c..bc0c902af 100644 --- a/adapter-cqld4/pom.xml +++ b/adapter-cqld4/pom.xml @@ -20,7 +20,7 @@ io.nosqlbench mvn-defaults - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -38,7 +38,7 @@ io.nosqlbench adapters-api - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/adapter-diag/pom.xml b/adapter-diag/pom.xml index 0811ac0e5..55c07a912 100644 --- a/adapter-diag/pom.xml +++ b/adapter-diag/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -37,13 +37,13 @@ io.nosqlbench nb-annotations - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench engine-api - 4.17.31 + 4.17.32-SNAPSHOT compile diff --git a/adapter-dynamodb/pom.xml b/adapter-dynamodb/pom.xml index c12b98c50..30c918124 100644 --- a/adapter-dynamodb/pom.xml +++ b/adapter-dynamodb/pom.xml @@ -20,7 +20,7 @@ io.nosqlbench mvn-defaults - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -39,7 +39,7 @@ io.nosqlbench adapters-api - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/adapter-http/pom.xml b/adapter-http/pom.xml index 847bb3835..5db53ca97 100644 --- a/adapter-http/pom.xml +++ b/adapter-http/pom.xml @@ -20,7 +20,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -38,7 +38,7 @@ io.nosqlbench engine-api - 4.17.31 + 4.17.32-SNAPSHOT compile diff --git a/adapter-mongodb/pom.xml b/adapter-mongodb/pom.xml index fb4c143d1..7cb99c906 100644 --- a/adapter-mongodb/pom.xml +++ b/adapter-mongodb/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -36,7 +36,7 @@ io.nosqlbench engine-api - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/adapter-pulsar/pom.xml b/adapter-pulsar/pom.xml index 891fe9d9a..d59350f27 100644 --- a/adapter-pulsar/pom.xml +++ b/adapter-pulsar/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -41,13 +41,13 @@ io.nosqlbench engine-api - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench adapters-api - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/adapter-stdout/pom.xml b/adapter-stdout/pom.xml index 95e747c1c..c094c1509 100644 --- a/adapter-stdout/pom.xml +++ b/adapter-stdout/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -38,14 +38,14 @@ io.nosqlbench nb-annotations - 4.17.31 + 4.17.32-SNAPSHOT compile io.nosqlbench adapters-api - 4.17.31 + 4.17.32-SNAPSHOT compile diff --git a/adapter-tcp/pom.xml b/adapter-tcp/pom.xml index 5c1029def..d75a37401 100644 --- a/adapter-tcp/pom.xml +++ b/adapter-tcp/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -40,7 +40,7 @@ io.nosqlbench engine-api - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/adapters-api/pom.xml b/adapters-api/pom.xml index 39beee7a0..e17269322 100644 --- a/adapters-api/pom.xml +++ b/adapters-api/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -39,19 +39,19 @@ io.nosqlbench nb-spectest - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench nb-api - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench virtdata-userlibs - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/docsys/pom.xml b/docsys/pom.xml index aa8d7a3cc..ade6388b4 100644 --- a/docsys/pom.xml +++ b/docsys/pom.xml @@ -28,7 +28,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -37,7 +37,7 @@ io.nosqlbench nb-api - 4.17.31 + 4.17.32-SNAPSHOT @@ -133,7 +133,7 @@ io.nosqlbench virtdata-api - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/engine-api/pom.xml b/engine-api/pom.xml index ef7547237..cc8b4087f 100644 --- a/engine-api/pom.xml +++ b/engine-api/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -39,31 +39,31 @@ io.nosqlbench nb-api - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench adapters-api - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench nb-spectest - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench nb-annotations - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench virtdata-userlibs - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/engine-cli/pom.xml b/engine-cli/pom.xml index f9e2bf0e4..8936515c5 100644 --- a/engine-cli/pom.xml +++ b/engine-cli/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -23,13 +23,13 @@ io.nosqlbench engine-core - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench engine-docker - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/engine-clients/pom.xml b/engine-clients/pom.xml index 6a3581c4e..7fad28143 100644 --- a/engine-clients/pom.xml +++ b/engine-clients/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -21,7 +21,7 @@ io.nosqlbench engine-api - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/engine-core/pom.xml b/engine-core/pom.xml index 00e2b0077..e21236d2e 100644 --- a/engine-core/pom.xml +++ b/engine-core/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -38,7 +38,7 @@ io.nosqlbench engine-api - 4.17.31 + 4.17.32-SNAPSHOT @@ -89,7 +89,7 @@ io.nosqlbench engine-clients - 4.17.31 + 4.17.32-SNAPSHOT compile diff --git a/engine-docker/pom.xml b/engine-docker/pom.xml index 88d25d2ae..977483fd3 100644 --- a/engine-docker/pom.xml +++ b/engine-docker/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -56,7 +56,7 @@ io.nosqlbench engine-api - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/engine-docs/pom.xml b/engine-docs/pom.xml index 7310fc54d..8b7e603dd 100644 --- a/engine-docs/pom.xml +++ b/engine-docs/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -28,7 +28,7 @@ io.nosqlbench docsys - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/engine-extensions/pom.xml b/engine-extensions/pom.xml index 376fc0f04..74120d0c3 100644 --- a/engine-extensions/pom.xml +++ b/engine-extensions/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -22,7 +22,7 @@ io.nosqlbench engine-api - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/engine-rest/pom.xml b/engine-rest/pom.xml index 0591cdaee..a42221e15 100644 --- a/engine-rest/pom.xml +++ b/engine-rest/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -35,7 +35,7 @@ io.nosqlbench engine-cli - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index d891e7c7f..11f90877d 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -19,7 +19,7 @@ io.nosqlbench mvn-defaults - 4.17.31 + 4.17.32-SNAPSHOT pom @@ -89,7 +89,7 @@ scm:git:https://github.com/nosqlbench/nosqlbench.git - nosqlbench-4.17.31 + HEAD diff --git a/nb-annotations/pom.xml b/nb-annotations/pom.xml index 0af134a6a..d2dc538f8 100644 --- a/nb-annotations/pom.xml +++ b/nb-annotations/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults diff --git a/nb-api/pom.xml b/nb-api/pom.xml index 1472b30fe..07982e077 100644 --- a/nb-api/pom.xml +++ b/nb-api/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -58,7 +58,7 @@ io.nosqlbench nb-annotations - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/nb-spectest/pom.xml b/nb-spectest/pom.xml index 3a6029bbc..55d9a88f6 100644 --- a/nb-spectest/pom.xml +++ b/nb-spectest/pom.xml @@ -20,7 +20,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults diff --git a/nb5/pom.xml b/nb5/pom.xml index 4fef97492..72f370c30 100644 --- a/nb5/pom.xml +++ b/nb5/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -40,7 +40,7 @@ io.nosqlbench nbr - 4.17.31 + 4.17.32-SNAPSHOT @@ -49,49 +49,49 @@ io.nosqlbench adapter-tcp - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench adapter-mongodb - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench adapter-stdout - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench adapter-diag - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench adapter-dynamodb - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench adapter-cqld4 - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench adapter-http - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench adapter-pulsar - 4.17.31 + 4.17.32-SNAPSHOT @@ -166,7 +166,7 @@ io.nosqlbench adapter-mongodb - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/nbr-examples/pom.xml b/nbr-examples/pom.xml index fea18e7b3..5613dc4ad 100644 --- a/nbr-examples/pom.xml +++ b/nbr-examples/pom.xml @@ -20,7 +20,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -39,13 +39,13 @@ io.nosqlbench nbr - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench adapter-diag - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/nbr/pom.xml b/nbr/pom.xml index 8e2be4939..cc59c9caa 100644 --- a/nbr/pom.xml +++ b/nbr/pom.xml @@ -21,7 +21,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -40,37 +40,37 @@ io.nosqlbench engine-rest - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench engine-cli - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench engine-docs - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench engine-core - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench engine-extensions - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench adapter-diag - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/pom.xml b/pom.xml index 5276d67f4..7aab0a099 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT mvn-defaults @@ -188,7 +188,7 @@ scm:git:https://github.com/nosqlbench/nosqlbench.git scm:git:git@github.com:nosqlbench/nosqlbench.git - nosqlbench-4.17.31 + HEAD diff --git a/virtdata-api/pom.xml b/virtdata-api/pom.xml index 62cb79ef9..f3c2ac971 100644 --- a/virtdata-api/pom.xml +++ b/virtdata-api/pom.xml @@ -7,7 +7,7 @@ io.nosqlbench mvn-defaults - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -23,14 +23,14 @@ io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT nb-api io.nosqlbench virtdata-lang - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/virtdata-lang/pom.xml b/virtdata-lang/pom.xml index 0fd7b2e89..f82be7747 100644 --- a/virtdata-lang/pom.xml +++ b/virtdata-lang/pom.xml @@ -23,7 +23,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults diff --git a/virtdata-lib-basics/pom.xml b/virtdata-lib-basics/pom.xml index e1434146b..ca1882850 100644 --- a/virtdata-lib-basics/pom.xml +++ b/virtdata-lib-basics/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -20,7 +20,7 @@ io.nosqlbench virtdata-api - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/virtdata-lib-curves4/pom.xml b/virtdata-lib-curves4/pom.xml index 801106ac8..744a88b0d 100644 --- a/virtdata-lib-curves4/pom.xml +++ b/virtdata-lib-curves4/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -22,13 +22,13 @@ io.nosqlbench virtdata-api - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench virtdata-lib-basics - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/virtdata-lib-random/pom.xml b/virtdata-lib-random/pom.xml index 3e1c0c61a..6e75f9ccb 100644 --- a/virtdata-lib-random/pom.xml +++ b/virtdata-lib-random/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -20,13 +20,13 @@ io.nosqlbench virtdata-api - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench virtdata-lib-basics - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/virtdata-lib-realer/pom.xml b/virtdata-lib-realer/pom.xml index 361fabc0d..83995db65 100644 --- a/virtdata-lib-realer/pom.xml +++ b/virtdata-lib-realer/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -20,7 +20,7 @@ io.nosqlbench virtdata-lib-basics - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/virtdata-realdata/pom.xml b/virtdata-realdata/pom.xml index dab911084..3741c45b6 100644 --- a/virtdata-realdata/pom.xml +++ b/virtdata-realdata/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -18,7 +18,7 @@ io.nosqlbench virtdata-api - 4.17.31 + 4.17.32-SNAPSHOT diff --git a/virtdata-userlibs/pom.xml b/virtdata-userlibs/pom.xml index ccc25bf19..b2d8e38ef 100644 --- a/virtdata-userlibs/pom.xml +++ b/virtdata-userlibs/pom.xml @@ -20,7 +20,7 @@ mvn-defaults io.nosqlbench - 4.17.31 + 4.17.32-SNAPSHOT ../mvn-defaults @@ -34,42 +34,42 @@ io.nosqlbench virtdata-realdata - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench virtdata-lib-realer - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench virtdata-api - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench virtdata-lib-random - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench virtdata-lib-basics - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench virtdata-lib-curves4 - 4.17.31 + 4.17.32-SNAPSHOT io.nosqlbench docsys - 4.17.31 + 4.17.32-SNAPSHOT compile From c102b213a8020f0bd32ab6f18a559a781aa504b5 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 7 Dec 2022 19:16:21 -0600 Subject: [PATCH 40/45] github actions release test --- RELEASENOTES.md | 203 ------------------------------------------------ 1 file changed, 203 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index ff78ef769..e69de29bb 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,203 +0,0 @@ -- d0f25f5c6 (HEAD -> main, origin/main) Merge pull request #831 from nosqlbench/nosqlbench-830 -- 03a3afd86 Merge pull request #828 from nosqlbench/nosqlbench-824 -- 2b63b8e64 (origin/nosqlbench-830, nosqlbench-830) fix camel case across all affected method signatures -- 6fbbfb5d0 (origin/nosqlbench-824, nosqlbench-824) diagram updates -- a1fbb6dcc use average instead of median, add safety check for average==0 -- 15f15ebc9 nosqlbench-828 nosqlbench-824 -- 7737ffc0f nosqlbench-828 nosqlbench-824 -- d90731ec7 typo fix -- b33e93eae Merge pull request #827 from nosqlbench/snyk-upgrade-5c7e9a58ec726ce1a5540b1007997a68 -- 3677a60ec nosqlbench-830 limit spin-looping with cqld4 retryreplace option. -- 97cf9a32c add ad-hoc doc to show publishing flow -- dc7428395 nosqlbench-824 Add function to provide clustered values with monotonic stepping -- eaa8c5671 Merge pull request #826 from nosqlbench/snyk-upgrade-62cc0785172d699541ae5c0e9e487268 -- a1e7eb198 Merge pull request #825 from nosqlbench/snyk-upgrade-32a05328633bd6d7b922681265c560ee -- 14df638f7 Merge pull request #823 from nosqlbench/snyk-upgrade-d560c5ed862bae1f5c8558515d8e4571 -- cde915914 Merge pull request #818 from nosqlbench/snyk-upgrade-e5f0f6d488b383219dd2ca00632f1dde -- b13c899e8 Merge pull request #815 from nosqlbench/javadoc_improvements -- f8623f3b7 (origin/snyk-upgrade-5c7e9a58ec726ce1a5540b1007997a68) fix: upgrade io.swagger.core.v3:swagger-models from 2.2.4 to 2.2.6 -- 76f5507e4 (origin/snyk-upgrade-62cc0785172d699541ae5c0e9e487268) fix: upgrade org.apache.commons:commons-compress from 1.21 to 1.22 -- 150793cda (origin/snyk-upgrade-32a05328633bd6d7b922681265c560ee) fix: upgrade org.glassfish.jersey.media:jersey-media-json-jackson from 3.0.8 to 3.1.0 -- 82bfa4485 (origin/snyk-upgrade-d560c5ed862bae1f5c8558515d8e4571) fix: upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.340 to 1.12.341 -- 2ca4320c2 Merge pull request #819 from nosqlbench/snyk-upgrade-cab2f09370a21c3a871a0379d2ad3792 -- 569d35f2b Merge pull request #820 from nosqlbench/snyk-upgrade-a30d7c35198a17d29ecdca13feba6579 -- 5595f5083 Merge pull request #821 from nosqlbench/snyk-upgrade-f3ed4565eab47a4f529c1dfe1685b618 -- 8eb934c56 Merge pull request #822 from nosqlbench/snyk-upgrade-292f7f0db6ee968fd7ce222173de764c -- 141bfe088 fix: upgrade org.graalvm.js:js-scriptengine from 21.3.3.1 to 21.3.4 -- fd4673f15 fix: upgrade io.dropwizard.metrics:metrics-graphite from 4.2.10 to 4.2.12 -- 3030e58df fix: upgrade org.graalvm.tools:profiler from 20.3.6.1 to 20.3.8 -- 1e6367ecb fix: upgrade org.graalvm.js:js from 21.3.3.1 to 21.3.4 -- 52ef69373 (origin/snyk-upgrade-e5f0f6d488b383219dd2ca00632f1dde) fix: upgrade io.netty:netty-handler from 4.1.84.Final to 4.1.85.Final -- 726028059 (origin/javadoc_improvements, javadoc_improvements) update docs for op types -- b4b09048f update docs for op types -- 98493cfcb Merge pull request #812 from nosqlbench/snyk-upgrade-1754fe16038ae415d7733b7b3fdfb0ad -- 842427e4f Merge pull request #813 from nosqlbench/snyk-upgrade-ca34705c36a67d06ea66413eae078875 -- b06996319 Merge pull request #814 from nosqlbench/snyk-upgrade-a37d7e3729b4fe18258665c47ebd0cf4 -- b3e20c16e Merge pull request #809 from nosqlbench/nosqlbench-808 -- 6ad405d9f fix: upgrade org.glassfish.jersey.core:jersey-server from 3.0.8 to 3.1.0 -- db6594d41 Merge pull request #811 from nosqlbench/snyk-upgrade-8275b1467b7f05117180335fa4d5f944 -- b6cc8fcf0 Merge pull request #810 from nosqlbench/snyk-upgrade-342e4814a0e11c6d2f22b112ba735c12 -- d88359a6a fix: upgrade org.snakeyaml:snakeyaml-engine from 2.4 to 2.5 -- f413a5af7 fix: upgrade org.codehaus.groovy:groovy from 3.0.12 to 3.0.13 -- 34b51b198 fix: upgrade com.datastax.oss:java-driver-query-builder from 4.14.1 to 4.15.0 -- c89339839 fix: upgrade joda-time:joda-time from 2.11.2 to 2.12.1 -- 9bca349e4 (origin/nosqlbench-808, nosqlbench-808) use correct value in exception handlers -- bf518c375 Merge pull request #804 from nosqlbench/snyk-upgrade-4a7ee2762e595acb421d164dc872273b -- b16de9edb Merge pull request #806 from nosqlbench/snyk-fix-d203bc55bd9076b01b05bf8a45a65046 -- 7c8727e15 Merge pull request #805 from nosqlbench/snyk-upgrade-18902492bf09462887646247db53d19b -- 017fcf8ff Merge pull request #803 from nosqlbench/snyk-upgrade-12d61683e1022753a4e624d6d62d5a19 -- 1f96e625d Merge pull request #802 from nosqlbench/snyk-upgrade-cf1da0c80b868193683f0a03748b3835 -- ce353d899 Merge pull request #801 from nosqlbench/snyk-upgrade-399641fb454b01fc3feead152acf52d5 -- 74ae911e5 Merge pull request #800 from nosqlbench/dependabot/maven/driver-cockroachdb/org.postgresql-postgresql-42.5.1 -- 82c253acb (origin/snyk-fix-d203bc55bd9076b01b05bf8a45a65046) fix: driver-cockroachdb/pom.xml to reduce vulnerabilities -- fec156a25 (origin/snyk-upgrade-18902492bf09462887646247db53d19b) fix: upgrade io.netty:netty-handler from 4.1.82.Final to 4.1.84.Final -- d622670f7 (origin/snyk-upgrade-4a7ee2762e595acb421d164dc872273b) fix: upgrade org.graalvm.sdk:graal-sdk from 21.3.3.1 to 21.3.4 -- ec858f69d (origin/snyk-upgrade-12d61683e1022753a4e624d6d62d5a19) fix: upgrade info.picocli:picocli from 4.6.3 to 4.7.0 -- 96be7811a (origin/snyk-upgrade-cf1da0c80b868193683f0a03748b3835) fix: upgrade com.google.code.gson:gson from 2.9.1 to 2.10 -- 64f11bed1 (origin/snyk-upgrade-399641fb454b01fc3feead152acf52d5) fix: upgrade com.github.oshi:oshi-core-java11 from 6.2.2 to 6.3.1 -- 702692058 (origin/dependabot/maven/driver-cockroachdb/org.postgresql-postgresql-42.5.1) build(deps): bump postgresql in /driver-cockroachdb -- f39ba9331 fix: upgrade io.swagger.core.v3:swagger-models from 2.2.3 to 2.2.4 (#799) -- 9afe376b4 fix: upgrade io.swagger.parser.v3:swagger-parser from 2.1.4 to 2.1.7 (#798) -- 79a2d0977 fix: upgrade org.apache.pulsar:pulsar-client from 2.10.1 to 2.10.2 (#796) -- 523f500cc Merge pull request #773 from nosqlbench/snyk-fix-8aa6ab863bb5432def89d1b31da171c7 -- 26ee71883 Merge pull request #772 from nosqlbench/snyk-fix-ab49d52aab0b82f5b5b84c1e7389d4b1 -- 8b87b10ca Merge branch 'main' into snyk-fix-ab49d52aab0b82f5b5b84c1e7389d4b1 -- 00e0596fd Merge pull request #771 from nosqlbench/snyk-fix-ac633ddba1da2c7ec65dff2ae08fa725 -- 1e0ac38b4 Merge branch 'main' into snyk-fix-ac633ddba1da2c7ec65dff2ae08fa725 -- f9adeff1d Merge pull request #774 from nosqlbench/snyk-fix-b2c236df12318590f34b590edea9705e -- 5ffda7d1e Merge branch 'main' into snyk-fix-b2c236df12318590f34b590edea9705e -- e5f03b91a Merge pull request #775 from nosqlbench/snyk-fix-8c31cd4a0b1166676ed0fe1e7cf9a216 -- c7e5e7291 Merge branch 'main' into snyk-fix-8c31cd4a0b1166676ed0fe1e7cf9a216 -- 3ceddb631 Merge pull request #776 from nosqlbench/snyk-fix-6ba0467ba1a5842fab939a7c4f8b6a14 -- e39954ccc Merge branch 'main' into snyk-fix-6ba0467ba1a5842fab939a7c4f8b6a14 -- 56470956b Merge pull request #778 from nosqlbench/snyk-fix-62d2529ba6efa1321216d8e295c48e5d -- 59b430b12 Merge branch 'main' into snyk-fix-62d2529ba6efa1321216d8e295c48e5d -- 4c764250d Merge pull request #794 from nosqlbench/snyk-fix-2ac9869bf1d47aa3f30a82f3655c5432 -- 7491370ba Merge branch 'main' into snyk-fix-2ac9869bf1d47aa3f30a82f3655c5432 -- 47db96c06 Merge pull request #795 from nosqlbench/snyk-fix-3f81768022b768b87ace72058200f7ac -- fa528eebe Merge pull request #779 from nosqlbench/shutdown_hooks -- 875e2d8f9 (origin/shutdown_hooks, shutdown_hooks) temporarily disable test for close exception handler -- 901034875 fix for second noisy shutdown test -- 2db6c6cf6 Update codeql-analysis.yml -- 18ba2fdaf Merge branch 'main' into shutdown_hooks -- 8602d5b53 Merge pull request #793 from yabinmeng/main -- 3b616d7d2 (origin/snyk-fix-3f81768022b768b87ace72058200f7ac) fix: adapter-dynamodb/pom.xml to reduce vulnerabilities -- eaefd9a1b Merge pull request #781 from nosqlbench/780-sporadic-buildtest-failures-gh-actions -- 69617042b (origin/780-sporadic-buildtest-failures-gh-actions, 780-sporadic-buildtest-failures-gh-actions) minor fixes on PR -- 28f561aed direct work-around for flaky test, simple improvements to logging, some improvements to consistent view of status, misc cleanups -- 0193ed355 (origin/snyk-fix-2ac9869bf1d47aa3f30a82f3655c5432) fix: adapter-dynamodb/pom.xml to reduce vulnerabilities -- c077218e8 Fix example yaml file producer and consumer parameter names -- 16417f37b additional diagnostics for spurious test -- f7c9e50f3 Add README file and adjust sample yaml files -- eb0013f16 logging updates for build sanity checking -- bc4773269 build workflow should upload logfiles for success() or failure() of previous jobs, but not when canceled -- 40c968234 Merge branch '780-sporadic-buildtest-failures-gh-actions' of github.com:nosqlbench/nosqlbench into 780-sporadic-buildtest-failures-gh-actions -- dcf7baf86 Dead code utest resurrection and timing w/ gh actions Cyclerate=10 to test github actions part2 Debug and exception handling Detailed error handler logging System out diagnostics Capture step included Try-catch diagnostics. sysout cleanup; general cleanup -- 20ed4950d removing broken actions plugin on main -- 2831d8c3f Merge branch 'main' of github.com:nosqlbench/nosqlbench -- 3e8153dd8 Merge pull request #792 from yabinmeng/main -- b8684107b memoize retries in hot code path -- ebb6f2c58 include space usage in diag for testing -- 48192cbde close native driver connections on on activity+space shutdown -- 4471d90cc implement closeable spaces via decorator interface -- bd8aff908 Added support for 1) subscription initial position and 2) regex subscription mode -- 35eec4d81 Dead code utest resurrection and timing w/ gh actions Cyclerate=10 to test github actions part2 Debug and exception handling Detailed error handler logging System out diagnostics Capture step included Try-catch diagnostics. sysout cleanup; general cleanup -- e80e9926c Merge pull request #791 from yabinmeng/main -- 0de710288 Add support for 1) producer message compression 2) consumer DLT/negAck/ackTimeout policy -- 12f8697e0 Complete the first draft of NB5 Pulsar adapter code, except the per-thread rate limiter. -- e128d2867 Merge branch 'nosqlbench:main' into main -- 0c71696b1 Completed NB4 Pulsar driver code migration to NB5. But there are still a few lingering issues due to the difference of how NB4 vs NB5 driver/adapter interacts with the NB engine. -- 33e330dc3 [Snyk] Security upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.325 to 1.12.330 (#768) -- 2158dfa33 fix: upgrade io.netty:netty-handler from 4.1.81.Final to 4.1.82.Final (#767) -- 5fc22bbaa fix: upgrade com.google.code.gson:gson from 2.9.0 to 2.9.1 (#766) -- 6999a675f [Snyk] Upgrade io.dropwizard.metrics:metrics-core from 4.2.10 to 4.2.12 (#765) -- ca6f2b052 Add PulsarAdapter specific metrics -- 8b76d1fcf add shutdown hooks for adapters and spaces -- 8c3d744a0 Merge pull request #764 from nosqlbench/snyk-upgrade-6693c12bca4fafc634662d8962afcb9a -- eb1de8f1d Timing changes for github actions to properly perform build/test step. -- a05c511de fix: upgrade org.eclipse.jetty:jetty-server from 11.0.11 to 11.0.12 (#763) -- c309d200a - Delete unsed code "driver-jms" - NB5 Pulsar driver Admin functionality in place - create/delete tenant, namespace, and topic -- 2b9452fde Buildfixes, trying to unblock current builds (#777) -- fe268d7f8 (origin/snyk-fix-62d2529ba6efa1321216d8e295c48e5d) fix: nb-api/pom.xml to reduce vulnerabilities -- d47462929 (origin/buildfixes, buildfixes) remove blocking issues github actions plugin, since it is broken -- 4dc830aa1 improve wording around op template resolution -- b2f74d329 make example/test script more robust for meager github actions environment -- cbb01fb7f (origin/snyk-fix-6ba0467ba1a5842fab939a7c4f8b6a14) fix: adapter-dynamodb/pom.xml to reduce vulnerabilities -- e50098800 (origin/snyk-fix-8c31cd4a0b1166676ed0fe1e7cf9a216) fix: nb-api/pom.xml to reduce vulnerabilities -- 422849209 (origin/snyk-fix-b2c236df12318590f34b590edea9705e) fix: adapter-dynamodb/pom.xml to reduce vulnerabilities -- c5b632518 (origin/snyk-fix-8aa6ab863bb5432def89d1b31da171c7) fix: docsys/pom.xml to reduce vulnerabilities -- f8af301fb (origin/snyk-fix-ab49d52aab0b82f5b5b84c1e7389d4b1) fix: nb-api/pom.xml to reduce vulnerabilities -- a5974919a (origin/snyk-fix-ac633ddba1da2c7ec65dff2ae08fa725) fix: adapter-dynamodb/pom.xml to reduce vulnerabilities -- 7c1fb1963 Merge pull request #770 from nosqlbench/ms-mongodb-keyvalue-schema-fix -- 98c7b1a97 Merge pull request #769 from nosqlbench/base64 -- 45ee75fef bump min_version -- 8f32c9496 Updated MongoDB key-value collection to have a 'value' field -- 9a92129a8 (origin/base64) base64 -- bb27a6f7f (origin/snyk-upgrade-6693c12bca4fafc634662d8962afcb9a) fix: upgrade io.swagger.core.v3:swagger-models from 2.2.2 to 2.2.3 -- 0e3893190 (origin/jeffb/nb736) Merge pull request #757 from nosqlbench/snyk-upgrade-de963ecc07996edad35bfd93b9db28c2 -- 92e41e7f7 Merge pull request #758 from nosqlbench/snyk-upgrade-38e72059fc3a8a2099fed05a4e8be9b0 -- 64093b2ab Merge pull request #759 from nosqlbench/snyk-upgrade-5f4f139c9c2fc23d4a2117f5bda8f31e -- 6d25c03ff Merge pull request #760 from nosqlbench/snyk-upgrade-c29375a9a40341a121dfc342ee787201 -- a92897b1d Merge pull request #761 from nosqlbench/snyk-upgrade-c51251ba7d6c428fd60b9aa362f5d4b4 -- 927f4e2c6 fix: upgrade org.eclipse.jetty:jetty-servlets from 11.0.11 to 11.0.12 -- f64a25280 fix: upgrade org.eclipse.jetty:jetty-servlet from 11.0.11 to 11.0.12 -- 74db1d97e fix: upgrade org.eclipse.jetty:jetty-rewrite from 11.0.11 to 11.0.12 -- 02257d41b fix: upgrade org.yaml:snakeyaml from 1.32 to 1.33 -- ed1365773 fix: upgrade joda-time:joda-time from 2.11.1 to 2.11.2 -- dae02bb60 Merge pull request #746 from nosqlbench/snyk-fix-c33fb16683078e999e1d6cab62020656 -- 90bf9dd47 Merge pull request #748 from nosqlbench/snyk-fix-4981c736a854543f414acc756aa6773d -- 2cbb844db Merge pull request #756 from nosqlbench/dependabot/maven/mvn-defaults/org.apache.commons-commons-text-1.10.0 -- 702187794 Merge pull request #744 from nosqlbench/snyk-fix-26f3021e2a4eb5d42b86ca025e907044 -- def46982f Merge branch 'main' into snyk-fix-26f3021e2a4eb5d42b86ca025e907044 -- 2292c6a21 Merge branch 'main' into snyk-fix-4981c736a854543f414acc756aa6773d -- 0021e1cf5 Merge pull request #740 from nosqlbench/snyk-upgrade-e3eae2348ed5c3a9c5a4df3da5434294 -- 2874bdd6a Merge branch 'main' into snyk-fix-c33fb16683078e999e1d6cab62020656 -- 5fb062c1a Merge pull request #749 from nosqlbench/snyk-fix-01ab345df3b2b9725654404c28451b69 -- f684760ee Merge branch 'main' into snyk-fix-01ab345df3b2b9725654404c28451b69 -- 25fe0a541 Merge pull request #739 from nosqlbench/snyk-upgrade-61e7fcecc7f6c453bf8ee04e1799fe04 -- 0cb654259 Merge pull request #738 from nosqlbench/snyk-upgrade-de204ed7275df7f0d05e2783982c6ac7 -- 6bcc1cfd2 build(deps): bump commons-text from 1.9 to 1.10.0 in /mvn-defaults -- 974b29f11 Merge pull request #742 from nosqlbench/snyk-upgrade-dc69205a67c70990049a72696a0c367b -- 88987fb04 Merge pull request #743 from nosqlbench/snyk-upgrade-ac680967c12a71fb0828a5fb85bd8259 -- 879a8778f Merge pull request #745 from nosqlbench/snyk-fix-6accd862435675058d9b3c98aefe4f4a -- af6380f70 Merge pull request #747 from nosqlbench/snyk-fix-fc4ff10e48c0a29879ed09f2c567dd08 -- 47489b075 Merge pull request #750 from nosqlbench/snyk-fix-30fed3e9e7bf54aa1715f6cb0cd3bd93 -- bc1039a11 Merge pull request #751 from nosqlbench/snyk-fix-eeda5403343ff40d314d648afcfdf408 -- 256d7b721 Merge pull request #752 from nosqlbench/snyk-fix-b9ee34aa87226cd6400a08d1f6f78f84 -- eb3e57041 Merge pull request #754 from nosqlbench/snyk-fix-5ff726edd68fbb5f2553166c8f1878d6 -- b478a3766 Merge pull request #755 from nosqlbench/snyk-fix-fec660b2a5471e3538d424b7a8a66438 -- f875f2c7f fix: nb-api/pom.xml to reduce vulnerabilities -- e90580de2 Merge pull request #753 from nosqlbench/ms-trivial-mongodb-workload-fix -- c35ec732e fix: adapter-dynamodb/pom.xml to reduce vulnerabilities -- 60bd26104 Minor update to timeseries clustering index name -- 3496f965c fix: mvn-defaults/pom.xml to reduce vulnerabilities -- e3187bc1e fix: mvn-defaults/pom.xml to reduce vulnerabilities -- 75aa1e45c fix: mvn-defaults/pom.xml to reduce vulnerabilities -- ad369143d fix: adapter-dynamodb/pom.xml to reduce vulnerabilities -- 9b279895e fix: nb-api/pom.xml to reduce vulnerabilities -- 49309872c fix: engine-rest/pom.xml to reduce vulnerabilities -- 35d57fca5 fix: nb-api/pom.xml & mvn-defaults/pom.xml to reduce vulnerabilities -- 5cb5bd5b0 fix: mvn-defaults/pom.xml to reduce vulnerabilities -- 8a3720792 fix: adapter-dynamodb/pom.xml to reduce vulnerabilities -- 7e5bd8f7f fix: upgrade com.datastax.oss:java-driver-core from 4.14.1 to 4.15.0 -- cb3191a1a fix: upgrade com.datastax.oss:pulsar-jms from 2.4.9 to 2.4.11 -- 706880c01 fix: upgrade org.apache.logging.log4j:log4j-core from 2.18.0 to 2.19.0 -- dc4c92eca fix: upgrade org.apache.logging.log4j:log4j-api from 2.18.0 to 2.19.0 -- e7a92d68d fix: upgrade org.apache.logging.log4j:log4j-jcl from 2.18.0 to 2.19.0 -- 87c3904f9 Merge pull request #730 from nosqlbench/snyk-upgrade-497803c521a3e4b5bf1f8320886e4693 -- c769c59af Merge pull request #731 from nosqlbench/snyk-upgrade-4de9e27b335c4ba704d719c6b8208fc9 -- ee4243af7 Merge pull request #732 from nosqlbench/snyk-upgrade-b07f4c6dc36ef0b84aac016e42bc0ac0 -- 59a0b3fbf Merge pull request #733 from nosqlbench/snyk-upgrade-9d3ff80bbe2914cac41eefa7bf016fc4 -- a67e9604b Merge pull request #734 from nosqlbench/snyk-upgrade-6f292d686b89f8516e45eb37ec3e77bb -- 1929bc87c Merge pull request #735 from nosqlbench/snyk-fix-d6c9e33a2e55402294f57e1ce7c62a6e -- ab7487e98 Merge pull request #737 from jeffbanks/jeffb/count-warn-nb561 -- 9bc3a2fd1 Notify user when using error=count vs. counter. -- b1d5411af Updated min_version to be 4.17.30 -- 5abef6db3 Updated min_version to be 4.17.30 -- 84b3b5e25 Updated min_version to be 4.17.30 -- a60c41256 (origin/snyk-fix-d6c9e33a2e55402294f57e1ce7c62a6e) fix: docsys/pom.xml to reduce vulnerabilities -- 35d5c0cee (origin/snyk-upgrade-6f292d686b89f8516e45eb37ec3e77bb) fix: upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.269 to 1.12.300 -- 9d3310413 (origin/snyk-upgrade-9d3ff80bbe2914cac41eefa7bf016fc4) fix: upgrade org.antlr:antlr4-runtime from 4.10.1 to 4.11.1 -- c41b5990d (origin/snyk-upgrade-b07f4c6dc36ef0b84aac016e42bc0ac0) fix: upgrade com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider from 2.13.3 to 2.13.4 -- e0fb11da4 (origin/snyk-upgrade-4de9e27b335c4ba704d719c6b8208fc9) fix: upgrade io.netty:netty-handler from 4.1.80.Final to 4.1.81.Final -- c6b327161 (origin/snyk-upgrade-497803c521a3e4b5bf1f8320886e4693) fix: upgrade com.amazonaws:aws-java-sdk-dynamodb from 1.12.262 to 1.12.299 From c08f8b6b3df0b2efe644c507dded2a48edd1897a Mon Sep 17 00:00:00 2001 From: yabinmeng Date: Wed, 7 Dec 2022 22:14:03 -0600 Subject: [PATCH 41/45] 1) Add NB5 S4J example yaml files 2) Update NB5 S4J readme file 3) Minor code adjustments --- adapter-s4j/pom.xml | 6 +- .../io/nosqlbench/adapter/s4j/S4JSpace.java | 94 ++++++- .../MessageConsumerOpDispenser.java | 21 +- .../MessageProducerOpDispenser.java | 2 +- .../s4j/dispensers/S4JBaseOpDispenser.java | 64 +---- .../adapter/s4j/ops/MessageConsumerOp.java | 2 +- .../adapter/s4j/ops/MessageProducerOp.java | 2 +- .../adapter/s4j/util/S4JAdapterUtil.java | 34 ++- .../main/resources/pulsar_s4j_consumer.yaml | 73 ++++++ .../main/resources/pulsar_s4j_producer.yaml | 54 ++++ adapter-s4j/src/main/resources/s4j.md | 244 ++++++++++++++++++ .../src/main/resources/s4j_config.properties | 61 +++++ nb5/pom.xml | 2 +- 13 files changed, 569 insertions(+), 90 deletions(-) create mode 100644 adapter-s4j/src/main/resources/pulsar_s4j_consumer.yaml create mode 100644 adapter-s4j/src/main/resources/pulsar_s4j_producer.yaml create mode 100644 adapter-s4j/src/main/resources/s4j.md create mode 100644 adapter-s4j/src/main/resources/s4j_config.properties diff --git a/adapter-s4j/pom.xml b/adapter-s4j/pom.xml index 41bb02db6..72e79c647 100644 --- a/adapter-s4j/pom.xml +++ b/adapter-s4j/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.17.31-SNAPSHOT + 4.17.32-SNAPSHOT ../mvn-defaults @@ -39,13 +39,13 @@ io.nosqlbench engine-api - 4.17.31-SNAPSHOT + 4.17.32-SNAPSHOT io.nosqlbench adapters-api - 4.17.31-SNAPSHOT + 4.17.32-SNAPSHOT diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JSpace.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JSpace.java index 8c552912f..f55196bf1 100644 --- a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JSpace.java +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JSpace.java @@ -17,6 +17,7 @@ package io.nosqlbench.adapter.s4j; import com.datastax.oss.pulsar.jms.PulsarConnectionFactory; +import com.datastax.oss.pulsar.jms.PulsarJMSContext; import io.nosqlbench.adapter.s4j.exception.S4JAdapterInvalidParamException; import io.nosqlbench.adapter.s4j.exception.S4JAdapterUnexpectedException; import io.nosqlbench.adapter.s4j.util.*; @@ -24,8 +25,10 @@ import io.nosqlbench.api.config.standard.ConfigModel; import io.nosqlbench.api.config.standard.NBConfigModel; import io.nosqlbench.api.config.standard.NBConfiguration; import io.nosqlbench.api.config.standard.Param; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -98,13 +101,19 @@ public class S4JSpace implements AutoCloseable { this.pulsarSvcUrl = cfg.get("service_url"); this.webSvcUrl = cfg.get("web_url"); - this.maxNumConn= cfg.getOrDefault("num_conn", Integer.valueOf(1)); - this.maxNumSessionPerConn = cfg.getOrDefault("num_session", Integer.valueOf(1)); - this.maxS4JOpTimeInSec= cfg.getOrDefault("max_s4jop_time", Long.valueOf(0)); - this.trackingMsgRecvCnt=cfg.getOrDefault("track_msg_cnt", Boolean.FALSE); - this.strictMsgErrorHandling = cfg.getOrDefault("strict_msg_error_handling", Boolean.FALSE); + this.maxNumConn= + NumberUtils.toInt(cfg.getOptional("num_conn").orElse("1")); + this.maxNumSessionPerConn = + NumberUtils.toInt(cfg.getOptional("num_session").orElse("1")); + this.maxS4JOpTimeInSec = + NumberUtils.toLong(cfg.getOptional("max_s4jop_time").orElse("0L")); + this.trackingMsgRecvCnt = + BooleanUtils.toBoolean(cfg.getOptional("track_msg_cnt").orElse("false")); + this.strictMsgErrorHandling = + BooleanUtils.toBoolean(cfg.getOptional("strict_msg_error_handling").orElse("false")); this.s4jClientConfFileName = cfg.get("config"); - this.sessionMode = S4JAdapterUtil.getSessionModeFromStr(cfg.get("session_mode")); + this.sessionMode = S4JAdapterUtil.getSessionModeFromStr( + cfg.getOptional("session_mode").orElse("")); this.s4JClientConf = new S4JClientConf(pulsarSvcUrl, webSvcUrl, s4jClientConfFileName); this.initializeSpace(s4JClientConf); @@ -340,7 +349,17 @@ public class S4JSpace implements AutoCloseable { S4JClientConf s4JClientConf, int sessionMode) { - boolean useCredentialsEnable = S4JAdapterUtil.isUseCredentialsEnabled(s4JClientConf); + if ( !S4JAdapterUtil.isAuthNRequired(s4JClientConf) && + S4JAdapterUtil.isUseCredentialsEnabled(s4JClientConf) ) { + throw new S4JAdapterInvalidParamException( + "'jms.useCredentialsFromCreateConnection' can't set be true " + + "when Pulsar client authN parameters are not set. " + ); + } + + boolean useCredentialsEnable = + S4JAdapterUtil.isAuthNRequired(s4JClientConf) && + S4JAdapterUtil.isUseCredentialsEnabled(s4JClientConf); JMSContext jmsConnContext; if (!useCredentialsEnable) @@ -361,4 +380,65 @@ public class S4JSpace implements AutoCloseable { return jmsConnContext; } + + public S4JJMSContextWrapper getOrCreateS4jJmsContextWrapper(long curCycle) { + return getOrCreateS4jJmsContextWrapper(curCycle, null); + } + + // Get the next JMSContext Wrapper in the following approach + // - The JMSContext wrapper pool has the following sequence (assuming 3 [c]onnections and 2 [s]essions per connection): + // c0s0, c0s1, c1s0, c1s1, c2s0, c2s1 + // - When getting the next JMSContext wrapper, always get from the next connection, starting from the first session + // When reaching the end of connection, move back to the first connection, but get the next session. + // e.g. first: c0s0 (0) + // next: c1s0 (1) + // next: c2s0 (2) + // next: c0s1 (3) + // next: c1s1 (4) + // next: c2s1 (5) + // next: c0s0 (6) <-- repeat the pattern + // next: c1s0 (7) + // next: c2s0 (8) + // next: c0s1 (9) + // ... ... + public S4JJMSContextWrapper getOrCreateS4jJmsContextWrapper( + long curCycle, + Map overrideS4jConfMap) + { + int totalConnNum = getMaxNumConn(); + int totalSessionPerConnNum = getMaxNumSessionPerConn(); + + int connSeqNum = (int) curCycle % totalConnNum; + int sessionSeqNum = ( (int)(curCycle / totalConnNum) ) % totalSessionPerConnNum; + + String jmsConnContextIdStr = getConnLvlJmsContextIdentifier(connSeqNum); + JMSContext connLvlJmsContext = connLvlJmsContexts.get(jmsConnContextIdStr); + // Connection level JMSContext objects should be already created during the initialization phase + assert (connLvlJmsContext != null); + + String jmsSessionContextIdStr = getSessionLvlJmsContextIdentifier(connSeqNum, sessionSeqNum); + S4JJMSContextWrapper jmsContextWrapper = sessionLvlJmsContexts.get(jmsSessionContextIdStr); + + if (jmsContextWrapper == null) { + JMSContext jmsContext = null; + + if (overrideS4jConfMap == null || overrideS4jConfMap.isEmpty()) { + jmsContext = connLvlJmsContext.createContext(connLvlJmsContext.getSessionMode()); + } else { + jmsContext = ((PulsarJMSContext) connLvlJmsContext).createContext( + connLvlJmsContext.getSessionMode(), overrideS4jConfMap); + } + + jmsContextWrapper = new S4JJMSContextWrapper(jmsSessionContextIdStr, jmsContext); + sessionLvlJmsContexts.put(jmsSessionContextIdStr, jmsContextWrapper); + + if (logger.isDebugEnabled()) { + logger.debug("[Session level JMSContext] {} -- {}", + Thread.currentThread().getName(), + jmsContextWrapper); + } + + } + return jmsContextWrapper; + } } diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageConsumerOpDispenser.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageConsumerOpDispenser.java index b71e9ad4a..ad7920686 100644 --- a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageConsumerOpDispenser.java +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageConsumerOpDispenser.java @@ -63,7 +63,7 @@ public class MessageConsumerOpDispenser extends S4JBaseOpDispenser { // which can be applied to many testing scenarios. // Setting them here will allow scenario-specific customer configurations. At the moment, only the // DLT related settings are supported - private final Map combinedConsumerConfigObjMap = new HashMap<>(); + private final Map combinedS4jConfigObjMap = new HashMap<>(); public MessageConsumerOpDispenser(DriverAdapter adapter, @@ -88,11 +88,22 @@ public class MessageConsumerOpDispenser extends S4JBaseOpDispenser { parsedOp.getStaticConfigOr("msg_ack_ratio", Float.valueOf(1.0f)); this.slowAckInSec = parsedOp.getStaticConfigOr("slow_ack_in_sec", Integer.valueOf(0)); - this.subNameStrFunc = - lookupMandtoryStrOpValueFunc("subscription_name"); this.localMsgSelectorFunc = lookupOptionalStrOpValueFunc("msg_selector"); + // Subscription name is OPTIONAL for queue and non-shared, non-durable topic; + // but mandatory for shared or shared topic + if ( StringUtils.equalsIgnoreCase(destType, S4JAdapterUtil.JMS_DEST_TYPES.QUEUE.label) || + ( StringUtils.equalsIgnoreCase(destType, S4JAdapterUtil.JMS_DEST_TYPES.TOPIC.label) && + !durableTopic && !sharedTopic) ) { + this.subNameStrFunc = + lookupOptionalStrOpValueFunc("subscription_name"); + } + else { + this.subNameStrFunc = + lookupMandtoryStrOpValueFunc("subscription_name"); + } + String[] stmtLvlConsumerConfKeyNameList = { "consumer.ackTimeoutMillis", "consumer.deadLetterPolicy", @@ -106,14 +117,14 @@ public class MessageConsumerOpDispenser extends S4JBaseOpDispenser { confVal); } - this.combinedConsumerConfigObjMap.putAll( + this.combinedS4jConfigObjMap.putAll( s4jSpace.getS4JClientConf().mergeExtraConsumerConfig(stmtLvlConsumerConfRawMap)); } @Override public MessageConsumerOp apply(long cycle) { S4JJMSContextWrapper s4JJMSContextWrapper = - getOrCreateS4jJmsContextWrapper(cycle, this.combinedConsumerConfigObjMap); + s4jSpace.getOrCreateS4jJmsContextWrapper(cycle, this.combinedS4jConfigObjMap); JMSContext jmsContext = s4JJMSContextWrapper.getJmsContext(); boolean commitTransact = !super.commitTransaction(txnBatchNum, jmsContext.getSessionMode(), cycle); diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageProducerOpDispenser.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageProducerOpDispenser.java index 8c2e5e1b7..b57c36e80 100644 --- a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageProducerOpDispenser.java +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/MessageProducerOpDispenser.java @@ -279,7 +279,7 @@ public class MessageProducerOpDispenser extends S4JBaseOpDispenser { throw new S4JAdapterInvalidParamException("Message payload must be specified and can't be empty!"); } - S4JJMSContextWrapper s4JJMSContextWrapper = getOrCreateS4jJmsContextWrapper(cycle); + S4JJMSContextWrapper s4JJMSContextWrapper = s4jSpace.getOrCreateS4jJmsContextWrapper(cycle); JMSContext jmsContext = s4JJMSContextWrapper.getJmsContext(); boolean commitTransaction = !super.commitTransaction(txnBatchNum, jmsContext.getSessionMode(), cycle); diff --git a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/S4JBaseOpDispenser.java b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/S4JBaseOpDispenser.java index f49fda8aa..d09f42c61 100644 --- a/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/S4JBaseOpDispenser.java +++ b/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/dispensers/S4JBaseOpDispenser.java @@ -167,67 +167,6 @@ public abstract class S4JBaseOpDispenser extends BaseOpDispenser overrideS4jConfMap) - { - int totalConnNum = s4jSpace.getMaxNumConn(); - int totalSessionPerConnNum = s4jSpace.getMaxNumSessionPerConn(); - - int connSeqNum = (int) curCycle % totalConnNum; - int sessionSeqNum = ( (int)(curCycle / totalConnNum) ) % totalSessionPerConnNum; - - String jmsConnContextIdStr = s4jSpace.getConnLvlJmsContextIdentifier(connSeqNum); - JMSContext connLvlJmsContext = connLvlJmsContexts.get(jmsConnContextIdStr); - // Connection level JMSContext objects should be already created during the initialization phase - assert (connLvlJmsContext != null); - - String jmsSessionContextIdStr = s4jSpace.getSessionLvlJmsContextIdentifier(connSeqNum, sessionSeqNum); - S4JJMSContextWrapper jmsContextWrapper = sessionLvlJmsContexts.get(jmsSessionContextIdStr); - - if (jmsContextWrapper == null) { - JMSContext jmsContext = null; - - if (overrideS4jConfMap == null || overrideS4jConfMap.isEmpty()) { - jmsContext = connLvlJmsContext.createContext(connLvlJmsContext.getSessionMode()); - } else { - jmsContext = ((PulsarJMSContext) connLvlJmsContext).createContext( - connLvlJmsContext.getSessionMode(), overrideS4jConfMap); - } - - jmsContextWrapper = new S4JJMSContextWrapper(jmsSessionContextIdStr, jmsContext); - sessionLvlJmsContexts.put(jmsSessionContextIdStr, jmsContextWrapper); - - if (logger.isDebugEnabled()) { - logger.debug("[Session level JMSContext] {} -- {}", - Thread.currentThread().getName(), - jmsContextWrapper); - } - - } - return jmsContextWrapper; - } - - public S4JJMSContextWrapper getOrCreateS4jJmsContextWrapper(long curCycle) { - return getOrCreateS4jJmsContextWrapper(curCycle, null); - } - /** * If the JMS destination that corresponds to a topic exists, reuse it; Otherwise, create it */ @@ -340,7 +279,8 @@ public abstract class S4JBaseOpDispenser extends BaseOpDispenser s4jClientConfObjMap = s4jClientConf.getS4jConfMapObj_client(); + + if (s4jClientConfObjMap.containsKey("authPlugin") && s4jClientConfObjMap.containsKey("authParams")) { + Object authPluginObj = s4jClientConfObjMap.get("authPlugin"); + Object authParamsObj = s4jClientConfObjMap.get("authParams"); + + if ( (authPluginObj != null) && StringUtils.isNotBlank(authPluginObj.toString()) && + (authParamsObj != null) && StringUtils.isNotBlank(authParamsObj.toString()) ) { + required = true; + } + } + return required; + } + + + public static boolean isUseCredentialsEnabled(S4JClientConf s4jClientConf) { + assert (s4jClientConf != null); boolean enabled = false; - Map s4jConfMap = s4JClientConf.getS4jConfObjMap(); + Map s4jConfMap = s4jClientConf.getS4jConfObjMap(); if (s4jConfMap.containsKey("jms.useCredentialsFromCreateConnection")) { enabled = BooleanUtils.toBoolean(s4jConfMap.get("jms.useCredentialsFromCreateConnection").toString()); @@ -278,12 +294,12 @@ public class S4JAdapterUtil { return enabled; } - public static String getCredentialUserName(S4JClientConf s4JClientConf) { + public static String getCredentialUserName(S4JClientConf s4jClientConf) { return "dummy"; } - public static String getCredentialPassword(S4JClientConf s4JClientConf) { - Map s4jConfMap = s4JClientConf.getS4jConfObjMap(); + public static String getCredentialPassword(S4JClientConf s4jClientConf) { + Map s4jConfMap = s4jClientConf.getS4jConfObjMap(); if (s4jConfMap.containsKey("authParams")) return s4jConfMap.get("authParams").toString(); else diff --git a/adapter-s4j/src/main/resources/pulsar_s4j_consumer.yaml b/adapter-s4j/src/main/resources/pulsar_s4j_consumer.yaml new file mode 100644 index 000000000..3313abe25 --- /dev/null +++ b/adapter-s4j/src/main/resources/pulsar_s4j_consumer.yaml @@ -0,0 +1,73 @@ +# document level parameters that apply to all Pulsar client types: +params: + temporary_dest: "false" + dest_type: "topic" + + # default: true + async_api: "false" + # whether to wait indefinitely (as the default behavior) + # - only applies when "async_api" is false (synchronous API) + # - only applies to message receiving + # - default: false + blocking_msg_recv: "true" + + ## (Optional) If shared topic or not (only relevant when the destination is a topic) + share_topic: "true" + + ## (Optional) If durable topic or not (only relevant when the destination is a topic) + durable_topic: "false" + +blocks: + msg-consume-block: + ops: + op1: + ## The value represents the destination (queue or topic) name) + MessageConsume: "mys4jtest_t" + + ## Subscription name + ## - optional for queue and non-shared, non-durable topic + ## - mandatory for shared and/or durable topic + subscription_name: "nbs4j-sub" + + ## (Optional) client side message selector + msg_selector: "" + + ## (Optional) No Local + no_local: "true" + + ## (Optional) Read Timeout + read_timeout: "10" + + ## (Optional) Receive message without wait + no_wait: "true" + + ## (Optional) Message acknowledgement ratio + msg_ack_ratio: "0.5" + + ## (Optional) Simulate slow consumer acknowledgement + # must be non-negative numbers. negative numbers will be treated as 0 + # 0 - means no simulation + # positive value - the number of seconds to pause before acknowledgement + slow_ack_in_sec: "0" + + ##### + ## (Optional) Statement level settings for Consumer + # + ## AckTimeout value (at least 1 second) + consumer.ackTimeoutMillis: 1000 + + ## DLQ policy + consumer.deadLetterPolicy: '{ "maxRedeliverCount": "2" }' + + ## NegativeAck Redelivery policy + consumer.negativeAckRedeliveryBackoff: | + { + } + + ## AckTimeout Redelivery policy + consumer.ackTimeoutRedeliveryBackoff: | + { + "minDelayMs":"10", + "maxDelayMs":"20", + "multiplier":"1.2" + } diff --git a/adapter-s4j/src/main/resources/pulsar_s4j_producer.yaml b/adapter-s4j/src/main/resources/pulsar_s4j_producer.yaml new file mode 100644 index 000000000..6a0528574 --- /dev/null +++ b/adapter-s4j/src/main/resources/pulsar_s4j_producer.yaml @@ -0,0 +1,54 @@ +bindings: + cur_cycle: ToString() + mykey: Mod(5); ToString(); Prefix("key-") + mytext_val: AlphaNumericString(30) + mymap_val1: AlphaNumericString(10) + mymap_val2: AlphaNumericString(20) + mystream_val1: AlphaNumericString(50) + +# document level parameters that apply to all Pulsar client types: +params: + temporary_dest: "false" + dest_type: "queue" + async_api: "true" + +blocks: + msg-produce-block: + ops: + op1: + ## The value represents the destination (queue or topic) name) + MessageProduce: "mys4jtest_t" + + ## (Optional) JMS headers (in JSON format). + msg_header: | + { + "JMSPriority": "9" + } + + ## (Optional) JMS properties, predefined or customized (in JSON format). + msg_property: | + { + "JMSXGroupID": "{mykey}" + } + + ## (Optional) JMS message types, default to be BYTES. + msg_type: "text" + + ## (Mandatory) JMS message body. Value depends on msg_type. + msg_body: "{mytext_val}" + +# # example of having "map" as the message type +# msg_type: "map" +# msg_body: | +# { +# "prop-key-1": "{mymap_val1}", +# "prop-key-2": "{mymap_val2}" +# } +# # example of having "stream" as the message type +# msg_type: "stream" +# msg_body: | +# [ +# "100", +# "{mystream_val1}", +# "abcdef" +# ] diff --git a/adapter-s4j/src/main/resources/s4j.md b/adapter-s4j/src/main/resources/s4j.md new file mode 100644 index 000000000..9b35b8e71 --- /dev/null +++ b/adapter-s4j/src/main/resources/s4j.md @@ -0,0 +1,244 @@ +- [1. Overview](#1-overview) +- [2. Execute NB S4J Workload](#2-execute-nb-s4j-workload) +- [3. NB S4J Driver Configuration Parameter File](#3-nb-s4j-driver-configuration-parameter-file) +- [4. NB S4J Scenario Definition File](#4-nb-s4j-scenario-definition-file) + - [4.1. Document Level Parameters](#41-document-level-parameters) + - [4.2. NB S4J Workload Types](#42-nb-s4j-workload-types) + - [4.2.1. Publish Messages to a JMS Destination, Queue or Topic](#421-publish-messages-to-a-jms-destination-queue-or-topic) + - [4.2.2. Receiving Messages from a JMS Destination, Queue or Topic](#422-receiving-messages-from-a-jms-destination-queue-or-topic) + + +# 1. Overview + +This driver is similar to [NB Pulsar driver](../../../../driver-pulsar/src/main/resources/pulsar.md) that allows NB based workload generation and performance testing against a Pulsar cluster. It also follows a similar pattern to configure and connect to the Pulsar cluster for workload execution. + +However, the major difference is instead of simulating native Pulsar client workloads, the NB S4J driver allows simulating JMS oriented workloads (that follows JMS spec 2.0 and 1.1) to be executed on the Pulsar cluster. Under the hood, this is achieved through DataStax's [Starlight for JMS API] (https://github.com/datastax/pulsar-jms). + +# 2. Execute NB S4J Workload + +The following is an example of executing a NB S4J workload (defined as *pulsar_s4j.yaml*) + +``` +$ run driver=s4j cycles=10000 threads=4 num_conn=2 num_session=2 session_mode="client_ack" strict_msg_error_handling="false" web_url=http://localhost:8080 service_url=pulsar://localhost:6650 config=/path/to/nb_s4j_config.properties yaml=/path/to/pulsar_s4j.yaml -vv --logs-dir=s4j_log +``` + +In the above NB CLI command, the S4J driver specific parameters are listed as below: +* num_conn: the number of JMS connections to be created +* num_session: the number of JMS sessions per JMS connection + * Note that multiple JMS sessions can be created from one JMS connection, and they share the same connection characteristics. +* session_mode: the session mode used when creating a JMS session +* web_url: the URL of the Pulsar web service +* service_url: the URL of the Pulsar native protocol service +* (optional) strict_msg_error_handling: whether to do strict error handling + * when true, Pulsar client error will not stop NB S4J execution + * otherwise, any Pulsar client error will stop NB S4J execution +* (optional) max_s4jop_time: maximum time (in seconds) to execute the actual S4J operations (e.g. message sending or receiving). If NB execution time is beyond this limit, each NB cycle is just a no-op. Please NOTE: + * this is useful when controlled NB execution is needed with NB CLI scripting. + * if this parameter is not specified or the value is 0, it means no time limitation. Every single NB cycle will trigger an actual S4J operation. +* (optional) track_msg_cnt: When set to true (with default as false), the S4J driver will keep track of the confirmed response count for message sending and receiving. + +Other NB engine parameters are straight forward: +* driver: must be **s4j** +* threads: depending on the workload type, the NB thread number determines how many producers or consumers will be created. All producers or consumers will share the available JMS connections and sessions +* yamL: the NB S4J scenario definition yaml file +* config: specify the file that contains the connection parameters used by the S4J API + +# 3. NB S4J Driver Configuration Parameter File + +The S4J API has a list of configuration options that can be found here: https://docs.datastax.com/en/fast-pulsar-jms/docs/1.1/pulsar-jms-reference.html#_configuration_options. + +The NB S4J driver supports these configuration options via a config property file, an example of which is listed below. The configuration parameters in this file are grouped into several groups. The comments below explain how the grouping works. + +``` +########### +# Overview: Starlight for JMS (S4J) API configuration items are listed at: +# https://docs.datastax.com/en/fast-pulsar-jms/docs/1.1/pulsar-jms-reference.html#_configuration_options +enableTransaction=true + +#### +# S4J API specific configurations (non Pulsar specific) - jms.*** + +jms.enableClientSideEmulation=true +jms.usePulsarAdmin=false +#... + +##### +# Pulsar client related configurations - client.*** +# - Valid settings: http://pulsar.apache.org/docs/en/client-libraries-java/#client +# +# - These Pulsar client settings (without the "client." prefix) will be +# directly used as S4J configuration settings, on a 1-to-1 basis. +#-------------------------------------- +# only relevant when authentication is enabled +client.authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationToken +client.authParams=file:///path/to/authentication/jwt/file +# only relevant when in-transit encryption is enabled +client.tlsTrustCertsFilePath=/path/to/certificate/file +#... + +##### +# Producer related configurations (global) - producer.*** +# - Valid settings: http://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer +# +# - These Pulsar producer settings (without "producer." prefix) will be collectively (as a map) +# mapped to S4J connection setting of "producerConfig" +#-------------------------------------- +producer.blockIfQueueFull=true +# disable producer batching +#producer.batchingEnabled=false +#... + +##### +# Consumer related configurations (global) - consumer.*** +# - Valid settings: http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer +# +# - These Pulsar producer settings (without "consumer." portion) will be collectively (as a map) +# mapped to S4J connection setting of "consumerConfig" +#-------------------------------------- +#... +``` + +# 4. NB S4J Scenario Definition File + +Like any NB scenario yaml file, the NB S4J yaml file is composed of 3 major components: +* bindings: define NB bindings +* params: define document level parameters +* blocks: define various statement blocks. Each statement block represents one JMS workload type + +``` +bindings: + ... ... +params: + ... ... +blocks: + ... ... +``` + +## 4.1. Document Level Parameters + +The parameters defined in this section will be applicable to all statement blocks. An example of some common parameters that can be set at the document level is listed below: +* temporary_dest: whether JMS workload is dealing with a temporary destination +* dest_type: JMS destination type - queue or topic + +``` +params: + temporary_dest: "false" + dest_type: "" + async_api: "true" + txn_batch_num: + blocking_msg_recv: + shared_topic: // only relevant when the destination type is a topic + durable_topic: // only relevant when the destination type is a topic +``` + +Please **NOTE** that the above parameters won't necessarily be specified at the document level. If they're specified at the statement level, they will only impact the statement within which they're specified. + +## 4.2. NB S4J Workload Types + +The NB S4J driver supports 2 types of JMS operations: +* One for message producing/sending/publishing + * this is identified by NB Op identifier ***MessageProduce*** +* One for message consuming/receiving/subscribing + * this is identified by NB Op identifier ***MessageConsume*** + +### 4.2.1. Publish Messages to a JMS Destination, Queue or Topic + +The NB S4J statement block for publishing messages to a JMS destination (either a Queue or a topic) has the following format. +* Optionally, you can specify the JMS headers (**msg_header**) and properties (**msg_property**) via valid JSON strings in key: value format. +* The default message type (**msg_type**) is "byte". But optionally, you can specify other message types such as "text", "map", etc. +* The message payload (**msg_body**) is the only mandatory field. + +``` +blocks: + msg-produce-block: + ops: + op1: + ## The value represents the destination (queue or topic) name) + MessageProduce: "mys4jtest_t" + + ## (Optional) JMS headers (in JSON format). + msg_header: | + { + "": "" + } + + ## (Optional) JMS properties, predefined or customized (in JSON format). + msg_property: | + { + "": "", + "": "" + } + + ## (Optional) JMS message types, default to be BYTES. + msg_type: "text" + + ## (Mandatory) JMS message body. Value depends on msg_type. + msg_body: "{mytext_val}" +``` + +### 4.2.2. Receiving Messages from a JMS Destination, Queue or Topic + +The generic NB S4J statement block for receiving messages to a JMS destination (either a Queue or a topic) has the following format. All the statement specific parameters are listed as below. +* **msg_selector**: Message selector string +* **no_local**: Only applicable to a Topic as the destination. This allows a subscriber to inhibit the delivery of messages published by its own connection. +* **read_timeout**: The timeout value for receiving a message from a destination + * This setting only works if **no_wait** is false + * If the **read_timeout** value is 0, it behaves the same as **no_wait** is true +* **no_wait**: Whether to receive the next message immediately if one is available +* **msg_ack_ratio**: the ratio of the received messages being acknowledged +* **slow_ack_in_sec**: whether to simulate a slow consumer (pause before acknowledging after receiving a message) + * value 0 means no simulation (consumer acknowledges right away) +* negative ack/ack timeout/deadletter topic related settings + * The settings here (as the scenario specific settings) will be merged with the + * global settings in *s4j_config.properties* file +``` +blocks: + msg-produce-block: + ops: + op1: + ## The value represents the destination (queue or topic) name) + MessageProduce: "mys4jtest_t" + + ## (Optional) client side message selector + msg_selector: "" + + ## (Optional) No Local + no_local: "true" + + ## (Optional) Read Timeout + read_timeout: "10" + + ## (Optional) Receive message without wait + no_wait: "true" + + ## (Optional) Message acknowledgement ratio + msg_ack_ratio: "0.5" + + ## (Optional) Simulate slow consumer acknowledgement + # must be non-negative numbers. negative numbers will be treated as 0 + # 0 - means no simulation + # positive value - the number of seconds to pause before acknowledgement + slow_ack_in_sec: "0" + + ##### + ## (Optional) Statement level settings for Consumer + # + ## AckTimeout value (at least 1 second) + consumer.ackTimeoutMillis: 1000 + + ## DLQ policy + consumer.deadLetterPolicy: '{ "maxRedeliverCount": "2" }' + + ## NegativeAck Redelivery policy + consumer.negativeAckRedeliveryBackoff: | + { + } + + ## AckTimeout Redelivery policy + consumer.ackTimeoutRedeliveryBackoff: | + { + "minDelayMs":"10", + "maxDelayMs":"20", + "multiplier":"1.2" + } +``` diff --git a/adapter-s4j/src/main/resources/s4j_config.properties b/adapter-s4j/src/main/resources/s4j_config.properties new file mode 100644 index 000000000..33bc65c11 --- /dev/null +++ b/adapter-s4j/src/main/resources/s4j_config.properties @@ -0,0 +1,61 @@ +########### +# Overview: Starlight for JMS (S4J) API configuration items are listed at: +# https://docs.datastax.com/en/fast-pulsar-jms/docs/1.1/pulsar-jms-reference.html#_configuration_options +enableTransaction=true + +#### +# S4J API specific configurations (non Pulsar specific) - jms.*** +#-------------------------------------- +jms.usePulsarAdmin=false +jms.precreateQueueSubscription=false +jms.enableClientSideEmulation=false +jms.useServerSideFiltering=true +jms.useCredentialsFromCreateConnection=false +jms.transactionsStickyPartitions=true +# for JMS priority +jms.enableJMSPriority=true +jms.priorityMapping=non-linear +#... + + +##### +# Pulsar client related configurations - client.*** +# - Valid settings: http://pulsar.apache.org/docs/en/client-libraries-java/#client +# +# - These Pulsar client settings (without the "client." prefix) will be +# directly used as S4J configuration settings, on 1-to-1 basis. +#-------------------------------------- +client.connectionTimeoutMs=5000 +#client.authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationToken +#client.authParams= +#... + + +##### +# Producer related configurations (global) - producer.*** +# - Valid settings: http://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer +# +# - These Pulsar producer settings (without "producer." prefix) will be collectively (as a map) +# mapped to S4J connection setting of "producerConfig" +#-------------------------------------- +#producer.sendTimeoutMs= +producer.blockIfQueueFull=true +#producer.maxPendingMessages=10000 +#producer.batchingMaxMessages=10000 +#... + + +##### +# Consumer related configurations (global) - consumer.*** +# - Valid settings: http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer +# +# - These Pulsar producer settings (without "consumer." portion) will be collectively (as a map) +# mapped to S4J connection setting of "consumerConfig" +#-------------------------------------- +consumer.receiverQueueSize=2000 +consumer.acknowledgementsGroupTimeMicros=0 +consumer.ackTimeoutMillis=2000 +consumer.deadLetterPolicy={ "maxRedeliverCount":"5", "deadLetterTopic":"", "initialSubscriptionName":"" } +consumer.ackTimeoutRedeliveryBackoff={"minDelayMs":"50", "maxDelayMs":"100", "multiplier":"2.0"} +consumer.negativeAckRedeliveryBackoff={} +#... diff --git a/nb5/pom.xml b/nb5/pom.xml index 9d426e8ab..555089d12 100644 --- a/nb5/pom.xml +++ b/nb5/pom.xml @@ -97,7 +97,7 @@ io.nosqlbench adapter-s4j - 4.17.31-SNAPSHOT + 4.17.32-SNAPSHOT From 4b04cb3aa70baeddde325e31b754aabb188477c3 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 9 Dec 2022 14:21:06 -0600 Subject: [PATCH 42/45] move warning to ctor --- .../errorhandling/modular/handlers/CountErrorHandler.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/handlers/CountErrorHandler.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/handlers/CountErrorHandler.java index eb32a69da..513dc499f 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/handlers/CountErrorHandler.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/handlers/CountErrorHandler.java @@ -30,11 +30,14 @@ import org.apache.logging.log4j.Logger; @Service(value = ErrorHandler.class, selector = "count") public class CountErrorHandler extends CounterErrorHandler { + public CountErrorHandler() { + logger.warn("Starting with v4.17 onward, use 'counter'. See cql-errors.md for usage."); + } + private static final Logger logger = LogManager.getLogger(CountErrorHandler.class); @Override public ErrorDetail handleError(String name, Throwable t, long cycle, long durationInNanos, ErrorDetail detail) { - logger.warn("Starting with v4.17 onward, use 'counter'. See cql-errors.md for usage."); return super.handleError(name, t, cycle, durationInNanos, detail); } } From 9d1e5c14d7569e7180fa07647723b07731dbdfca Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 9 Dec 2022 14:26:09 -0600 Subject: [PATCH 43/45] nosqlbench-836 expand default data retention in promql from 6 mo to 10 years --- .../src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java index 43d431b93..8cb3520ec 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java @@ -179,7 +179,7 @@ public class NBCLIOptions { private Path statepath; private final List statePathAccesses = new ArrayList<>(); private final String hdrForChartFileName = DEFAULT_CHART_HDR_LOG_NAME; - private String dockerPromRetentionDays = "183d"; + private String dockerPromRetentionDays = "3650d"; private String reportSummaryTo = REPORT_SUMMARY_TO_DEFAULT; private boolean enableAnsi = System.getenv("TERM")!=null && !System.getenv("TERM").isEmpty(); private Maturity minMaturity = Maturity.Unspecified; @@ -371,7 +371,7 @@ public class NBCLIOptions { break; case DOCKER_PROM_RETENTION_DAYS: arglist.removeFirst(); - dockerPromRetentionDays = readWordOrThrow(arglist, "prometheus retention (183d by default)"); + dockerPromRetentionDays = readWordOrThrow(arglist, "prometheus retention (3650d by default)"); break; case DOCKER_GRAFANA_TAG: arglist.removeFirst(); From 862058d80f201416edc199877a52a2c5597cb401 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 9 Dec 2022 14:27:58 -0600 Subject: [PATCH 44/45] make TriangularStep ctor public --- ...ularStepFunction.java => TriangularStep.java} | 14 +++++++------- ...FunctionTest.java => TriangularStepTest.java} | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) rename virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/{TriangularStepFunction.java => TriangularStep.java} (85%) rename virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/{TriangularStepFunctionTest.java => TriangularStepTest.java} (89%) diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStep.java similarity index 85% rename from virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java rename to virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStep.java index 9d7d8f8ae..c9bc86a9f 100644 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunction.java +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStep.java @@ -38,7 +38,7 @@ import java.util.function.LongUnaryOperator; * join the project or let us know.

*/ @ThreadSafeMapper -public class TriangularStepFunction implements LongUnaryOperator { +public class TriangularStep implements LongUnaryOperator { private final Hash hasher = new Hash(); private final long average; @@ -47,9 +47,9 @@ public class TriangularStepFunction implements LongUnaryOperator { private final long variance; - @Example({"TriangularStepFunction(100,20)","Create a sequence of values where the average is 100, but the range of values is between 80 and 120."}) - @Example({"TriangularStepFunction(80,10)","Create a sequence of values where the average is 80, but the range of values is between 70 and 90."}) - TriangularStepFunction(long average, long variance) { + @Example({"TriangularStep(100,20)","Create a sequence of values where the average is 100, but the range of values is between 80 and 120."}) + @Example({"TriangularStep(80,10)","Create a sequence of values where the average is 80, but the range of values is between 70 and 90."}) + public TriangularStep(long average, long variance) { if (average<=0 || variance < 0 || variance > average) { throw new RuntimeException( "The average must be positive, variance non-negative and the variance must be less than the average. " + @@ -61,7 +61,7 @@ public class TriangularStepFunction implements LongUnaryOperator { this.sizer = new HashRange(average-variance,average+variance); } - TriangularStepFunction(long average) { + TriangularStep(long average) { this(average, average/2); } @@ -71,9 +71,9 @@ public class TriangularStepFunction implements LongUnaryOperator { long count = operand / average; // offset within window long offset = operand % average; - // base of window + // base of window, same as count*average long base = operand - offset; - // variate up to window size + // variate + or - from average long variance = sizer.applyAsLong(base); // variate offset from start of window long slice = base + variance; diff --git a/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunctionTest.java b/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepTest.java similarity index 89% rename from virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunctionTest.java rename to virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepTest.java index a835fa3d5..8a8ab8574 100644 --- a/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepFunctionTest.java +++ b/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/TriangularStepTest.java @@ -25,14 +25,14 @@ import java.util.stream.IntStream; import static org.assertj.core.api.Assertions.assertThat; -public class TriangularStepFunctionTest { +public class TriangularStepTest { private static final int LABEL=0; private static final int FREQUENCY=1; @Test - public void testExample1() { - TriangularStepFunction e1 = new TriangularStepFunction(100, 20); + public void testStepExample1() { + TriangularStep e1 = new TriangularStep(100, 20); int[] runLengths = this.rleStatsFor(e1, 0, 10000); System.out.println(Arrays.toString(runLengths)); assertThat(IntStream.of(runLengths).min().orElseThrow()).isEqualTo(80L); @@ -40,8 +40,8 @@ public class TriangularStepFunctionTest { } @Test - public void testExample2() { - TriangularStepFunction e1 = new TriangularStepFunction(80, 10); + public void testStepExample2() { + TriangularStep e1 = new TriangularStep(80, 10); int[] runLengths = this.rleStatsFor(e1, 0, 10000); System.out.println(Arrays.toString(runLengths)); assertThat(IntStream.of(runLengths).min().orElseThrow()).isEqualTo(70L); @@ -50,7 +50,7 @@ public class TriangularStepFunctionTest { @Test public void testIncrementalVariance() { - TriangularStepFunction f = new TriangularStepFunction(100, 0); + TriangularStep f = new TriangularStep(100, 0); assertThat(f.applyAsLong(0L)).isEqualTo(0L); assertThat(f.applyAsLong(1L)).isEqualTo(0L); assertThat(f.applyAsLong(99L)).isEqualTo(0L); @@ -60,7 +60,7 @@ public class TriangularStepFunctionTest { @Test public void testVariance() { long first=0; - TriangularStepFunction f = new TriangularStepFunction(100,1); + TriangularStep f = new TriangularStep(100,1); var rlestats = rleStatsFor(f, 0, 100000); LongSummaryStatistics stats99to101 = statsForRle((int) f.applyAsLong(first),rlestats); assertThat(stats99to101.getMin()).isEqualTo(99L); @@ -74,7 +74,7 @@ public class TriangularStepFunctionTest { assertThat(histoStats.getAverage()).isEqualTo(100); } - private int[] rleStatsFor(TriangularStepFunction f, long firstTrialIncl, long lastTrialExcl) { + private int[] rleStatsFor(TriangularStep f, long firstTrialIncl, long lastTrialExcl) { long firstBucket = f.applyAsLong(firstTrialIncl); long lastBucket = f.applyAsLong(lastTrialExcl); if (firstBucket>Integer.MAX_VALUE||lastBucket>Integer.MAX_VALUE) { From 8a48a020dcf8c6e0c091cf9307ea5f09cd9e4064 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 9 Dec 2022 17:13:46 -0600 Subject: [PATCH 45/45] allow zero width ranges --- .../shared/from_long/to_collection/HashedRangeToLongList.java | 2 +- .../tests/long_collections/HashedRangeToLongListTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/HashedRangeToLongList.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/HashedRangeToLongList.java index 097cb1818..dd760e9c0 100644 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/HashedRangeToLongList.java +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/HashedRangeToLongList.java @@ -36,7 +36,7 @@ public class HashedRangeToLongList implements LongFunction> { private final HashRange sizeRange; public HashedRangeToLongList(int minVal, int maxVal, int minSize, int maxSize) { - if (minSize>=maxSize || minSize>=maxSize) { + if (minSize>maxSize || minVal>maxVal) { throw new RuntimeException("HashedRangeToLongList must have minval, maxval, minsize, maxsize, where min list= gener.apply(i); assertThat(list.size()).isBetween(9,12); for (Long longVal : list) {