From bb27a6f7fbc7f6236e0557cdd4cf8dcc822758f0 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 27 Oct 2022 18:58:07 +0000 Subject: [PATCH 01/14] fix: upgrade io.swagger.core.v3:swagger-models from 2.2.2 to 2.2.3 Snyk has created this PR to upgrade io.swagger.core.v3:swagger-models from 2.2.2 to 2.2.3. See this package in Maven Repository: https://mvnrepository.com/artifact/io.swagger.core.v3/swagger-models/ See this project in Snyk: https://app.snyk.io/org/jshook/project/b808ba5a-fa96-49c2-9cae-4c2c2f8a1384?utm_source=github&utm_medium=referral&page=upgrade-pr --- engine-rest/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine-rest/pom.xml b/engine-rest/pom.xml index c1b72cd64..25583745a 100644 --- a/engine-rest/pom.xml +++ b/engine-rest/pom.xml @@ -29,7 +29,7 @@ io.swagger.core.v3 swagger-models - 2.2.2 + 2.2.3 From 2b9452fde435383b9af7ef9fd2623ce438fa7271 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 11 Nov 2022 16:17:36 -0600 Subject: [PATCH 02/14] Buildfixes, trying to unblock current builds (#777) * make example/test script more robust for meager github actions environment * improve wording around op template resolution * remove blocking issues github actions plugin, since it is broken --- .github/workflows/blocking_issues.yml | 15 --------------- .../resources/workload_definition/README.md | 18 +++++++++++------- .../examples/extension_histostatslogger.js | 8 +++----- 3 files changed, 14 insertions(+), 27 deletions(-) delete mode 100644 .github/workflows/blocking_issues.yml diff --git a/.github/workflows/blocking_issues.yml b/.github/workflows/blocking_issues.yml deleted file mode 100644 index 7bea810ac..000000000 --- a/.github/workflows/blocking_issues.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: Blocking Issues - -on: - issues: - types: [closed] - pull_request_target: - types: [opened, edited] - -jobs: - blocking_issues: - runs-on: ubuntu-latest - name: Checks for blocking issues - - steps: - - uses: Levi-Lesches/blocking-issues@v1.1 diff --git a/adapters-api/src/main/resources/workload_definition/README.md b/adapters-api/src/main/resources/workload_definition/README.md index 7bcd0ba9c..c0464384e 100644 --- a/adapters-api/src/main/resources/workload_definition/README.md +++ b/adapters-api/src/main/resources/workload_definition/README.md @@ -63,17 +63,21 @@ The process of loading a workload definition occurs in several discrete steps du session: 1. The workload file is loaded. -2. Template variables are interposed. +2. Template variables from the activity parameters are interposed into the raw contents of the + file. 3. The file is deserialized from its native form into a raw data structure. 4. The raw data structure is transformed into a normalized data structure according to the Op Template normalization rules. -5. The data is provided to the ParsedOp API for use by the developer. -6. The DriverAdapter is loaded which understands the op fields provided in the op template. -7. The DriverAdapter uses its documented rules to determine which types of native driver operations +5. Each op template is then denormalized as a self-contained data + structure, containing all the provided bindings, params, and tags from the upper layers of the + doc structure. +6. The data is provided to the ParsedOp API for use by the developer. +7. The DriverAdapter is loaded which understands the op fields provided in the op template. +8. The DriverAdapter uses its documented rules to determine which types of native driver operations each op template is intended to represent. This is called **Op Mapping**. -8. The DriverAdapter uses the identified types to create dispensers of native driver operations. - This is called **Op Dispensing**. -9. The op dispensers are arranged into an indexed bank of op sources according to the specified +9. The DriverAdapter (via the selected Op Mapper) uses the identified types to create dispensers of + native driver operations. This is called **Op Dispensing**. +10. The op dispensers are arranged into an indexed bank of op sources according to the specified ratios and or sequencing strategy. From this point on, NoSQLBench has the ability to construct an operation for any given cycle at high speed. diff --git a/nbr-examples/src/test/resources/scripts/examples/extension_histostatslogger.js b/nbr-examples/src/test/resources/scripts/examples/extension_histostatslogger.js index 377c1b357..01b1f1194 100644 --- a/nbr-examples/src/test/resources/scripts/examples/extension_histostatslogger.js +++ b/nbr-examples/src/test/resources/scripts/examples/extension_histostatslogger.js @@ -18,16 +18,14 @@ activitydef = { "alias" : "testhistostatslogger", "driver" : "diag", "cycles" : "50000", - "threads" : "20", - "interval" : "2000", - "targetrate" : "10000.0", + "threads" : "5", + "rate" : "100.0", "op" : "noop" }; histostatslogger.logHistoStats("testing extention histostatslogger", ".*", "logs/histostats.csv", "0.5s"); print("started logging to logs/histostats.csv for all metrics at 1/2" + " second intervals."); - scenario.start(activitydef); -scenario.waitMillis(2000); +scenario.waitMillis(4000); scenario.stop(activitydef); From c309d200acef2137dedac36b667051f76a3dbee8 Mon Sep 17 00:00:00 2001 From: yabinmeng Date: Fri, 11 Nov 2022 18:36:06 -0600 Subject: [PATCH 03/14] - Delete unsed code "driver-jms" - NB5 Pulsar driver Admin functionality in place - create/delete tenant, namespace, and topic --- .run/NBCLI web foreground dryrun.run.xml | 16 - {driver-jms => adapter-pulsar}/pom.xml | 70 ++- .../adapter/pulsar/PulsarDriverAdapter.java | 54 ++ .../adapter/pulsar/PulsarOpMapper.java | 86 +++ .../adapter/pulsar/PulsarOpType.java | 23 +- .../adapter/pulsar/PulsarSpace.java | 191 +++++++ .../dispensers/AdminNamespaceOpDispenser.java | 43 ++ .../dispensers/AdminTenantOpDispenser.java | 51 ++ .../dispensers/AdminTopicOpDispenser.java | 54 ++ .../MessageConsumerOpDispenser.java | 41 ++ .../MessageProducerOpDispenser.java | 41 ++ .../dispensers/MessageReaderOpDispenser.java | 41 ++ .../dispensers/PulsarAdminOpDispenser.java | 40 ++ .../dispensers/PulsarBaseOpDispenser.java | 85 +++ .../dispensers/PulsarClientOpDispenser.java | 40 ++ .../PulsarAdapterInvalidParamException.java | 25 + .../PulsarAdapterUnexpectedException.java | 30 + .../PulsarAdapterUnsupportedOpException.java | 25 + .../adapter/pulsar/ops/AdminNamespaceOp.java | 120 ++++ .../adapter/pulsar/ops/AdminTenantOp.java | 144 +++++ .../adapter/pulsar/ops/AdminTopicOp.java | 181 ++++++ .../adapter/pulsar/ops/MessageConsumerOp.java | 31 + .../adapter/pulsar/ops/MessageProducerOp.java | 32 ++ .../adapter/pulsar/ops/MessageReaderOp.java | 23 +- .../adapter/pulsar/ops/PulsarAdminOp.java | 32 ++ .../adapter/pulsar/ops/PulsarClientOp.java | 31 + .../adapter/pulsar/ops/PulsarOp.java | 23 + .../adapter/pulsar/util/AvroUtil.java | 123 ++++ .../pulsar/util/PulsarAdapterUtil.java | 538 ++++++++++++++++++ .../pulsar/util/PulsarNBClientConf.java | 336 +++++++++++ .../src/main/resources/admin-namespace.yaml | 16 + .../src/main/resources/admin-tenant.yaml | 17 + .../src/main/resources/admin-topic.yaml | 19 + .../src/main/resources/bindingtest.yaml | 4 + .../src/main/resources/config.properties | 52 ++ .../io/nosqlbench/driver/jms/JmsAction.java | 84 --- .../io/nosqlbench/driver/jms/JmsActivity.java | 179 ------ .../driver/jms/JmsActivityType.java | 48 -- .../io/nosqlbench/driver/jms/ReadyJmsOp.java | 78 --- .../driver/jms/ReadyPulsarJmsOp.java | 264 --------- .../driver/jms/conn/JmsConnInfo.java | 37 -- .../driver/jms/conn/JmsPulsarConnInfo.java | 58 -- .../driver/jms/ops/JmsMsgReadMapper.java | 88 --- .../driver/jms/ops/JmsMsgReadOp.java | 130 ----- .../driver/jms/ops/JmsMsgSendMapper.java | 71 --- .../driver/jms/ops/JmsMsgSendOp.java | 139 ----- .../driver/jms/ops/JmsOpMapper.java | 39 -- .../nosqlbench/driver/jms/util/JmsHeader.java | 82 --- .../driver/jms/util/JmsHeaderLongFunc.java | 47 -- .../nosqlbench/driver/jms/util/JmsUtil.java | 120 ---- .../driver/jms/util/PulsarConfig.java | 115 ---- driver-jms/src/main/resources/jms.md | 1 - .../main/resources/pulsar_config.properties | 33 -- driver-jms/src/main/resources/pulsar_jms.yaml | 89 --- nb/pom.xml | 6 + nb5/pom.xml | 6 + 56 files changed, 2611 insertions(+), 1781 deletions(-) delete mode 100644 .run/NBCLI web foreground dryrun.run.xml rename {driver-jms => adapter-pulsar}/pom.xml (58%) create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java rename driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsTimeTrackOp.java => adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java (62%) create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnexpectedException.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnsupportedOpException.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java rename driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOp.java => adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java (60%) create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/AvroUtil.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarNBClientConf.java create mode 100644 adapter-pulsar/src/main/resources/admin-namespace.yaml create mode 100644 adapter-pulsar/src/main/resources/admin-tenant.yaml create mode 100644 adapter-pulsar/src/main/resources/admin-topic.yaml create mode 100644 adapter-pulsar/src/main/resources/bindingtest.yaml create mode 100644 adapter-pulsar/src/main/resources/config.properties delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsAction.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivityType.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyJmsOp.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyPulsarJmsOp.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsConnInfo.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsPulsarConnInfo.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadMapper.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadOp.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendMapper.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendOp.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOpMapper.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeader.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeaderLongFunc.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsUtil.java delete mode 100644 driver-jms/src/main/java/io/nosqlbench/driver/jms/util/PulsarConfig.java delete mode 100644 driver-jms/src/main/resources/jms.md delete mode 100644 driver-jms/src/main/resources/pulsar_config.properties delete mode 100644 driver-jms/src/main/resources/pulsar_jms.yaml diff --git a/.run/NBCLI web foreground dryrun.run.xml b/.run/NBCLI web foreground dryrun.run.xml deleted file mode 100644 index 4180a9a89..000000000 --- a/.run/NBCLI web foreground dryrun.run.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - \ No newline at end of file diff --git a/driver-jms/pom.xml b/adapter-pulsar/pom.xml similarity index 58% rename from driver-jms/pom.xml rename to adapter-pulsar/pom.xml index c3cbcc803..2428c54fe 100644 --- a/driver-jms/pom.xml +++ b/adapter-pulsar/pom.xml @@ -17,61 +17,49 @@ 4.0.0 + adapter-pulsar + jar + mvn-defaults io.nosqlbench - 4.17.22-SNAPSHOT + 4.17.31-SNAPSHOT ../mvn-defaults - driver-jms - jar ${project.artifactId} - - A JMS driver for nosqlbench. This provides the ability to inject synthetic data - into a pulsar system via JMS 2.0 compatibile APIs. - - NOTE: this is JMS compatible driver from DataStax that allows using a Pulsar cluster - as the potential JMS Destination + A Pulsar driver for nosqlbench. This provides the ability to inject synthetic data + into a pulsar system. - - - - - - - - - - - - - - + + 2.10.1 + - io.nosqlbench engine-api - 4.17.22-SNAPSHOT + 4.17.31-SNAPSHOT - - org.apache.commons - commons-lang3 - 3.12.0 + io.nosqlbench + adapters-api + 4.17.31-SNAPSHOT - - org.projectlombok - lombok - 1.18.24 - provided + org.apache.pulsar + pulsar-client + ${pulsar.version} + + + + org.apache.pulsar + pulsar-client-admin + ${pulsar.version} @@ -88,13 +76,19 @@ 2.8.0 - + - com.datastax.oss - pulsar-jms - 2.4.11 + org.apache.avro + avro + 1.11.1 + + + org.apache.commons + commons-lang3 + 3.12.0 + diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java new file mode 100644 index 000000000..b121ca5e3 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar; + +import io.nosqlbench.adapter.pulsar.ops.PulsarOp; +import io.nosqlbench.engine.api.activityimpl.OpMapper; +import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.nb.annotations.Maturity; +import io.nosqlbench.nb.annotations.Service; +import io.nosqlbench.api.config.standard.NBConfigModel; +import io.nosqlbench.api.config.standard.NBConfiguration; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.Function; + +@Service(value = DriverAdapter.class, selector = "pulsar-nb5", maturity = Maturity.Experimental) +public class PulsarDriverAdapter extends BaseDriverAdapter { + + private final static Logger logger = LogManager.getLogger(PulsarDriverAdapter.class); + + @Override + public OpMapper getOpMapper() { + DriverSpaceCache spaceCache = getSpaceCache(); + NBConfiguration adapterConfig = getConfiguration(); + return new PulsarOpMapper(this, adapterConfig, spaceCache); + } + + @Override + public Function getSpaceInitializer(NBConfiguration cfg) { + return (s) -> new PulsarSpace(s, cfg); + } + + @Override + public NBConfigModel getConfigModel() { + return super.getConfigModel().add(PulsarSpace.getConfigModel()); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java new file mode 100644 index 000000000..babcb0349 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar; + +import io.nosqlbench.adapter.pulsar.dispensers.*; +import io.nosqlbench.adapter.pulsar.ops.PulsarOp; +import io.nosqlbench.api.config.standard.NBConfiguration; +import io.nosqlbench.engine.api.activityimpl.OpDispenser; +import io.nosqlbench.engine.api.activityimpl.OpMapper; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.engine.api.templating.ParsedOp; +import io.nosqlbench.engine.api.templating.TypeAndTarget; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +public class PulsarOpMapper implements OpMapper { + + private final static Logger logger = LogManager.getLogger(PulsarOpMapper.class); + + private final NBConfiguration cfg; + private final DriverSpaceCache cache; + private final DriverAdapter adapter; + + public PulsarOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache cache) { + this.cfg = cfg; + this.cache = cache; + this.adapter = adapter; + } + + @Override + public OpDispenser apply(ParsedOp op) { + String space = op.getStaticConfigOr("space", "default"); + + PulsarClient pulsarClient = cache.get(space).getPulsarClient(); + PulsarAdmin pulsarAdmin = cache.get(space).getPulsarAdmin(); + Schema pulsarSchema = cache.get(space).getPulsarSchema(); + + + + /* + * If the user provides a body element, then they want to provide the JSON or + * a data structure that can be converted into JSON, bypassing any further + * specialized type-checking or op-type specific features + */ + if (op.isDefined("body")) { + throw new RuntimeException("This mode is reserved for later. Do not use the 'body' op field."); + } + else { + TypeAndTarget opType = op.getTypeAndTarget(PulsarOpType.class, String.class); + + return switch (opType.enumId) { + case AdminTenant -> + new AdminTenantOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin); + case AdminNamespace -> + new AdminNamespaceOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin); + case AdminTopic -> + new AdminTopicOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin); + case MessageProduce -> + new MessageProducerOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema); + case MessageConsume -> + new MessageConsumerOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema); + case MessageRead -> + new MessageReaderOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema); + }; + } + } + +} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsTimeTrackOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java similarity index 62% rename from driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsTimeTrackOp.java rename to adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java index 8d588faff..c4bf6c3ee 100644 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsTimeTrackOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java @@ -14,20 +14,13 @@ * limitations under the License. */ -package io.nosqlbench.driver.jms.ops; +package io.nosqlbench.adapter.pulsar; -/** - * Base type of all Sync Pulsar Operations including Producers and Consumers. - */ -public abstract class JmsTimeTrackOp implements JmsOp { - - public void run(Runnable timeTracker) { - try { - this.run(); - } finally { - timeTracker.run(); - } - } - - public abstract void run(); +public enum PulsarOpType { + AdminTenant, + AdminNamespace, + AdminTopic, + MessageProduce, + MessageConsume, + MessageRead } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java new file mode 100644 index 000000000..708007527 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar; + +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; +import io.nosqlbench.adapter.pulsar.util.PulsarNBClientConf; +import io.nosqlbench.api.config.standard.ConfigModel; +import io.nosqlbench.api.config.standard.NBConfigModel; +import io.nosqlbench.api.config.standard.NBConfiguration; +import io.nosqlbench.api.config.standard.Param; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.schema.KeyValueEncodingType; + +import java.util.Map; + +public class PulsarSpace { + + private final static Logger logger = LogManager.getLogger(PulsarSpace.class); + + private final String name; + private final NBConfiguration cfg; + + private final String pulsarSvcUrl; + private final String webSvcUrl; + + private PulsarNBClientConf pulsarNBClientConf; + private PulsarClient pulsarClient; + private PulsarAdmin pulsarAdmin; + private Schema pulsarSchema; + + public PulsarSpace(String name, NBConfiguration cfg) { + this.name = name; + this.cfg = cfg; + + this.pulsarSvcUrl = cfg.get("service_url"); + this.webSvcUrl = cfg.get("web_url"); + + this.pulsarNBClientConf = new PulsarNBClientConf(cfg.get("config")); + + initPulsarAdminAndClientObj(); + createPulsarSchemaFromConf(); + } + + public static NBConfigModel getConfigModel() { + return ConfigModel.of(PulsarSpace.class) + .add(Param.defaultTo("service_url", "pulsar://localhost:6650") + .setDescription("Pulsar broker service URL.")) + .add(Param.defaultTo("web_url", "http://localhost:8080") + .setDescription("Pulsar web service URL.")) + .add(Param.defaultTo("config", "config.properties") + .setDescription("Pulsar client connection configuration property file.")) + .add(Param.defaultTo("cyclerate_per_thread", false) + .setDescription("Apply cycle rate per NB thread")) + .asReadOnly(); + } + + public String getPulsarSvcUrl() { return pulsarSvcUrl; } + public String getWebSvcUrl() { return webSvcUrl; } + public PulsarNBClientConf getPulsarNBClientConf() { return pulsarNBClientConf; } + public PulsarClient getPulsarClient() { return pulsarClient; } + public PulsarAdmin getPulsarAdmin() { return pulsarAdmin; } + public Schema getPulsarSchema() { return pulsarSchema; } + + /** + * Initialize + * - PulsarAdmin object for adding/deleting tenant, namespace, and topic + * - PulsarClient object for message publishing and consuming + */ + private void initPulsarAdminAndClientObj() { + PulsarAdminBuilder adminBuilder = + PulsarAdmin.builder() + .serviceHttpUrl(webSvcUrl); + + ClientBuilder clientBuilder = PulsarClient.builder(); + + try { + Map clientConfMap = pulsarNBClientConf.getClientConfMap(); + + // Override "client.serviceUrl" setting in config.properties + clientConfMap.remove("serviceUrl"); + clientBuilder.loadConf(clientConfMap).serviceUrl(pulsarSvcUrl); + + // Pulsar Authentication + String authPluginClassName = + (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authPulginClassName.label); + String authParams = + (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authParams.label); + + if ( !StringUtils.isAnyBlank(authPluginClassName, authParams) ) { + adminBuilder.authentication(authPluginClassName, authParams); + clientBuilder.authentication(authPluginClassName, authParams); + } + + boolean useTls = StringUtils.contains(pulsarSvcUrl, "pulsar+ssl"); + if ( useTls ) { + String tlsHostnameVerificationEnableStr = + (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsHostnameVerificationEnable.label); + boolean tlsHostnameVerificationEnable = BooleanUtils.toBoolean(tlsHostnameVerificationEnableStr); + + adminBuilder + .enableTlsHostnameVerification(tlsHostnameVerificationEnable); + clientBuilder + .enableTlsHostnameVerification(tlsHostnameVerificationEnable); + + String tlsTrustCertsFilePath = + (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsTrustCertsFilePath.label); + if (!StringUtils.isBlank(tlsTrustCertsFilePath)) { + adminBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath); + clientBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath); + } + + String tlsAllowInsecureConnectionStr = + (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsAllowInsecureConnection.label); + boolean tlsAllowInsecureConnection = BooleanUtils.toBoolean(tlsAllowInsecureConnectionStr); + adminBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection); + clientBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection); + } + + pulsarAdmin = adminBuilder.build(); + pulsarClient = clientBuilder.build(); + + } catch (PulsarClientException e) { + logger.error("Fail to create PulsarAdmin and/or PulsarClient object from the global configuration!"); + throw new RuntimeException("Fail to create PulsarAdmin and/or PulsarClient object from global configuration!"); + } + } + + /** + * Get Pulsar schema from the definition string + */ + + private Schema buildSchemaFromDefinition(String schemaTypeConfEntry, + String schemaDefinitionConfEntry) { + Object value = pulsarNBClientConf.getSchemaConfValue(schemaTypeConfEntry); + Object schemaDefinition = pulsarNBClientConf.getSchemaConfValue(schemaDefinitionConfEntry); + String schemaType = (value != null) ? value.toString() : ""; + + Schema result; + if (PulsarAdapterUtil.isAvroSchemaTypeStr(schemaType)) { + String schemaDefStr = (schemaDefinition != null) ? schemaDefinition.toString() : ""; + result = PulsarAdapterUtil.getAvroSchema(schemaType, schemaDefStr); + } else if (PulsarAdapterUtil.isPrimitiveSchemaTypeStr(schemaType)) { + result = PulsarAdapterUtil.getPrimitiveTypeSchema(schemaType); + } else if (PulsarAdapterUtil.isAutoConsumeSchemaTypeStr(schemaType)) { + result = Schema.AUTO_CONSUME(); + } else { + throw new RuntimeException("Unsupported schema type string: " + schemaType + "; " + + "Only primitive type, Avro type and AUTO_CONSUME are supported at the moment!"); + } + return result; + } + private void createPulsarSchemaFromConf() { + pulsarSchema = buildSchemaFromDefinition("schema.type", "schema.definition"); + + // this is to allow KEY_VALUE schema + if (pulsarNBClientConf.hasSchemaConfKey("schema.key.type")) { + Schema pulsarKeySchema = buildSchemaFromDefinition("schema.key.type", "schema.key.definition"); + Object encodingType = pulsarNBClientConf.getSchemaConfValue("schema.keyvalue.encodingtype"); + KeyValueEncodingType keyValueEncodingType = KeyValueEncodingType.SEPARATED; + if (encodingType != null) { + keyValueEncodingType = KeyValueEncodingType.valueOf(encodingType.toString()); + } + pulsarSchema = Schema.KeyValue(pulsarKeySchema, pulsarSchema, keyValueEncodingType); + } + } +} + + diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java new file mode 100644 index 000000000..89514d4e6 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.AdminNamespaceOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.admin.PulsarAdmin; + +import java.util.function.LongFunction; + +public class AdminNamespaceOpDispenser extends PulsarAdminOpDispenser { + + public AdminNamespaceOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarAdmin pulsarAdmin) { + super(adapter, op, tgtNameFunc, pulsarAdmin); + } + + @Override + public AdminNamespaceOp apply(long cycle) { + return new AdminNamespaceOp( + pulsarAdmin, + asyncApiFunc.apply(cycle), + adminDelOpFunc.apply(cycle), + tgtNameFunc.apply(cycle)); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java new file mode 100644 index 000000000..5bca4ec78 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.AdminTenantOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.admin.PulsarAdmin; + +import java.util.*; +import java.util.function.LongFunction; + +public class AdminTenantOpDispenser extends PulsarAdminOpDispenser { + + private final LongFunction> adminRolesFunc; + private final LongFunction> allowedClustersFunc; + public AdminTenantOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarAdmin pulsarAdmin) { + super(adapter, op, tgtNameFunc, pulsarAdmin); + + adminRolesFunc = lookupStaticStrSetOpValueFunc("admin_roles"); + allowedClustersFunc = lookupStaticStrSetOpValueFunc("allowed_clusters"); + } + + @Override + public AdminTenantOp apply(long cycle) { + return new AdminTenantOp( + pulsarAdmin, + asyncApiFunc.apply(cycle), + adminDelOpFunc.apply(cycle), + tgtNameFunc.apply(cycle), + adminRolesFunc.apply(cycle), + allowedClustersFunc.apply(cycle)); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java new file mode 100644 index 000000000..70af09180 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.AdminTopicOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.admin.PulsarAdmin; + +import java.util.function.LongFunction; + +public class AdminTopicOpDispenser extends PulsarAdminOpDispenser { + + private final LongFunction enablePartFunc; + private final LongFunction partNumFunc; + + public AdminTopicOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarAdmin pulsarAdmin) { + super(adapter, op, tgtNameFunc, pulsarAdmin); + + // Non-partitioned topic is default + enablePartFunc = lookupStaticBoolConfigValueFunc("enable_partition", false); + partNumFunc = lookupStaticIntOpValueFunc("partition_num", 1); + } + + @Override + public AdminTopicOp apply(long cycle) { + + return new AdminTopicOp( + pulsarAdmin, + asyncApiFunc.apply(cycle), + adminDelOpFunc.apply(cycle), + tgtNameFunc.apply(cycle), + enablePartFunc.apply(cycle), + partNumFunc.apply(cycle) + ); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java new file mode 100644 index 000000000..8a0cb6ba6 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.MessageConsumerOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +import java.util.function.LongFunction; + +public class MessageConsumerOpDispenser extends PulsarClientOpDispenser { + + public MessageConsumerOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarClient pulsarClient, + Schema pulsarSchema) { + super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema); + } + + @Override + public MessageConsumerOp apply(long cycle) { + return new MessageConsumerOp(pulsarClient, pulsarSchema); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java new file mode 100644 index 000000000..ef9b2547d --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.MessageProducerOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +import java.util.function.LongFunction; + +public class MessageProducerOpDispenser extends PulsarClientOpDispenser { + + public MessageProducerOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarClient pulsarClient, + Schema pulsarSchema) { + super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema); + } + + @Override + public MessageProducerOp apply(long cycle) { + return new MessageProducerOp(pulsarClient, pulsarSchema); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java new file mode 100644 index 000000000..3e9f819fd --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.MessageReaderOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +import java.util.function.LongFunction; + +public class MessageReaderOpDispenser extends PulsarClientOpDispenser { + + public MessageReaderOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarClient pulsarClient, + Schema pulsarSchema) { + super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema); + } + + @Override + public MessageReaderOp apply(long cycle) { + return new MessageReaderOp(pulsarClient, pulsarSchema); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java new file mode 100644 index 000000000..d18eee305 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.admin.PulsarAdmin; + +import java.util.function.LongFunction; + +public abstract class PulsarAdminOpDispenser extends PulsarBaseOpDispenser { + + protected final PulsarAdmin pulsarAdmin; + protected final LongFunction adminDelOpFunc; + + public PulsarAdminOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarAdmin pulsarAdmin) { + super(adapter, op, tgtNameFunc); + this.pulsarAdmin = pulsarAdmin; + + // Creating admin objects (tenant, namespace, topic) is the default + this.adminDelOpFunc = lookupStaticBoolConfigValueFunc("admin_delop", false); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java new file mode 100644 index 000000000..1a0350ac1 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java @@ -0,0 +1,85 @@ +package io.nosqlbench.adapter.pulsar.dispensers; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import io.nosqlbench.adapter.pulsar.PulsarSpace; +import io.nosqlbench.adapter.pulsar.ops.PulsarOp; +import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.*; +import java.util.function.LongFunction; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public abstract class PulsarBaseOpDispenser extends BaseOpDispenser { + + private final static Logger logger = LogManager.getLogger("PulsarBaseOpDispenser"); + protected final ParsedOp parsedOp; + protected final LongFunction asyncApiFunc; + protected final LongFunction tgtNameFunc; + + public PulsarBaseOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc) { + + super(adapter, op); + + this.parsedOp = op; + this.tgtNameFunc = tgtNameFunc; + // Async API is the default + this.asyncApiFunc = lookupStaticBoolConfigValueFunc("async_api", true); + } + + protected LongFunction lookupStaticBoolConfigValueFunc(String paramName, boolean defaultValue) { + return (l) -> parsedOp.getOptionalStaticConfig(paramName, String.class) + .filter(Predicate.not(String::isEmpty)) + .map(value -> BooleanUtils.toBoolean(value)) + .orElse(defaultValue); + } + + protected LongFunction lookupStaticIntOpValueFunc(String paramName, int defaultValue) { + return (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) + .filter(Predicate.not(String::isEmpty)) + .map(value -> NumberUtils.toInt(value)) + .map(value -> { + if (value < 0) return 0; + else return value; + }).orElse(defaultValue); + } + + protected LongFunction> lookupStaticStrSetOpValueFunc(String paramName) { + return (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) + .filter(Predicate.not(String::isEmpty)) + .map(value -> { + Set set = new HashSet<>(); + + if (StringUtils.contains(value,',')) { + set = Arrays.stream(value.split(",")) + .map(String::trim) + .filter(Predicate.not(String::isEmpty)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + return set; + }).orElse(Collections.emptySet()); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java new file mode 100644 index 000000000..43acd0bb8 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +import java.util.function.LongFunction; + +public abstract class PulsarClientOpDispenser extends PulsarBaseOpDispenser { + + protected final PulsarClient pulsarClient; + protected final Schema pulsarSchema; + + public PulsarClientOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarClient pulsarClient, + Schema pulsarSchema) { + super(adapter, op, tgtNameFunc); + this.pulsarClient = pulsarClient; + this.pulsarSchema = pulsarSchema; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java new file mode 100644 index 000000000..1004a7a72 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.pulsar.exception; + +public class PulsarAdapterInvalidParamException extends RuntimeException { + + public PulsarAdapterInvalidParamException(String paramName, String errDesc) { + super("Invalid setting for parameter (" + paramName + "): " + errDesc); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnexpectedException.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnexpectedException.java new file mode 100644 index 000000000..4f0031fce --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnexpectedException.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.pulsar.exception; + +public class PulsarAdapterUnexpectedException extends RuntimeException { + + public PulsarAdapterUnexpectedException(String message) { + super(message); + printStackTrace(); + } + public PulsarAdapterUnexpectedException(Exception e) { + super(e); + printStackTrace(); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnsupportedOpException.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnsupportedOpException.java new file mode 100644 index 000000000..475d358ea --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnsupportedOpException.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.pulsar.exception; + +public class PulsarAdapterUnsupportedOpException extends RuntimeException { + + public PulsarAdapterUnsupportedOpException(String pulsarOpType) { + super("Unsupported Pulsar adapter operation type: \"" + pulsarOpType + "\""); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java new file mode 100644 index 000000000..71e6f79ee --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.admin.Namespaces; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; + +import java.util.concurrent.CompletableFuture; + +public class AdminNamespaceOp extends PulsarAdminOp { + + private final static Logger logger = LogManager.getLogger(AdminNamespaceOp.class); + + // in format: / + private final String nsName; + + public AdminNamespaceOp(PulsarAdmin pulsarAdmin, + boolean asyncApi, + boolean adminDelOp, + String nsName) { + super(pulsarAdmin, asyncApi, adminDelOp); + this.nsName = nsName; + } + + @Override + public Void apply(long value) { + + // Do nothing if the namespace name is empty + if ( !StringUtils.isBlank(nsName) ) { + + Namespaces namespaces = pulsarAdmin.namespaces(); + + // Admin API - create tenants and namespaces + if (!adminDelOp) { + try { + if (!asyncApi) { + namespaces.createNamespace(nsName); + if (logger.isDebugEnabled()) { + logger.debug("Successful sync creation of namespace \"{}\"", nsName); + } + } else { + CompletableFuture future = namespaces.createNamespaceAsync(nsName); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.trace("Successful async creation of namespace \"{}\"", nsName); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error("Failed async creation of namespace \"{}\"", nsName); + } + return null; + }); + } + } + catch (PulsarAdminException.ConflictException ce) { + if (logger.isDebugEnabled()) { + logger.error("Namespace \"{}\" already exists - skip creation!", nsName); + } + } + catch (PulsarAdminException e) { + throw new PulsarAdapterUnexpectedException( + "Unexpected error when creating pulsar namespace \"" + nsName + "\""); + } + } + // Admin API - delete tenants and namespaces + else { + try { + if (!asyncApi) { + namespaces.deleteNamespace(nsName); + if (logger.isDebugEnabled()) { + logger.debug("Successful sync deletion of namespace \"{}\"", nsName); + } + } else { + CompletableFuture future = namespaces.deleteNamespaceAsync(nsName, true); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug("Successful sync deletion of namespace \"{}\"", nsName); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error("Failed async deletion of namespace \"{}\"", nsName); + } + return null; + }); + } + } + catch (PulsarAdminException.NotFoundException nfe) { + if (logger.isDebugEnabled()) { + logger.error("Namespace \"{}\" doesn't exists - skip deletion!", nsName); + } + } + catch (PulsarAdminException e) { + throw new PulsarAdapterUnexpectedException( + "Unexpected error when deleting pulsar namespace \"" + nsName + "\""); + } + } + } + + return null; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java new file mode 100644 index 000000000..b40013ff7 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.adapter.pulsar.PulsarDriverAdapter; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.admin.*; +import org.apache.pulsar.common.policies.data.TenantInfo; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +public class AdminTenantOp extends PulsarAdminOp { + + private final static Logger logger = LogManager.getLogger(AdminTenantOp.class); + + private final Set adminRoles; + private final Set allowedClusters; + private final String tntName; + + public AdminTenantOp(PulsarAdmin pulsarAdmin, + boolean asyncApi, + boolean adminDelOp, + String tntName, + Set adminRoles, + Set allowedClusters) { + super(pulsarAdmin, asyncApi, adminDelOp); + this.tntName = tntName; + this.adminRoles = adminRoles; + this.allowedClusters = allowedClusters; + } + + @Override + public Void apply(long value) { + + // Do nothing if the tenant name is empty + if ( !StringUtils.isBlank(tntName) ) { + Tenants tenants = pulsarAdmin.tenants(); + + // Admin API - create tenants and namespaces + if (!adminDelOp) { + try { + Set existingPulsarClusters = new HashSet<>(); + Clusters clusters = pulsarAdmin.clusters(); + CollectionUtils.addAll(existingPulsarClusters, clusters.getClusters().listIterator()); + + TenantInfo tenantInfo = TenantInfo.builder() + .adminRoles(adminRoles) + .allowedClusters(!allowedClusters.isEmpty() ? allowedClusters : existingPulsarClusters) + .build(); + + if (!asyncApi) { + tenants.createTenant(tntName, tenantInfo); + if (logger.isDebugEnabled()) { + logger.debug("Successful sync creation of tenant \"{}\"", tntName); + } + } + else { + CompletableFuture future = tenants.createTenantAsync(tntName, tenantInfo); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug("Successful async creation of tenant \"{}\"", tntName); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error("Failed async creation of tenant \"{}\"", tntName); + } + return null; + }); + } + } + catch (PulsarAdminException.ConflictException ce) { + if (logger.isDebugEnabled()) { + logger.error("Tenant \"{}\" already exists - skip creation!", tntName); + } + } + catch (PulsarAdminException e) { + throw new PulsarAdapterUnexpectedException( + "Unexpected error when creating pulsar tenant \"" + tntName + "\""); + } + } + // Admin API - delete tenants and namespaces + else { + try { + Namespaces namespaces = pulsarAdmin.namespaces(); + int nsNum = namespaces.getNamespaces(tntName).size(); + + // Only delete a tenant when there is no underlying namespaces + if ( nsNum == 0 ) { + if (!asyncApi) { + tenants.deleteTenant(tntName); + if (logger.isDebugEnabled()) { + logger.debug("Successful sync deletion of tenant \"{}\"", tntName); + } + } + else { + CompletableFuture future = tenants.deleteTenantAsync(tntName); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug("Successful async deletion of tenant \"{}\"", tntName); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error("Failed async deletion of tenant \"{}\"", tntName); + } + return null; + }); + } + } + } + catch (PulsarAdminException.NotFoundException nfe) { + if (logger.isDebugEnabled()) { + logger.error("Tenant \"{}\" doesn't exists - skip deletion!", tntName); + } + } + catch (PulsarAdminException e) { + throw new PulsarAdapterUnexpectedException( + "Unexpected error when deleting pulsar tenant \"" + tntName + "\""); + } + } + } + + return null; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java new file mode 100644 index 000000000..2c3735658 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.admin.Topics; + +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +public class AdminTopicOp extends PulsarAdminOp { + + private final static Logger logger = LogManager.getLogger(AdminTopicOp.class); + + private final String topicName; + private final boolean enablePart; + private final int partNum; + + public AdminTopicOp(PulsarAdmin pulsarAdmin, + boolean asyncApi, + boolean adminDelOp, + String topicName, + boolean enablePart, + int partNum) { + super(pulsarAdmin, asyncApi, adminDelOp); + this.topicName = topicName; + this.enablePart = enablePart; + this.partNum = partNum; + } + + @Override + public Void apply(long value) { + + // Do nothing if the topic name is empty + if ( !StringUtils.isBlank(topicName) ) { + Topics topics = pulsarAdmin.topics(); + + try { + // Create the topic + if (!adminDelOp) { + if (!enablePart) { + if (!asyncApi) { + topics.createNonPartitionedTopic(topicName); + if (logger.isDebugEnabled()) { + logger.debug("Successful sync creation of non-partitioned topic \"{}\"", topicName); + } + } else { + CompletableFuture future = topics.createNonPartitionedTopicAsync(topicName); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug("Successful async creation of non-partitioned topic \"{}\"", topicName); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error("Failed async creation non-partitioned topic \"{}\"", topicName); + return null; + } + return null; + }); + } + } else { + if (!asyncApi) { + topics.createPartitionedTopic(topicName, partNum); + if (logger.isDebugEnabled()) { + logger.debug( + "Successful sync creation of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + } else { + CompletableFuture future = topics.createPartitionedTopicAsync(topicName, partNum); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug( + "Successful async creation of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + }) + .exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error( + "Successful async creation of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + return null; + }); + } + } + } + // Delete the topic + else { + if (!enablePart) { + if (!asyncApi) { + topics.delete(topicName); + if (logger.isDebugEnabled()) { + logger.debug( + "Successful sync deletion of non-partitioned topic \"{}\"", + topicName); + } + } else { + CompletableFuture future = topics.deleteAsync(topicName); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug( + "Successful async deletion of non-partitioned topic \"{}\"", + topicName); + } + }) + .exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error( + "Failed async deletion of non-partitioned topic \"{}\"", + topicName); + } + return null; + }); + } + } else { + if (!asyncApi) { + topics.deletePartitionedTopic(topicName); + if (logger.isDebugEnabled()) { + logger.debug( + "Successful sync deletion of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + } else { + CompletableFuture future = topics.deletePartitionedTopicAsync(topicName); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug( + "Successful async deletion of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error( + "Failed async deletion of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + return null; + }); + } + } + } + } + catch (PulsarAdminException.NotFoundException nfe) { + if (logger.isDebugEnabled()) { + logger.error("Topic \"{}\" doesn't exists - skip deletion!", topicName); + } + } + catch (PulsarAdminException e) { + String errMsg = String.format("Unexpected error when %s pulsar topic: %s (partition enabled: %b; partition number: %d)", + (!adminDelOp ? "creating" : "deleting"), + topicName, + enablePart, + partNum); + throw new PulsarAdapterUnexpectedException(errMsg); + } + } + + return null; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java new file mode 100644 index 000000000..d9f28c7d3 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +public class MessageConsumerOp extends PulsarClientOp { + public MessageConsumerOp(PulsarClient pulsarClient, Schema pulsarSchema) { + super(pulsarClient, pulsarSchema); + } + + @Override + public Object apply(long value) { + return null; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java new file mode 100644 index 000000000..664509ec8 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +public class MessageProducerOp extends PulsarClientOp { + + public MessageProducerOp(PulsarClient pulsarClient, Schema pulsarSchema) { + super(pulsarClient, pulsarSchema); + } + + @Override + public Object apply(long value) { + return null; + } +} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java similarity index 60% rename from driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOp.java rename to adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java index b8f227ffd..c9d47ec30 100644 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java @@ -14,16 +14,19 @@ * limitations under the License. */ -package io.nosqlbench.driver.jms.ops; +package io.nosqlbench.adapter.pulsar.ops; -/** - * Base type of all Pulsar Operations including Producers and Consumers. - */ -public interface JmsOp { +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; - /** - * Execute the operation, invoke the timeTracker when the operation ended. - * The timeTracker can be invoked in a separate thread, it is only used for metrics. - */ - void run(Runnable timeTracker); +public class MessageReaderOp extends PulsarClientOp { + + public MessageReaderOp(PulsarClient pulsarClient, Schema pulsarSchema) { + super(pulsarClient, pulsarSchema); + } + + @Override + public Object apply(long value) { + return null; + } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java new file mode 100644 index 000000000..4729a8cc5 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp; +import org.apache.pulsar.client.admin.PulsarAdmin; + +public abstract class PulsarAdminOp extends PulsarOp { + protected PulsarAdmin pulsarAdmin; + protected boolean asyncApi; + protected boolean adminDelOp; + + public PulsarAdminOp(PulsarAdmin pulsarAdmin, boolean asyncApi, boolean adminDelOp) { + this.pulsarAdmin = pulsarAdmin; + this.asyncApi = asyncApi; + this.adminDelOp = adminDelOp; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java new file mode 100644 index 000000000..957616681 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +public abstract class PulsarClientOp extends PulsarOp { + protected PulsarClient pulsarClient; + protected Schema pulsarScheam; + + public PulsarClientOp(PulsarClient pulsarClient, Schema pulsarScheam) { + this.pulsarClient = pulsarClient; + this.pulsarScheam = pulsarScheam; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java new file mode 100644 index 000000000..0c68c52d7 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp; + +public abstract class PulsarOp implements CycleOp { +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/AvroUtil.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/AvroUtil.java new file mode 100644 index 000000000..882f060f9 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/AvroUtil.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.util; + +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.JsonDecoder; +import org.apache.avro.io.BinaryDecoder; +import org.apache.pulsar.client.api.schema.Field; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.GenericRecordBuilder; +import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; + +public class AvroUtil { + //////////////////////// + // Get an OSS Apache Avro schema from a string definition + public static org.apache.avro.Schema GetSchema_ApacheAvro(String avroSchemDef) { + return new org.apache.avro.Schema.Parser().parse(avroSchemDef); + } + + // Get an OSS Apache Avro schema record from a JSON string that matches a specific OSS Apache Avro schema + public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, String jsonData) { + org.apache.avro.generic.GenericRecord record = null; + + try { + org.apache.avro.generic.GenericDatumReader reader; + reader = new org.apache.avro.generic.GenericDatumReader<>(schema); + + JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, jsonData); + + record = reader.read(null, decoder); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + + return record; + } + + // Get an OSS Apache Avro schema record from a byte array that matches a specific OSS Apache Avro schema + public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, byte[] bytesData) { + org.apache.avro.generic.GenericRecord record = null; + + try { + org.apache.avro.generic.GenericDatumReader reader; + reader = new org.apache.avro.generic.GenericDatumReader<>(schema); + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytesData, null); + + record = reader.read(null, decoder); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + + return record; + } + + + //////////////////////// + // Get a Pulsar Avro schema from a string definition + public static GenericAvroSchema GetSchema_PulsarAvro(String schemaName, String avroSchemDef) { + SchemaInfo schemaInfo = SchemaInfoImpl.builder() + .schema(avroSchemDef.getBytes(StandardCharsets.UTF_8)) + .type(SchemaType.AVRO) + .properties(new HashMap<>()) + .name(schemaName) + .build(); + return new GenericAvroSchema(schemaInfo); + } + + // Get a Pulsar Avro record from an OSS Avro schema record, matching a specific Pulsar Avro schema + public static GenericRecord GetGenericRecord_PulsarAvro( + GenericAvroSchema pulsarGenericAvroSchema, + org.apache.avro.generic.GenericRecord apacheAvroGenericRecord) + { + GenericRecordBuilder recordBuilder = pulsarGenericAvroSchema.newRecordBuilder(); + + List fieldList = pulsarGenericAvroSchema.getFields(); + for (Field field : fieldList) { + String fieldName = field.getName(); + recordBuilder.set(fieldName, apacheAvroGenericRecord.get(fieldName)); + } + + return recordBuilder.build(); + } + + // Get a Pulsar Avro record (GenericRecord) from a JSON string that matches a specific Pulsar Avro schema + public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, String avroSchemDefStr, String jsonData) { + org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr); + return GetGenericRecord_PulsarAvro(genericAvroSchema, avroSchema, jsonData); + } + + public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, org.apache.avro.Schema avroSchema, String jsonData) { + org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData); + return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord); + } + public static GenericRecord GetGenericRecord_PulsarAvro(String schemaName, String avroSchemDefStr, String jsonData) { + GenericAvroSchema genericAvroSchema = GetSchema_PulsarAvro(schemaName, avroSchemDefStr); + org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr); + org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData); + + return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java new file mode 100644 index 000000000..f2efbb8b6 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java @@ -0,0 +1,538 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.api.Schema; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class PulsarAdapterUtil { + + private final static Logger logger = LogManager.getLogger(PulsarAdapterUtil.class); + + // Supported message operation types + // TODO: websocket-producer and managed-ledger + public enum OP_TYPES { + ADMIN_TENANT("admin-tenant"), + ADMIN_NAMESPACE("admin-namespace"), + ADMIN_TOPIC("admin-topic"), + E2E_MSG_PROC_SEND("e22-msg-proc-send"), + E2E_MSG_PROC_CONSUME("e22-msg-proc-consume"), +// BATCH_MSG_SEND_START("batch-msg-send-start"), +// BATCH_MSG_SEND("batch-msg-send"), +// BATCH_MSG_SEND_END("batch-msg-send-end"), + MSG_SEND("msg-send"), + MSG_CONSUME("msg-consume"), + MSG_READ("msg-read"), + MSG_MULTI_CONSUME("msg-mt-consume"); + + public final String label; + + OP_TYPES(String label) { + this.label = label; + } + } + + + public static boolean isValidClientType(String type) { + return Arrays.stream(OP_TYPES.values()).anyMatch(t -> t.label.equals(type)); + } + + public static final String MSG_SEQUENCE_NUMBER = "sequence_number"; + + /////// + // Valid document level parameters for Pulsar NB yaml file + public enum DOC_LEVEL_PARAMS { + TOPIC_URI("topic_uri"), + ASYNC_API("async_api"), + USE_TRANSACTION("use_transaction"), + ADMIN_DELOP("admin_delop"), + SEQ_TRACKING("seq_tracking"), + MSG_DEDUP_BROKER("msg_dedup_broker"), + E2E_STARTING_TIME_SOURCE("e2e_starting_time_source"); + + public final String label; + + DOC_LEVEL_PARAMS(String label) { + this.label = label; + } + } + public static boolean isValidDocLevelParam(String param) { + return Arrays.stream(DOC_LEVEL_PARAMS.values()).anyMatch(t -> t.label.equals(param)); + } + + /////// + // Valid Pulsar API type + public enum PULSAR_API_TYPE { + PRODUCER("producer"), + CONSUMER("consumer"), + READER("reader"); + + public final String label; + + PULSAR_API_TYPE(String label) { + this.label = label; + } + } + public static boolean isValidPulsarApiType(String param) { + return Arrays.stream(PULSAR_API_TYPE.values()).anyMatch(t -> t.label.equals(param)); + } + public static String getValidPulsarApiTypeList() { + return Arrays.stream(PULSAR_API_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + /////// + // Valid persistence type + public enum PERSISTENT_TYPES { + PERSISTENT("persistent"), + NON_PERSISTENT("non-persistent") + ; + + public final String label; + PERSISTENT_TYPES(String label) { + this.label = label; + } + } + public static boolean isValidPersistenceType(String type) { + return Arrays.stream(PERSISTENT_TYPES.values()).anyMatch(t -> t.label.equals(type)); + } + + /////// + // Valid Pulsar client configuration (activity-level settings) + // - https://pulsar.apache.org/docs/en/client-libraries-java/#client + public enum CLNT_CONF_KEY { + serviceUrl("serviceUrl"), + authPulginClassName("authPluginClassName"), + authParams("authParams"), + pperationTimeoutMs("operationTimeoutMs"), + statsIntervalSeconds("statsIntervalSeconds"), + numIoThreads("numIoThreads"), + numListenerThreads("numListenerThreads"), + useTcpNoDelay("useTcpNoDelay"), + enableTls("enableTls"), + tlsTrustCertsFilePath("tlsTrustCertsFilePath"), + tlsAllowInsecureConnection("tlsAllowInsecureConnection"), + tlsHostnameVerificationEnable("tlsHostnameVerificationEnable"), + concurrentLookupRequest("concurrentLookupRequest"), + maxLookupRequest("maxLookupRequest"), + maxNumberOfRejectedRequestPerConnection("maxNumberOfRejectedRequestPerConnection"), + keepAliveIntervalSeconds("keepAliveIntervalSeconds"), + connectionTimeoutMs("connectionTimeoutMs"), + requestTimeoutMs("requestTimeoutMs"), + defaultBackoffIntervalNanos("defaultBackoffIntervalNanos"), + maxBackoffIntervalNanos("maxBackoffIntervalNanos"), + socks5ProxyAddress("socks5ProxyAddress"), + socks5ProxyUsername("socks5ProxyUsername"), + socks5ProxyPassword("socks5ProxyPassword") + ; + + public final String label; + CLNT_CONF_KEY(String label) { + this.label = label; + } + } + public static boolean isValidClientConfItem(String item) { + return Arrays.stream(CLNT_CONF_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Standard producer configuration (activity-level settings) + // - https://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer + public enum PRODUCER_CONF_STD_KEY { + topicName("topicName"), + producerName("producerName"), + sendTimeoutMs("sendTimeoutMs"), + blockIfQueueFull("blockIfQueueFull"), + maxPendingMessages("maxPendingMessages"), + maxPendingMessagesAcrossPartitions("maxPendingMessagesAcrossPartitions"), + messageRoutingMode("messageRoutingMode"), + hashingScheme("hashingScheme"), + cryptoFailureAction("cryptoFailureAction"), + batchingMaxPublishDelayMicros("batchingMaxPublishDelayMicros"), + batchingMaxMessages("batchingMaxMessages"), + batchingEnabled("batchingEnabled"), + compressionType("compressionType"); + + public final String label; + + PRODUCER_CONF_STD_KEY(String label) { + this.label = label; + } + } + public static boolean isStandardProducerConfItem(String item) { + return Arrays.stream(PRODUCER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Standard consumer configuration (activity-level settings) + // - https://pulsar.apache.org/docs/en/client-libraries-java/#consumer + public enum CONSUMER_CONF_STD_KEY { + topicNames("topicNames"), + topicsPattern("topicsPattern"), + subscriptionName("subscriptionName"), + subscriptionType("subscriptionType"), + receiverQueueSize("receiverQueueSize"), + acknowledgementsGroupTimeMicros("acknowledgementsGroupTimeMicros"), + negativeAckRedeliveryDelayMicros("negativeAckRedeliveryDelayMicros"), + maxTotalReceiverQueueSizeAcrossPartitions("maxTotalReceiverQueueSizeAcrossPartitions"), + consumerName("consumerName"), + ackTimeoutMillis("ackTimeoutMillis"), + tickDurationMillis("tickDurationMillis"), + priorityLevel("priorityLevel"), + cryptoFailureAction("cryptoFailureAction"), + properties("properties"), + readCompacted("readCompacted"), + subscriptionInitialPosition("subscriptionInitialPosition"), + patternAutoDiscoveryPeriod("patternAutoDiscoveryPeriod"), + regexSubscriptionMode("regexSubscriptionMode"), + deadLetterPolicy("deadLetterPolicy"), + autoUpdatePartitions("autoUpdatePartitions"), + replicateSubscriptionState("replicateSubscriptionState"); + + public final String label; + + CONSUMER_CONF_STD_KEY(String label) { + this.label = label; + } + } + public static boolean isStandardConsumerConfItem(String item) { + return Arrays.stream(CONSUMER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Custom consumer configuration (activity-level settings) + // - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#consumer + // - NB Pulsar driver consumer operation specific + public enum CONSUMER_CONF_CUSTOM_KEY { + timeout("timeout"); + + public final String label; + + CONSUMER_CONF_CUSTOM_KEY(String label) { + this.label = label; + } + } + public static boolean isCustomConsumerConfItem(String item) { + return Arrays.stream(CONSUMER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Pulsar subscription type + public enum SUBSCRIPTION_TYPE { + Exclusive("Exclusive"), + Failover("Failover"), + Shared("Shared"), + Key_Shared("Key_Shared"); + + public final String label; + + SUBSCRIPTION_TYPE(String label) { + this.label = label; + } + } + public static boolean isValidSubscriptionType(String item) { + return Arrays.stream(SUBSCRIPTION_TYPE.values()).anyMatch(t -> t.label.equals(item)); + } + public static String getValidSubscriptionTypeList() { + return Arrays.stream(SUBSCRIPTION_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + /////// + // Standard reader configuration (activity-level settings) + // - https://pulsar.apache.org/docs/en/client-libraries-java/#reader + public enum READER_CONF_STD_KEY { + topicName("topicName"), + receiverQueueSize("receiverQueueSize"), + readerListener("readerListener"), + readerName("readerName"), + subscriptionRolePrefix("subscriptionRolePrefix"), + cryptoKeyReader("cryptoKeyReader"), + cryptoFailureAction("cryptoFailureAction"), + readCompacted("readCompacted"), + resetIncludeHead("resetIncludeHead"); + + public final String label; + + READER_CONF_STD_KEY(String label) { + this.label = label; + } + } + public static boolean isStandardReaderConfItem(String item) { + return Arrays.stream(READER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Custom reader configuration (activity-level settings) + // - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#reader + // - NB Pulsar driver reader operation specific + public enum READER_CONF_CUSTOM_KEY { + startMessagePos("startMessagePos"); + + public final String label; + + READER_CONF_CUSTOM_KEY(String label) { + this.label = label; + } + } + public static boolean isCustomReaderConfItem(String item) { + return Arrays.stream(READER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Valid read positions for a Pulsar reader + public enum READER_MSG_POSITION_TYPE { + earliest("earliest"), + latest("latest"), + custom("custom"); + + public final String label; + + READER_MSG_POSITION_TYPE(String label) { + this.label = label; + } + } + public static boolean isValideReaderStartPosition(String item) { + return Arrays.stream(READER_MSG_POSITION_TYPE.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Pulsar subscription type + public enum SEQ_ERROR_SIMU_TYPE { + OutOfOrder("out_of_order"), + MsgLoss("msg_loss"), + MsgDup("msg_dup"); + + public final String label; + + SEQ_ERROR_SIMU_TYPE(String label) { + this.label = label; + } + + private static final Map MAPPING = new HashMap<>(); + + static { + for (SEQ_ERROR_SIMU_TYPE simuType : values()) { + MAPPING.put(simuType.label, simuType); + MAPPING.put(simuType.label.toLowerCase(), simuType); + MAPPING.put(simuType.label.toUpperCase(), simuType); + MAPPING.put(simuType.name(), simuType); + MAPPING.put(simuType.name().toLowerCase(), simuType); + MAPPING.put(simuType.name().toUpperCase(), simuType); + } + } + + public static Optional parseSimuType(String simuTypeString) { + return Optional.ofNullable(MAPPING.get(simuTypeString.trim())); + } + } + public static boolean isValidSeqErrSimuType(String item) { + return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).anyMatch(t -> t.label.equals(item)); + } + public static String getValidSeqErrSimuTypeList() { + return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + /////// + // Valid websocket-producer configuration (activity-level settings) + // TODO: to be added + public enum WEBSKT_PRODUCER_CONF_KEY { + ; + + public final String label; + + WEBSKT_PRODUCER_CONF_KEY(String label) { + this.label = label; + } + } + + /////// + // Valid managed-ledger configuration (activity-level settings) + // TODO: to be added + public enum MANAGED_LEDGER_CONF_KEY { + ; + + public final String label; + MANAGED_LEDGER_CONF_KEY(String label) { + this.label = label; + } + } + + /////// + // Primitive Schema type + public static boolean isPrimitiveSchemaTypeStr(String typeStr) { + boolean isPrimitive = false; + + // Use "BYTES" as the default type if the type string is not explicitly specified + if (StringUtils.isBlank(typeStr)) { + typeStr = "BYTES"; + } + + if (typeStr.equalsIgnoreCase("BOOLEAN") || typeStr.equalsIgnoreCase("INT8") || + typeStr.equalsIgnoreCase("INT16") || typeStr.equalsIgnoreCase("INT32") || + typeStr.equalsIgnoreCase("INT64") || typeStr.equalsIgnoreCase("FLOAT") || + typeStr.equalsIgnoreCase("DOUBLE") || typeStr.equalsIgnoreCase("BYTES") || + typeStr.equalsIgnoreCase("DATE") || typeStr.equalsIgnoreCase("TIME") || + typeStr.equalsIgnoreCase("TIMESTAMP") || typeStr.equalsIgnoreCase("INSTANT") || + typeStr.equalsIgnoreCase("LOCAL_DATE") || typeStr.equalsIgnoreCase("LOCAL_TIME") || + typeStr.equalsIgnoreCase("LOCAL_DATE_TIME")) { + isPrimitive = true; + } + + return isPrimitive; + } + public static Schema getPrimitiveTypeSchema(String typeStr) { + Schema schema; + + switch (typeStr.toUpperCase()) { + case "BOOLEAN": + schema = Schema.BOOL; + break; + case "INT8": + schema = Schema.INT8; + break; + case "INT16": + schema = Schema.INT16; + break; + case "INT32": + schema = Schema.INT32; + break; + case "INT64": + schema = Schema.INT64; + break; + case "FLOAT": + schema = Schema.FLOAT; + break; + case "DOUBLE": + schema = Schema.DOUBLE; + break; + case "DATE": + schema = Schema.DATE; + break; + case "TIME": + schema = Schema.TIME; + break; + case "TIMESTAMP": + schema = Schema.TIMESTAMP; + break; + case "INSTANT": + schema = Schema.INSTANT; + break; + case "LOCAL_DATE": + schema = Schema.LOCAL_DATE; + break; + case "LOCAL_TIME": + schema = Schema.LOCAL_TIME; + break; + case "LOCAL_DATE_TIME": + schema = Schema.LOCAL_DATE_TIME; + break; + // Use BYTES as the default schema type if the type string is not specified + case "": + case "BYTES": + schema = Schema.BYTES; + break; + // Report an error if non-valid, non-empty schema type string is provided + default: + throw new RuntimeException("Invalid Pulsar primitive schema type string : " + typeStr); + } + + return schema; + } + + /////// + // Complex strut type: Avro or Json + public static boolean isAvroSchemaTypeStr(String typeStr) { + return typeStr.equalsIgnoreCase("AVRO"); + } + public static boolean isKeyValueTypeStr(String typeStr) { + return typeStr.equalsIgnoreCase("KEY_VALUE"); + } + + // automatic decode the type from the Registry + public static boolean isAutoConsumeSchemaTypeStr(String typeStr) { + return typeStr.equalsIgnoreCase("AUTO_CONSUME"); + } + public static Schema getAvroSchema(String typeStr, String definitionStr) { + String schemaDefinitionStr = definitionStr; + String filePrefix = "file://"; + Schema schema; + + // Check if payloadStr points to a file (e.g. "file:///path/to/a/file") + if (isAvroSchemaTypeStr(typeStr)) { + if (StringUtils.isBlank(schemaDefinitionStr)) { + throw new RuntimeException("Schema definition must be provided for \"Avro\" schema type!"); + } else if (schemaDefinitionStr.startsWith(filePrefix)) { + try { + Path filePath = Paths.get(URI.create(schemaDefinitionStr)); + schemaDefinitionStr = Files.readString(filePath, StandardCharsets.US_ASCII); + } catch (IOException ioe) { + throw new RuntimeException("Error reading the specified \"Avro\" schema definition file: " + definitionStr + ": " + ioe.getMessage()); + } + } + + schema = AvroUtil.GetSchema_PulsarAvro("NBAvro", schemaDefinitionStr); + } else { + throw new RuntimeException("Trying to create a \"Avro\" schema for a non-Avro schema type string: " + typeStr); + } + + return schema; + } + + /////// + // Generate effective key string + public static String buildCacheKey(String... keyParts) { + // Ignore blank keyPart + String joinedKeyStr = + Stream.of(keyParts) + .filter(s -> !StringUtils.isBlank(s)) + .collect(Collectors.joining(",")); + + return Base64.getEncoder().encodeToString(joinedKeyStr.getBytes()); + } + + /////// + // Convert JSON string to a key/value map + public static Map convertJsonToMap(String jsonStr) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(jsonStr, Map.class); + } + + /////// + // Get full namespace name (/) from a Pulsar topic URI + public static String getFullNamespaceName(String topicUri) { + // Get tenant/namespace string + // - topicUri : persistent://// + // - tmpStr : // + // - fullNsName : / + + String tmpStr = StringUtils.substringAfter(topicUri,"://"); + return StringUtils.substringBeforeLast(tmpStr, "/"); + } +} + diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarNBClientConf.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarNBClientConf.java new file mode 100644 index 000000000..a434bc10d --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarNBClientConf.java @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.util; + +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.FileBasedConfiguration; +import org.apache.commons.configuration2.PropertiesConfiguration; +import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder; +import org.apache.commons.configuration2.builder.fluent.Parameters; +import org.apache.commons.configuration2.ex.ConfigurationException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +public class PulsarNBClientConf { + + private final static Logger logger = LogManager.getLogger(PulsarNBClientConf.class); + + private String canonicalFilePath = ""; + + public static final String SCHEMA_CONF_PREFIX = "schema"; + public static final String CLIENT_CONF_PREFIX = "client"; + public static final String PRODUCER_CONF_PREFIX = "producer"; + public static final String CONSUMER_CONF_PREFIX = "consumer"; + public static final String READER_CONF_PREFIX = "reader"; + private final HashMap schemaConfMap = new HashMap<>(); + private final HashMap clientConfMap = new HashMap<>(); + private final HashMap producerConfMap = new HashMap<>(); + private final HashMap consumerConfMap = new HashMap<>(); + private final HashMap readerConfMap = new HashMap<>(); + // TODO: add support for other operation types: websocket-producer, managed-ledger + + public PulsarNBClientConf(String fileName) { + File file = new File(fileName); + + try { + canonicalFilePath = file.getCanonicalPath(); + + Parameters params = new Parameters(); + + FileBasedConfigurationBuilder builder = + new FileBasedConfigurationBuilder(PropertiesConfiguration.class) + .configure(params.properties() + .setFileName(fileName)); + + Configuration config = builder.getConfiguration(); + + // Get schema specific configuration settings + for (Iterator it = config.getKeys(SCHEMA_CONF_PREFIX); it.hasNext(); ) { + String confKey = it.next(); + String confVal = config.getProperty(confKey).toString(); + if (!StringUtils.isBlank(confVal)) + schemaConfMap.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), config.getProperty(confKey)); + } + + // Get client connection specific configuration settings + for (Iterator it = config.getKeys(CLIENT_CONF_PREFIX); it.hasNext(); ) { + String confKey = it.next(); + String confVal = config.getProperty(confKey).toString(); + if (!StringUtils.isBlank(confVal)) + clientConfMap.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), config.getProperty(confKey)); + } + + // Get producer specific configuration settings + for (Iterator it = config.getKeys(PRODUCER_CONF_PREFIX); it.hasNext(); ) { + String confKey = it.next(); + String confVal = config.getProperty(confKey).toString(); + if (!StringUtils.isBlank(confVal)) + producerConfMap.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); + } + + // Get consumer specific configuration settings + for (Iterator it = config.getKeys(CONSUMER_CONF_PREFIX); it.hasNext(); ) { + String confKey = it.next(); + String confVal = config.getProperty(confKey).toString(); + if (!StringUtils.isBlank(confVal)) + consumerConfMap.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); + } + + // Get reader specific configuration settings + for (Iterator it = config.getKeys(READER_CONF_PREFIX); it.hasNext(); ) { + String confKey = it.next(); + String confVal = config.getProperty(confKey).toString(); + if (!StringUtils.isBlank(confVal)) + readerConfMap.put(confKey.substring(READER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); + } + } catch (IOException ioe) { + logger.error("Can't read the specified config properties file!"); + ioe.printStackTrace(); + } catch (ConfigurationException cex) { + logger.error("Error loading configuration items from the specified config properties file: " + canonicalFilePath); + cex.printStackTrace(); + } + } + + + ////////////////// + // Get Schema related config + public Map getSchemaConfMap() { + return this.schemaConfMap; + } + public boolean hasSchemaConfKey(String key) { + if (key.contains(SCHEMA_CONF_PREFIX)) + return schemaConfMap.containsKey(key.substring(SCHEMA_CONF_PREFIX.length() + 1)); + else + return schemaConfMap.containsKey(key); + } + public Object getSchemaConfValue(String key) { + if (key.contains(SCHEMA_CONF_PREFIX)) + return schemaConfMap.get(key.substring(SCHEMA_CONF_PREFIX.length()+1)); + else + return schemaConfMap.get(key); + } + public void setSchemaConfValue(String key, Object value) { + if (key.contains(SCHEMA_CONF_PREFIX)) + schemaConfMap.put(key.substring(SCHEMA_CONF_PREFIX.length() + 1), value); + else + schemaConfMap.put(key, value); + } + + + ////////////////// + // Get Pulsar client related config + public Map getClientConfMap() { + return this.clientConfMap; + } + public boolean hasClientConfKey(String key) { + if (key.contains(CLIENT_CONF_PREFIX)) + return clientConfMap.containsKey(key.substring(CLIENT_CONF_PREFIX.length() + 1)); + else + return clientConfMap.containsKey(key); + } + public Object getClientConfValue(String key) { + if (key.contains(CLIENT_CONF_PREFIX)) + return clientConfMap.get(key.substring(CLIENT_CONF_PREFIX.length()+1)); + else + return clientConfMap.get(key); + } + public void setClientConfValue(String key, Object value) { + if (key.contains(CLIENT_CONF_PREFIX)) + clientConfMap.put(key.substring(CLIENT_CONF_PREFIX.length() + 1), value); + else + clientConfMap.put(key, value); + } + + + ////////////////// + // Get Pulsar producer related config + public Map getProducerConfMap() { + return this.producerConfMap; + } + public boolean hasProducerConfKey(String key) { + if (key.contains(PRODUCER_CONF_PREFIX)) + return producerConfMap.containsKey(key.substring(PRODUCER_CONF_PREFIX.length() + 1)); + else + return producerConfMap.containsKey(key); + } + public Object getProducerConfValue(String key) { + if (key.contains(PRODUCER_CONF_PREFIX)) + return producerConfMap.get(key.substring(PRODUCER_CONF_PREFIX.length()+1)); + else + return producerConfMap.get(key); + } + public void setProducerConfValue(String key, Object value) { + if (key.contains(PRODUCER_CONF_PREFIX)) + producerConfMap.put(key.substring(PRODUCER_CONF_PREFIX.length()+1), value); + else + producerConfMap.put(key, value); + } + // other producer helper functions ... + public String getProducerName() { + Object confValue = getProducerConfValue( + "producer." + PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.producerName.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getProducerTopicName() { + Object confValue = getProducerConfValue( + "producer." + PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.topicName); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + + + ////////////////// + // Get Pulsar consumer related config + public Map getConsumerConfMap() { + return this.consumerConfMap; + } + public boolean hasConsumerConfKey(String key) { + if (key.contains(CONSUMER_CONF_PREFIX)) + return consumerConfMap.containsKey(key.substring(CONSUMER_CONF_PREFIX.length() + 1)); + else + return consumerConfMap.containsKey(key); + } + public Object getConsumerConfValue(String key) { + if (key.contains(CONSUMER_CONF_PREFIX)) + return consumerConfMap.get(key.substring(CONSUMER_CONF_PREFIX.length() + 1)); + else + return consumerConfMap.get(key); + } + public void setConsumerConfValue(String key, Object value) { + if (key.contains(CONSUMER_CONF_PREFIX)) + consumerConfMap.put(key.substring(CONSUMER_CONF_PREFIX.length() + 1), value); + else + consumerConfMap.put(key, value); + } + // Other consumer helper functions ... + public String getConsumerTopicNames() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicNames.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getConsumerTopicPattern() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getConsumerSubscriptionName() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getConsumerSubscriptionType() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getConsumerName() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.consumerName.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + // NOTE: Below are not a standard Pulsar consumer configuration parameter as + // listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer" + // They're custom-made configuration properties for NB pulsar driver consumer. + public int getConsumerTimeoutSeconds() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label); + if (confValue == null) + return -1; // infinite + else + return Integer.parseInt(confValue.toString()); + } + + ////////////////// + // Get Pulsar reader related config + public Map getReaderConfMap() { + return this.readerConfMap; + } + public boolean hasReaderConfKey(String key) { + if (key.contains(READER_CONF_PREFIX)) + return readerConfMap.containsKey(key.substring(READER_CONF_PREFIX.length() + 1)); + else + return readerConfMap.containsKey(key); + } + public Object getReaderConfValue(String key) { + if (key.contains(READER_CONF_PREFIX)) + return readerConfMap.get(key.substring(READER_CONF_PREFIX.length() + 1)); + else + return readerConfMap.get(key); + } + public void setReaderConfValue(String key, Object value) { + if (key.contains(READER_CONF_PREFIX)) + readerConfMap.put(key.substring(READER_CONF_PREFIX.length() + 1), value); + else + readerConfMap.put(key, value); + } + // Other reader helper functions ... + public String getReaderTopicName() { + Object confValue = getReaderConfValue( + "reader." + PulsarAdapterUtil.READER_CONF_STD_KEY.topicName.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getReaderName() { + Object confValue = getReaderConfValue( + "reader." + PulsarAdapterUtil.READER_CONF_STD_KEY.readerName.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + // NOTE: Below are not a standard Pulsar reader configuration parameter as + // listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#reader" + // They're custom-made configuration properties for NB pulsar driver reader. + public String getStartMsgPosStr() { + Object confValue = getReaderConfValue( + "reader." + PulsarAdapterUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } +} diff --git a/adapter-pulsar/src/main/resources/admin-namespace.yaml b/adapter-pulsar/src/main/resources/admin-namespace.yaml new file mode 100644 index 000000000..b80e9cff2 --- /dev/null +++ b/adapter-pulsar/src/main/resources/admin-namespace.yaml @@ -0,0 +1,16 @@ +bindings: + # 20 topics: 10 tenants, 2 namespaces/tenant + tenant: Mod(20); Div(2); ToString(); Prefix("tnt") + namespace: Mod(2); ToString(); Prefix("ns") + +params: + async_api: "false" + admin_delop: "true" + +blocks: + admin-namespace-block: + tags: + phase: admin-namespace + ops: + op1: + AdminNamespace: "{tenant}/{namespace}" diff --git a/adapter-pulsar/src/main/resources/admin-tenant.yaml b/adapter-pulsar/src/main/resources/admin-tenant.yaml new file mode 100644 index 000000000..2278e8c41 --- /dev/null +++ b/adapter-pulsar/src/main/resources/admin-tenant.yaml @@ -0,0 +1,17 @@ +bindings: + # 10 tenants + tenant: Mod(10); ToString(); Prefix("tnt") + +params: + async_api: "false" + admin_delop: "true" + +blocks: + admin-tenant-block: + tags: + phase: admin-tenant + ops: + op1: + AdminTenant: "{tenant}" + admin_roles: "" + allowed_clusters: "" diff --git a/adapter-pulsar/src/main/resources/admin-topic.yaml b/adapter-pulsar/src/main/resources/admin-topic.yaml new file mode 100644 index 000000000..a5f81d0dd --- /dev/null +++ b/adapter-pulsar/src/main/resources/admin-topic.yaml @@ -0,0 +1,19 @@ +bindings: + # 100 topics: 10 tenants, 2 namespaces/tenant, 5 topics/namespace + tenant: Mod(100); Div(10); ToString(); Prefix("tnt") + namespace: Mod(10); Div(5); ToString(); Prefix("ns") + topic: Mod(5); ToString(); Prefix("tp") + +params: + async_api: "false" + admin_delop: "true" + +blocks: + admin-topic-block: + tags: + phase: admin-topic + ops: + op1: + AdminTopic: "{tenant}/{namespace}/{topic}" + enable_partition: "true" + partition_num: "5" diff --git a/adapter-pulsar/src/main/resources/bindingtest.yaml b/adapter-pulsar/src/main/resources/bindingtest.yaml new file mode 100644 index 000000000..e687f5fa4 --- /dev/null +++ b/adapter-pulsar/src/main/resources/bindingtest.yaml @@ -0,0 +1,4 @@ +bindings: + tenant: Mod(100); Div(10); ToString(); Prefix("tnt") + namespace: Mod(10); Div(5); ToString(); Prefix("ns") + topic: Mod(5); ToString(); Prefix("tp") diff --git a/adapter-pulsar/src/main/resources/config.properties b/adapter-pulsar/src/main/resources/config.properties new file mode 100644 index 000000000..37be64384 --- /dev/null +++ b/adapter-pulsar/src/main/resources/config.properties @@ -0,0 +1,52 @@ +### Schema related configurations - schema.xxx +# valid types: +# - primitive type (https://pulsar.apache.org/docs/en/schema-understand/#primitive-type) +# - keyvalue (https://pulsar.apache.org/docs/en/schema-understand/#keyvalue) +# - strut (complex type) (https://pulsar.apache.org/docs/en/schema-understand/#struct) +# avro, json, protobuf +# +# TODO: as a starting point, only supports the following types +# 1) primitive types, including bytearray (byte[]) which is default, for messages without schema +# 2) Avro for messages with schema +#schema.type=avro +#schema.definition=file:///Users/yabinmeng/DataStax/MyNoSQLBench/nosqlbench/driver-pulsar/src/main/resources/activities/iot-example.avsc +schema.type= +schema.definition= + + +### Pulsar client related configurations - client.xxx +# http://pulsar.apache.org/docs/en/client-libraries-java/#client +client.connectionTimeoutMs=5000 +client.authPluginClassName=org.apache.pulsar.client.impl.auth.AuthenticationToken +# Cluster admin +client.authParams= +client.tlsAllowInsecureConnection=true + + +### Producer related configurations (global) - producer.xxx +# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer +producer.producerName= +producer.topicName= +producer.sendTimeoutMs= +producer.blockIfQueueFull=true +producer.maxPendingMessages=5000 +producer.batchingMaxMessages=5000 + + +### Consumer related configurations (global) - consumer.xxx +# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer +consumer.topicNames= +consumer.topicsPattern= +consumer.subscriptionName= +consumer.subscriptionType= +consumer.consumerName= +consumer.receiverQueueSize= + + +### Reader related configurations (global) - reader.xxx +# https://pulsar.apache.org/docs/en/client-libraries-java/#reader +# - valid Pos: earliest, latest, custom::file://// +reader.topicName= +reader.receiverQueueSize= +reader.readerName= +reader.startMessagePos=earliest diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsAction.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsAction.java deleted file mode 100644 index 3a3ef9b2a..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsAction.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms; - -import com.codahale.metrics.Timer; -import io.nosqlbench.driver.jms.ops.JmsOp; -import io.nosqlbench.engine.api.activityapi.core.SyncAction; -import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorDetail; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.function.LongFunction; - -public class JmsAction implements SyncAction { - - private final static Logger logger = LogManager.getLogger(JmsAction.class); - - private final JmsActivity activity; - private final int slot; - - int maxTries; - - public JmsAction(JmsActivity activity, int slot) { - this.activity = activity; - this.slot = slot; - this.maxTries = activity.getActivityDef().getParams().getOptionalInteger("maxtries").orElse(10); - } - - @Override - public void init() { } - - @Override - public int runCycle(long cycle) { - // let's fail the action if some async operation failed - activity.failOnAsyncOperationFailure(); - - long start = System.nanoTime(); - - JmsOp jmsOp; - try (Timer.Context ctx = activity.getBindTimer().time()) { - LongFunction readyJmsOp = activity.getSequencer().apply(cycle); - jmsOp = readyJmsOp.apply(cycle); - } catch (Exception bindException) { - // if diagnostic mode ... - activity.getErrorhandler().handleError(bindException, cycle, 0); - throw new RuntimeException( - "while binding request in cycle " + cycle + ": " + bindException.getMessage(), bindException - ); - } - - for (int i = 0; i < maxTries; i++) { - Timer.Context ctx = activity.getExecuteTimer().time(); - try { - // it is up to the jmsOp to call Context#close when the activity is executed - // this allows us to track time for async operations - jmsOp.run(ctx::close); - break; - } catch (RuntimeException err) { - ErrorDetail errorDetail = activity - .getErrorhandler() - .handleError(err, cycle, System.nanoTime() - start); - if (!errorDetail.isRetryable()) { - break; - } - } - } - - return 0; - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java deleted file mode 100644 index 2da3a440c..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.Timer; -import com.datastax.oss.pulsar.jms.PulsarConnectionFactory; -import io.nosqlbench.driver.jms.conn.JmsConnInfo; -import io.nosqlbench.driver.jms.conn.JmsPulsarConnInfo; -import io.nosqlbench.driver.jms.ops.JmsOp; -import io.nosqlbench.driver.jms.util.JmsUtil; -import io.nosqlbench.driver.jms.util.PulsarConfig; -import io.nosqlbench.engine.api.activityapi.errorhandling.modular.NBErrorHandler; -import io.nosqlbench.engine.api.activityapi.planning.OpSequence; -import io.nosqlbench.api.engine.activityimpl.ActivityDef; -import io.nosqlbench.engine.api.activityimpl.OpDispenser; -import io.nosqlbench.engine.api.activityimpl.SimpleActivity; -import io.nosqlbench.api.engine.metrics.ActivityMetrics; -import org.apache.commons.lang3.StringUtils; - -import javax.jms.Destination; -import javax.jms.JMSContext; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - - -public class JmsActivity extends SimpleActivity { - - private final ConcurrentHashMap jmsDestinations = new ConcurrentHashMap<>(); - - private String jmsProviderType; - private JmsConnInfo jmsConnInfo; - - private JMSContext jmsContext; - - private OpSequence> sequence; - private volatile Throwable asyncOperationFailure; - private NBErrorHandler errorhandler; - - private Timer bindTimer; - private Timer executeTimer; - private Counter bytesCounter; - private Histogram messagesizeHistogram; - - public JmsActivity(ActivityDef activityDef) { - super(activityDef); - } - - @Override - public void initActivity() { - super.initActivity(); - - // default JMS type: Pulsar - // - currently this is the only supported JMS provider - jmsProviderType = - activityDef.getParams() - .getOptionalString(JmsUtil.JMS_PROVIDER_TYPE_KEY_STR) - .orElse(JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label); - - // "Pulsar" as the JMS provider - if (StringUtils.equalsIgnoreCase(jmsProviderType, JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label )) { - - String webSvcUrl = - activityDef.getParams() - .getOptionalString(JmsUtil.JMS_PULSAR_PROVIDER_WEB_URL_KEY_STR) - .orElse("http://localhost:8080"); - String pulsarSvcUrl = - activityDef.getParams() - .getOptionalString(JmsUtil.JMS_PULSAR_PROVIDER_SVC_URL_KEY_STR) - .orElse("pulsar://localhost:6650"); - - if (StringUtils.isAnyBlank(webSvcUrl, pulsarSvcUrl)) { - throw new RuntimeException("For \"" + JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label + "\" type, " + - "\"" + JmsUtil.JMS_PULSAR_PROVIDER_WEB_URL_KEY_STR + "\" and " + - "\"" + JmsUtil.JMS_PULSAR_PROVIDER_SVC_URL_KEY_STR + "\" parameters are manadatory!"); - } - - // Check if extra Pulsar config. file is in place - // - default file: "pulsar_config.properties" under the current directory - String pulsarCfgFile = - activityDef.getParams() - .getOptionalString(JmsUtil.JMS_PULSAR_PROVIDER_CFG_FILE_KEY_STR) - .orElse(JmsUtil.JMS_PULSAR_PROVIDER_DFT_CFG_FILE_NAME); - - PulsarConfig pulsarConfig = new PulsarConfig(pulsarCfgFile); - - jmsConnInfo = new JmsPulsarConnInfo(jmsProviderType, webSvcUrl, pulsarSvcUrl, pulsarConfig); - } - else { - throw new RuntimeException("Unsupported JMS driver type : " + jmsProviderType); - } - - PulsarConnectionFactory factory; - factory = new PulsarConnectionFactory(jmsConnInfo.getJmsConnConfig()); - this.jmsContext = factory.createContext(); - - bindTimer = ActivityMetrics.timer(activityDef, "bind", this.getHdrDigits()); - executeTimer = ActivityMetrics.timer(activityDef, "execute", this.getHdrDigits()); - bytesCounter = ActivityMetrics.counter(activityDef, "bytes"); - messagesizeHistogram = ActivityMetrics.histogram(activityDef, "messagesize", this.getHdrDigits()); - - if (StringUtils.equalsIgnoreCase(jmsProviderType, JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label )) { - this.sequence = createOpSequence((ot) -> new ReadyPulsarJmsOp(ot, this), false, Optional.empty()); - } - - setDefaultsFromOpSequence(sequence); - onActivityDefUpdate(activityDef); - - this.errorhandler = new NBErrorHandler( - () -> activityDef.getParams().getOptionalString("errors").orElse("stop"), - this::getExceptionMetrics - ); - } - - private static String buildCacheKey(String... keyParts) { - return String.join("::", keyParts); - } - - /** - * If the JMS destination that corresponds to a topic exists, reuse it; Otherwise, create it - */ - public Destination getOrCreateJmsDestination(String jmsDestinationType, String destName) { - String destinationCacheKey = buildCacheKey(jmsDestinationType, destName); - Destination destination = jmsDestinations.get(destinationCacheKey); - - if ( destination == null ) { - // TODO: should we match Persistent/Non-peristent JMS Delivery mode with - // Pulsar Persistent/Non-prsistent topic? - if (StringUtils.equalsIgnoreCase(jmsDestinationType, JmsUtil.JMS_DESTINATION_TYPES.QUEUE.label)) { - destination = jmsContext.createQueue(destName); - } else if (StringUtils.equalsIgnoreCase(jmsDestinationType, JmsUtil.JMS_DESTINATION_TYPES.TOPIC.label)) { - destination = jmsContext.createTopic(destName); - } - - jmsDestinations.put(destinationCacheKey, destination); - } - - return destination; - } - - @Override - public synchronized void onActivityDefUpdate(ActivityDef activityDef) { super.onActivityDefUpdate(activityDef); } - public OpSequence> getSequencer() { return sequence; } - - public String getJmsProviderType() { return jmsProviderType; } - public JmsConnInfo getJmsConnInfo() { return jmsConnInfo; } - public JMSContext getJmsContext() { return jmsContext; } - - public Timer getBindTimer() { return bindTimer; } - public Timer getExecuteTimer() { return this.executeTimer; } - public Counter getBytesCounter() { return bytesCounter; } - public Histogram getMessagesizeHistogram() { return messagesizeHistogram; } - - public NBErrorHandler getErrorhandler() { return errorhandler; } - - public void failOnAsyncOperationFailure() { - if (asyncOperationFailure != null) { - throw new RuntimeException(asyncOperationFailure); - } - } - public void asyncOperationFailed(Throwable ex) { - this.asyncOperationFailure = ex; - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivityType.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivityType.java deleted file mode 100644 index b964516ea..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivityType.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms; - -import io.nosqlbench.engine.api.activityapi.core.Action; -import io.nosqlbench.engine.api.activityapi.core.ActionDispenser; -import io.nosqlbench.engine.api.activityapi.core.ActivityType; -import io.nosqlbench.api.engine.activityimpl.ActivityDef; -import io.nosqlbench.nb.annotations.Service; - -@Service(value = ActivityType.class, selector = "jms") -public class JmsActivityType implements ActivityType { - @Override - public ActionDispenser getActionDispenser(JmsActivity activity) { - return new PulsarJmsActionDispenser(activity); - } - - @Override - public JmsActivity getActivity(ActivityDef activityDef) { - return new JmsActivity(activityDef); - } - - private static class PulsarJmsActionDispenser implements ActionDispenser { - private final JmsActivity activity; - public PulsarJmsActionDispenser(JmsActivity activity) { - this.activity = activity; - } - - @Override - public Action getAction(int slot) { - return new JmsAction(activity, slot); - } - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyJmsOp.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyJmsOp.java deleted file mode 100644 index bd532033f..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyJmsOp.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms; - -import io.nosqlbench.driver.jms.ops.JmsOp; -import io.nosqlbench.driver.jms.util.JmsUtil; -import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser; -import io.nosqlbench.engine.api.templating.CommandTemplate; -import org.apache.commons.lang3.BooleanUtils; - -import java.util.function.LongFunction; - -abstract public class ReadyJmsOp extends BaseOpDispenser { - - protected final OpTemplate optpl; - protected final CommandTemplate cmdTpl; - protected final JmsActivity jmsActivity; - - protected final String stmtOpType; - protected LongFunction asyncApiFunc; - protected LongFunction jmsDestinationTypeFunc; - - protected final LongFunction opFunc; - - public ReadyJmsOp(OpTemplate opTemplate, JmsActivity jmsActivity) { - super(opTemplate); - this.optpl = opTemplate; - this.cmdTpl = new CommandTemplate(optpl); - this.jmsActivity = jmsActivity; - - if (!cmdTpl.containsKey("optype") || !cmdTpl.isStatic("optype")) { - throw new RuntimeException("Statement parameter \"optype\" must be static and have a valid value!"); - } - this.stmtOpType = cmdTpl.getStatic("optype"); - - // Global/Doc-level parameter: async_api - if (cmdTpl.containsKey(JmsUtil.ASYNC_API_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.ASYNC_API_KEY_STR)) { - boolean value = BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.ASYNC_API_KEY_STR)); - this.asyncApiFunc = (l) -> value; - } else { - throw new RuntimeException("\"" + JmsUtil.ASYNC_API_KEY_STR + "\" parameter cannot be dynamic!"); - } - } - - // Global/Doc-level parameter: jms_desitation_type - // - queue: point-to-point - // - topic: pub/sub - if (cmdTpl.containsKey(JmsUtil.JMS_DESTINATION_TYPE_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_DESTINATION_TYPE_KEY_STR)) { - jmsDestinationTypeFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_DESTINATION_TYPE_KEY_STR); - } else { - throw new RuntimeException("\"" + JmsUtil.JMS_DESTINATION_TYPE_KEY_STR + "\" parameter cannot be dynamic!"); - } - } - - this.opFunc = resolveJms(); - } - - public JmsOp apply(long value) { return opFunc.apply(value); } - - abstract LongFunction resolveJms(); -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyPulsarJmsOp.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyPulsarJmsOp.java deleted file mode 100644 index 72c397a97..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyPulsarJmsOp.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms; - -import io.nosqlbench.driver.jms.ops.JmsMsgReadMapper; -import io.nosqlbench.driver.jms.ops.JmsMsgSendMapper; -import io.nosqlbench.driver.jms.ops.JmsOp; -import io.nosqlbench.driver.jms.util.JmsHeaderLongFunc; -import io.nosqlbench.driver.jms.util.JmsUtil; -import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; - -import javax.jms.DeliveryMode; -import javax.jms.Destination; -import javax.jms.JMSRuntimeException; -import javax.jms.Message; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.function.LongFunction; -import java.util.stream.Collectors; - -public class ReadyPulsarJmsOp extends ReadyJmsOp { - - public ReadyPulsarJmsOp(OpTemplate opTemplate, JmsActivity jmsActivity) { - super(opTemplate, jmsActivity); - } - - public LongFunction resolveJms() { - // Global/Doc-level parameter: topic_uri - LongFunction topicUriFunc = (l) -> null; - if (cmdTpl.containsKey(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR)) { - topicUriFunc = (l) -> cmdTpl.getStatic(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR); - } else { - topicUriFunc = (l) -> cmdTpl.getDynamic(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR, l); - } - } - - // Global: JMS destination - LongFunction jmsDestinationFunc; - try { - LongFunction finalTopicUriFunc = topicUriFunc; - jmsDestinationFunc = (l) -> jmsActivity.getOrCreateJmsDestination( - jmsDestinationTypeFunc.apply(l), - finalTopicUriFunc.apply(l)); - } - catch (JMSRuntimeException ex) { - throw new RuntimeException("Unable to create JMS destination!"); - } - - if (StringUtils.equalsIgnoreCase(stmtOpType, JmsUtil.OP_TYPES.MSG_SEND.label)) { - return resolveMsgSend(asyncApiFunc, jmsDestinationFunc); - } else if (StringUtils.equalsIgnoreCase(stmtOpType, JmsUtil.OP_TYPES.MSG_READ.label)) { - return resolveMsgRead(asyncApiFunc, jmsDestinationFunc); - } else { - throw new RuntimeException("Unsupported JMS operation type"); - } - } - - private LongFunction resolveMsgSend( - LongFunction async_api_func, - LongFunction jmsDestinationFunc - ) { - JmsHeaderLongFunc jmsHeaderLongFunc = new JmsHeaderLongFunc(); - - // JMS header: delivery mode - LongFunction msgDeliveryModeFunc = (l) -> DeliveryMode.PERSISTENT; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label)) { - msgDeliveryModeFunc = (l) -> NumberUtils.toInt(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label)); - } - else { - msgDeliveryModeFunc = (l) -> NumberUtils.toInt(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label, l)); - } - } - jmsHeaderLongFunc.setDeliveryModeFunc(msgDeliveryModeFunc); - - // JMS header: message priority - LongFunction msgPriorityFunc = (l) -> Message.DEFAULT_PRIORITY; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label)) { - msgPriorityFunc = (l) -> NumberUtils.toInt(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label)); - } - else { - msgPriorityFunc = (l) -> NumberUtils.toInt(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label, l)); - } - } - jmsHeaderLongFunc.setMsgPriorityFunc(msgPriorityFunc); - - // JMS header: message TTL - LongFunction msgTtlFunc = (l) -> Message.DEFAULT_TIME_TO_LIVE; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label)) { - msgTtlFunc = (l) -> NumberUtils.toLong(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label)); - } - else { - msgTtlFunc = (l) -> NumberUtils.toLong(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label, l)); - } - } - jmsHeaderLongFunc.setMsgTtlFunc(msgTtlFunc); - - // JMS header: message delivery delay - LongFunction msgDeliveryDelayFunc = (l) -> Message.DEFAULT_DELIVERY_DELAY; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label)) { - msgDeliveryDelayFunc = (l) -> NumberUtils.toLong(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label)); - } - else { - msgDeliveryDelayFunc = (l) -> NumberUtils.toLong(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label, l)); - } - } - jmsHeaderLongFunc.setMsgDeliveryDelayFunc(msgDeliveryDelayFunc); - - // JMS header: disable message timestamp - LongFunction disableMsgTimestampFunc = (l) -> false; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label)) { - disableMsgTimestampFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label)); - } - else { - disableMsgTimestampFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label, l)); - } - } - jmsHeaderLongFunc.setDisableMsgTimestampFunc(disableMsgTimestampFunc); - - // JMS header: disable message ID - LongFunction disableMsgIdFunc = (l) -> false; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label)) { - disableMsgIdFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label)); - } - else { - disableMsgIdFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label, l)); - } - } - jmsHeaderLongFunc.setDisableMsgIdFunc(disableMsgIdFunc); - - // JMS message properties - String jmsMsgPropertyListStr = ""; - if (cmdTpl.containsKey(JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR)) { - jmsMsgPropertyListStr = cmdTpl.getStatic(JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR); - } else { - throw new RuntimeException("\"" + JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR + "\" parameter cannot be dynamic!"); - } - } - - Map jmsMsgProperties = new HashMap<>(); - if ( !StringUtils.isEmpty(jmsMsgPropertyListStr) ) { - jmsMsgProperties = Arrays.stream(jmsMsgPropertyListStr.split(";")) - .map(s -> s.split("=", 2)) - .collect(Collectors.toMap(a -> a[0], a -> a.length > 1 ? a[1] : "")); - } - - LongFunction msgBodyFunc; - if (cmdTpl.containsKey(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR)) { - msgBodyFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR)) { - msgBodyFunc = (l) -> cmdTpl.getDynamic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR, l); - } else { - msgBodyFunc = (l) -> null; - } - } else { - throw new RuntimeException("JMS message send:: \"msg_body\" field must be specified!"); - } - - return new JmsMsgSendMapper( - jmsActivity, - async_api_func, - jmsDestinationFunc, - jmsHeaderLongFunc, - jmsMsgProperties, - msgBodyFunc); - } - - private LongFunction resolveMsgRead( - LongFunction async_api_func, - LongFunction jmsDestinationFunc - ) { - // For Pulsar JMS, make "durable" as the default - LongFunction jmsConsumerDurableFunc = (l) -> true; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR)) { - jmsConsumerDurableFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR)); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR)) { - jmsConsumerDurableFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR, l)); - } - } - - LongFunction jmsConsumerSharedFunc = (l) -> true; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR)) { - jmsConsumerSharedFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR)); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR)) { - jmsConsumerSharedFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR, l)); - } - } - - LongFunction jmsMsgSubscriptionFunc = (l) -> ""; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR)) { - jmsMsgSubscriptionFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR)) { - jmsMsgSubscriptionFunc = (l) -> cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR, l); - } - } - - LongFunction jmsMsgReadSelectorFunc = (l) -> ""; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR)) { - jmsMsgReadSelectorFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR)) { - jmsMsgReadSelectorFunc = (l) -> cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR, l); - } - } - - LongFunction jmsMsgNoLocalFunc = (l) -> true; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR)) { - jmsMsgNoLocalFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR)); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR)) { - jmsMsgNoLocalFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR, l)); - } - } - - LongFunction jmsReadTimeoutFunc = (l) -> 0L; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR)) { - jmsReadTimeoutFunc = (l) -> NumberUtils.toLong(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR)); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR)) { - jmsReadTimeoutFunc = (l) -> NumberUtils.toLong(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR, l)); - } - } - - return new JmsMsgReadMapper( - jmsActivity, - async_api_func, - jmsDestinationFunc, - jmsConsumerDurableFunc, - jmsConsumerSharedFunc, - jmsMsgSubscriptionFunc, - jmsMsgReadSelectorFunc, - jmsMsgNoLocalFunc, - jmsReadTimeoutFunc); - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsConnInfo.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsConnInfo.java deleted file mode 100644 index 738f83678..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsConnInfo.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.conn; - -import java.util.HashMap; -import java.util.Map; - -public class JmsConnInfo { - - protected final String jmsProviderType; - protected final Map jmsConnConfig; - - protected JmsConnInfo(String jmsProviderType) { - this.jmsProviderType = jmsProviderType; - this.jmsConnConfig = new HashMap<>(); - } - - public Map getJmsConnConfig() { return this.jmsConnConfig; } - public void resetJmsConnConfig() { this.jmsConnConfig.clear(); } - public void addJmsConnConfigItems(Map cfgItems) { this.jmsConnConfig.putAll(cfgItems); } - public void addJmsConnConfigItem(String key, Object value) { this.jmsConnConfig.put(key, value); } - public void removeJmsConnConfigItem(String key) { this.jmsConnConfig.remove(key); } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsPulsarConnInfo.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsPulsarConnInfo.java deleted file mode 100644 index f1e09fe78..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsPulsarConnInfo.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.conn; - -import io.nosqlbench.driver.jms.util.PulsarConfig; - -import java.util.Map; - -public class JmsPulsarConnInfo extends JmsConnInfo { - - private final String webSvcUrl; - private final String pulsarSvcUrl; - private final PulsarConfig extraPulsarConfig; - - public JmsPulsarConnInfo(String jmsProviderType, String webSvcUrl, String pulsarSvcUrl, PulsarConfig pulsarConfig) { - super(jmsProviderType); - - this.webSvcUrl = webSvcUrl; - this.pulsarSvcUrl = pulsarSvcUrl; - this.extraPulsarConfig = pulsarConfig; - - this.addJmsConnConfigItem("webServiceUrl", this.webSvcUrl); - this.addJmsConnConfigItem("brokerServiceUrl", this.pulsarSvcUrl); - - Map clientCfgMap = this.extraPulsarConfig.getClientConfMap(); - if (!clientCfgMap.isEmpty()) { - this.addJmsConnConfigItems(clientCfgMap); - } - - Map producerCfgMap = this.extraPulsarConfig.getProducerConfMap(); - if (!producerCfgMap.isEmpty()) { - this.addJmsConnConfigItem("producerConfig", producerCfgMap); - } - - Map consumerCfgMap = this.extraPulsarConfig.getConsumerConfMap(); - if (!consumerCfgMap.isEmpty()) { - this.addJmsConnConfigItem("consumerConfig", consumerCfgMap); - } - } - - public String getWebSvcUrl() { return this.webSvcUrl; } - public String getPulsarSvcUrl() { return this.pulsarSvcUrl; } - public PulsarConfig getExtraPulsarConfig() { return this.extraPulsarConfig; } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadMapper.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadMapper.java deleted file mode 100644 index 0108c0608..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadMapper.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.ops; - -import io.nosqlbench.driver.jms.JmsActivity; - -import javax.jms.Destination; -import java.util.function.LongFunction; - -/** - * This maps a set of specifier functions to a pulsar operation. The pulsar operation contains - * enough state to define a pulsar operation such that it can be executed, measured, and possibly - * retried if needed. - * - * This function doesn't act *as* the operation. It merely maps the construction logic into - * a simple functional type, given the component functions. - * - * For additional parameterization, the command template is also provided. - */ -public class JmsMsgReadMapper extends JmsOpMapper { - - private final LongFunction jmsConsumerDurableFunc; - private final LongFunction jmsConsumerSharedFunc; - private final LongFunction jmsMsgSubscriptionFunc; - private final LongFunction jmsMsgReadSelectorFunc; - private final LongFunction jmsMsgNoLocalFunc; - private final LongFunction jmsReadTimeoutFunc; - - public JmsMsgReadMapper(JmsActivity jmsActivity, - LongFunction asyncApiFunc, - LongFunction jmsDestinationFunc, - LongFunction jmsConsumerDurableFunc, - LongFunction jmsConsumerSharedFunc, - LongFunction jmsMsgSubscriptionFunc, - LongFunction jmsMsgReadSelectorFunc, - LongFunction jmsMsgNoLocalFunc, - LongFunction jmsReadTimeoutFunc) { - super(jmsActivity, asyncApiFunc, jmsDestinationFunc); - - this.jmsConsumerDurableFunc = jmsConsumerDurableFunc; - this.jmsConsumerSharedFunc = jmsConsumerSharedFunc; - this.jmsMsgSubscriptionFunc = jmsMsgSubscriptionFunc; - this.jmsMsgReadSelectorFunc = jmsMsgReadSelectorFunc; - this.jmsMsgNoLocalFunc = jmsMsgNoLocalFunc; - this.jmsReadTimeoutFunc = jmsReadTimeoutFunc; - } - - @Override - public JmsOp apply(long value) { - boolean asyncApi = asyncApiFunc.apply(value); - Destination jmsDestination = jmsDestinationFunc.apply(value); - boolean jmsConsumerDurable = jmsConsumerDurableFunc.apply(value); - boolean jmsConsumerShared = jmsConsumerSharedFunc.apply(value); - String jmsMsgSubscription = jmsMsgSubscriptionFunc.apply(value); - String jmsMsgReadSelector = jmsMsgReadSelectorFunc.apply(value); - boolean jmsMsgNoLocal = jmsMsgNoLocalFunc.apply(value); - long jmsReadTimeout = jmsReadTimeoutFunc.apply(value); - - // Default to NO read timeout - if (jmsReadTimeout < 0) jmsReadTimeout = 0; - - return new JmsMsgReadOp( - jmsActivity, - asyncApi, - jmsDestination, - jmsConsumerDurable, - jmsConsumerShared, - jmsMsgSubscription, - jmsMsgReadSelector, - jmsMsgNoLocal, - jmsReadTimeout - ); - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadOp.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadOp.java deleted file mode 100644 index de92f73c5..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadOp.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.ops; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Histogram; -import io.nosqlbench.driver.jms.JmsActivity; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import javax.jms.*; - -public class JmsMsgReadOp extends JmsTimeTrackOp { - - private final static Logger logger = LogManager.getLogger(JmsMsgReadOp.class); - - private final JmsActivity jmsActivity; - private final boolean asyncJmsOp; - private final Destination jmsDestination; - - private final JMSContext jmsContext; - private final JMSConsumer jmsConsumer; - private final boolean jmsConsumerDurable; - private final boolean jmsConsumerShared; - private final String jmsMsgSubscrption; - private final String jmsMsgReadSelector; - private final boolean jmsMsgNoLocal; - private final long jmsReadTimeout; - - private final Counter bytesCounter; - private final Histogram messagesizeHistogram; - - public JmsMsgReadOp(JmsActivity jmsActivity, - boolean asyncJmsOp, - Destination jmsDestination, - boolean jmsConsumerDurable, - boolean jmsConsumerShared, - String jmsMsgSubscrption, - String jmsMsgReadSelector, - boolean jmsMsgNoLocal, - long jmsReadTimeout) { - this.jmsActivity = jmsActivity; - this.asyncJmsOp = asyncJmsOp; - this.jmsDestination = jmsDestination; - this.jmsConsumerDurable = jmsConsumerDurable; - this.jmsConsumerShared = jmsConsumerShared; - this.jmsMsgReadSelector = jmsMsgReadSelector; - this.jmsMsgSubscrption = jmsMsgSubscrption; - this.jmsMsgNoLocal = jmsMsgNoLocal; - this.jmsReadTimeout = jmsReadTimeout; - - this.jmsContext = jmsActivity.getJmsContext(); - this.jmsConsumer = createJmsConsumer(); - - this.bytesCounter = jmsActivity.getBytesCounter(); - this.messagesizeHistogram = jmsActivity.getMessagesizeHistogram(); - } - - private JMSConsumer createJmsConsumer() { - JMSConsumer jmsConsumer; - - try { - if (jmsConsumerDurable) { - if (jmsConsumerShared) - jmsConsumer = jmsContext.createSharedDurableConsumer((Topic) jmsDestination, jmsMsgSubscrption, jmsMsgReadSelector); - else - jmsConsumer = jmsContext.createDurableConsumer((Topic) jmsDestination, jmsMsgSubscrption, jmsMsgReadSelector, jmsMsgNoLocal); - } else { - if (jmsConsumerShared) - jmsConsumer = jmsContext.createSharedConsumer((Topic) jmsDestination, jmsMsgSubscrption, jmsMsgReadSelector); - else - jmsConsumer = jmsContext.createConsumer(jmsDestination, jmsMsgReadSelector, jmsMsgNoLocal); - } - } - catch (InvalidDestinationRuntimeException invalidDestinationRuntimeException) { - throw new RuntimeException("Failed to create JMS consumer: invalid destination!"); - } - catch (InvalidSelectorRuntimeException invalidSelectorRuntimeException) { - throw new RuntimeException("Failed to create JMS consumer: invalid message selector!"); - } - catch (JMSRuntimeException jmsRuntimeException) { - jmsRuntimeException.printStackTrace(); - throw new RuntimeException("Failed to create JMS consumer: runtime internal error!"); - } - - // TODO: async consumer -// if (this.asyncJmsOp) { -// jmsConsumer.setMessageListener(); -// } - - return jmsConsumer; - } - - @Override - public void run() { - // FIXME: jmsReadTimeout being 0 behaves like receiveNoWait() instead of waiting indefinitley - Message receivedMsg = jmsConsumer.receive(jmsReadTimeout); - try { - if (receivedMsg != null) { - receivedMsg.acknowledge(); - byte[] receivedMsgBody = receivedMsg.getBody(byte[].class); - - if (logger.isDebugEnabled()) { - logger.debug("received msg-payload={}", new String(receivedMsgBody)); - } - - int messagesize = receivedMsgBody.length; - bytesCounter.inc(messagesize); - messagesizeHistogram.update(messagesize); - } - } catch (JMSException e) { - e.printStackTrace(); - throw new RuntimeException("Failed to acknowledge the received JMS message."); - } - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendMapper.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendMapper.java deleted file mode 100644 index fb649f013..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendMapper.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.ops; - -import io.nosqlbench.driver.jms.JmsActivity; -import io.nosqlbench.driver.jms.util.JmsHeader; -import io.nosqlbench.driver.jms.util.JmsHeaderLongFunc; - -import javax.jms.Destination; -import java.util.Map; -import java.util.function.LongFunction; - -/** - * This maps a set of specifier functions to a pulsar operation. The pulsar operation contains - * enough state to define a pulsar operation such that it can be executed, measured, and possibly - * retried if needed. - * - * This function doesn't act *as* the operation. It merely maps the construction logic into - * a simple functional type, given the component functions. - * - * For additional parameterization, the command template is also provided. - */ -public class JmsMsgSendMapper extends JmsOpMapper { - private final JmsHeaderLongFunc jmsHeaderLongFunc; - private final Map jmsMsgProperties; - private final LongFunction msgBodyFunc; - - public JmsMsgSendMapper(JmsActivity jmsActivity, - LongFunction asyncApiFunc, - LongFunction jmsDestinationFunc, - JmsHeaderLongFunc jmsHeaderLongFunc, - Map jmsMsgProperties, - LongFunction msgBodyFunc) { - super(jmsActivity, asyncApiFunc, jmsDestinationFunc); - - this.jmsHeaderLongFunc = jmsHeaderLongFunc; - this.jmsMsgProperties = jmsMsgProperties; - this.msgBodyFunc = msgBodyFunc; - } - - @Override - public JmsOp apply(long value) { - boolean asyncApi = asyncApiFunc.apply(value); - Destination jmsDestination = jmsDestinationFunc.apply(value); - JmsHeader jmsHeader = (JmsHeader)jmsHeaderLongFunc.apply(value); - String msgBody = msgBodyFunc.apply(value); - - return new JmsMsgSendOp( - jmsActivity, - asyncApi, - jmsDestination, - jmsHeader, - jmsMsgProperties, - msgBody - ); - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendOp.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendOp.java deleted file mode 100644 index 2f432502c..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendOp.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.ops; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Histogram; -import io.nosqlbench.driver.jms.JmsActivity; -import io.nosqlbench.driver.jms.util.JmsHeader; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import javax.jms.*; -import java.nio.charset.StandardCharsets; -import java.util.Iterator; -import java.util.Map; - -public class JmsMsgSendOp extends JmsTimeTrackOp { - - private final static Logger logger = LogManager.getLogger(JmsMsgSendOp.class); - - private final JmsActivity jmsActivity; - private final boolean asyncJmsOp; - private final Destination jmsDestination; - private final JmsHeader jmsHeader; - private final Map jmsMsgProperties; - - private final JMSContext jmsContext; - private final JMSProducer jmsProducer; - private final String msgBody; - - private final Counter bytesCounter; - private final Histogram messagesizeHistogram; - - public JmsMsgSendOp(JmsActivity jmsActivity, - boolean asyncJmsOp, - Destination jmsDestination, - JmsHeader jmsHeader, - Map jmsMsgProperties, - String msgBody) { - this.jmsActivity = jmsActivity; - this.asyncJmsOp = asyncJmsOp; - this.jmsDestination = jmsDestination; - - this.jmsHeader = jmsHeader; - this.jmsMsgProperties = jmsMsgProperties; - this.msgBody = msgBody; - - if (!jmsHeader.isValidHeader()) { - throw new RuntimeException(jmsHeader.getInvalidJmsHeaderMsgText()); - } - - if ((msgBody == null) || msgBody.isEmpty()) { - throw new RuntimeException("JMS message body can't be empty!"); - } - - this.jmsContext = jmsActivity.getJmsContext(); - this.jmsProducer = createJmsProducer(); - - this.bytesCounter = jmsActivity.getBytesCounter(); - this.messagesizeHistogram = jmsActivity.getMessagesizeHistogram(); - } - - private JMSProducer createJmsProducer() { - JMSProducer jmsProducer = this.jmsContext.createProducer(); - - jmsProducer.setDeliveryMode(this.jmsHeader.getDeliveryMode()); - jmsProducer.setPriority(this.jmsHeader.getMsgPriority()); - jmsProducer.setDeliveryDelay(this.jmsHeader.getMsgDeliveryDelay()); - jmsProducer.setDisableMessageTimestamp(this.jmsHeader.isDisableMsgTimestamp()); - jmsProducer.setDisableMessageID(this.jmsHeader.isDisableMsgId()); - - if (this.asyncJmsOp) { - jmsProducer.setAsync(new CompletionListener() { - @Override - public void onCompletion(Message msg) { - try { - byte[] msgBody = msg.getBody(byte[].class); - if (logger.isTraceEnabled()) { - logger.trace("Async message send success - message body: " + new String(msgBody)); - } - } - catch (JMSException jmsException) { - jmsException.printStackTrace(); - logger.warn("Unexpected error when parsing message body: " + jmsException.getMessage()); - } - } - - @Override - public void onException(Message msg, Exception e) { - try { - byte[] msgBody = msg.getBody(byte[].class); - if (logger.isTraceEnabled()) { - logger.trace("Async message send failure - message body: " + new String(msgBody)); - } - } - catch (JMSException jmsException) { - jmsException.printStackTrace(); - logger.warn("Unexpected error when parsing message body: " + jmsException.getMessage()); - } - } - }); - } - - for (Map.Entry entry : jmsMsgProperties.entrySet()) { - jmsProducer.setProperty(entry.getKey(), entry.getValue()); - } - - return jmsProducer; - } - - @Override - public void run() { - try { - byte[] msgBytes = msgBody.getBytes(StandardCharsets.UTF_8); - int messageSize = msgBytes.length; - jmsProducer.send(jmsDestination, msgBytes); - - messagesizeHistogram.update(messageSize); - bytesCounter.inc(messageSize); - } - catch (Exception ex) { - logger.error("Failed to send JMS message - " + msgBody); - } - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOpMapper.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOpMapper.java deleted file mode 100644 index b52385cdd..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOpMapper.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.ops; - -import io.nosqlbench.driver.jms.JmsActivity; -import io.nosqlbench.driver.jms.util.JmsHeaderLongFunc; - -import javax.jms.Destination; -import java.util.Map; -import java.util.function.LongFunction; - -public abstract class JmsOpMapper implements LongFunction { - protected final JmsActivity jmsActivity; - protected final LongFunction asyncApiFunc; - protected final LongFunction jmsDestinationFunc; - - public JmsOpMapper(JmsActivity jmsActivity, - LongFunction asyncApiFunc, - LongFunction jmsDestinationFunc) - { - this.jmsActivity = jmsActivity; - this.asyncApiFunc = asyncApiFunc; - this.jmsDestinationFunc = jmsDestinationFunc; - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeader.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeader.java deleted file mode 100644 index e7d889d8a..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeader.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.util; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import org.apache.commons.lang.StringUtils; - -import javax.jms.DeliveryMode; - -@Setter -@Getter -@AllArgsConstructor -@ToString -public class JmsHeader { - private int deliveryMode; - private int msgPriority; - private long msgTtl; - private long msgDeliveryDelay; - private boolean disableMsgTimestamp; - private boolean disableMsgId; - - public boolean isValidDeliveryMode() { - return (deliveryMode == DeliveryMode.NON_PERSISTENT) || (deliveryMode == DeliveryMode.PERSISTENT); - } - - public boolean isValidPriority() { - return (msgPriority >= 0) && (msgPriority <= 9); - } - - public boolean isValidTtl() { - return msgTtl >= 0; - } - - public boolean isValidDeliveryDelay() { - return msgTtl >= 0; - } - - public boolean isValidHeader() { - return isValidDeliveryMode() - && isValidPriority() - && isValidTtl() - && isValidDeliveryDelay(); - } - - public String getInvalidJmsHeaderMsgText() { - StringBuilder sb = new StringBuilder(); - - if (!isValidDeliveryMode()) - sb.append("delivery mode - " + deliveryMode + "; "); - if (!isValidPriority()) - sb.append("message priority - " + msgPriority + "; "); - if (!isValidTtl()) - sb.append("message TTL - " + msgTtl + "; "); - if (!isValidDeliveryDelay()) - sb.append("message delivery delay - " + msgDeliveryDelay + "; "); - - String invalidMsgText = sb.toString(); - if (StringUtils.length(invalidMsgText) > 0) - invalidMsgText = StringUtils.substringBeforeLast(invalidMsgText, ";"); - else - invalidMsgText = "none"; - - return "Invalid JMS header values: " + invalidMsgText; - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeaderLongFunc.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeaderLongFunc.java deleted file mode 100644 index 75616091a..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeaderLongFunc.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.util; - -import lombok.*; - -import javax.jms.DeliveryMode; -import javax.jms.Message; -import java.util.function.LongFunction; - -@Setter -@Getter -@NoArgsConstructor -public class JmsHeaderLongFunc implements LongFunction { - private LongFunction deliveryModeFunc; - private LongFunction msgPriorityFunc; - private LongFunction msgTtlFunc; - private LongFunction msgDeliveryDelayFunc; - private LongFunction disableMsgTimestampFunc; - private LongFunction disableMsgIdFunc; - - @Override - public Object apply(long value) { - return new JmsHeader( - (deliveryModeFunc != null) ? deliveryModeFunc.apply(value) : DeliveryMode.PERSISTENT, - (msgPriorityFunc != null) ? msgPriorityFunc.apply(value) : Message.DEFAULT_PRIORITY, - (msgTtlFunc != null) ? msgTtlFunc.apply(value) : Message.DEFAULT_TIME_TO_LIVE, - (msgTtlFunc != null) ? msgTtlFunc.apply(value) : Message.DEFAULT_DELIVERY_DELAY, - (disableMsgTimestampFunc != null) ? disableMsgTimestampFunc.apply(value) : false, - (disableMsgIdFunc != null) ? disableMsgIdFunc.apply(value) : false - ); - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsUtil.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsUtil.java deleted file mode 100644 index 278cb189b..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsUtil.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.util; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.Arrays; - -public class JmsUtil { - - private final static Logger logger = LogManager.getLogger(JmsUtil.class); - - // Supported JMS provider type - public enum JMS_PROVIDER_TYPES { - PULSAR("pulsar"); - - public final String label; - JMS_PROVIDER_TYPES(String label) { - this.label = label; - } - } - public static boolean isValidJmsProviderType(String type) { - return Arrays.stream(JMS_PROVIDER_TYPES.values()).anyMatch(t -> t.label.equals(type)); - } - - ///// - // NB command line parameters - // - JMS provider type - public final static String JMS_PROVIDER_TYPE_KEY_STR = "provider_type"; - - /// Only applicable when the provider is "Pulsar" - // - Pulsar configuration properties file - public final static String JMS_PULSAR_PROVIDER_CFG_FILE_KEY_STR = "pulsar_cfg_file"; - public final static String JMS_PULSAR_PROVIDER_DFT_CFG_FILE_NAME = "pulsar_config.properties"; - // - Pulsar web url - public final static String JMS_PULSAR_PROVIDER_WEB_URL_KEY_STR = "web_url"; - // - Pulsar service url - public final static String JMS_PULSAR_PROVIDER_SVC_URL_KEY_STR = "service_url"; - - - public final static String ASYNC_API_KEY_STR = "async_api"; - public final static String JMS_DESTINATION_TYPE_KEY_STR = "jms_desitation_type"; - - ///// JMS Producer - // Supported JMS provider type - public enum JMS_MSG_HEADER_KEYS { - DELIVERY_MODE("jms_producer_header_msg_delivery_mode"), - PRIORITY("jms_producer_header_msg_priority"), - TTL("jms_producer_header_msg_ttl"), - DELIVERY_DELAY("jms_producer_header_msg_delivery_delay"), - DISABLE_TIMESTAMP("jms_producer_header_disable_msg_timestamp"), - DISABLE_ID("jms_producer_header_disable_msg_id"); - - public final String label; - JMS_MSG_HEADER_KEYS(String label) { - this.label = label; - } - } - public static boolean isValidJmsHeaderKey(String type) { - return Arrays.stream(JMS_MSG_HEADER_KEYS.values()).anyMatch(t -> t.label.equals(type)); - } - public final static String JMS_PRODUCER_MSG_PROPERTY_KEY_STR = "jms_producer_msg_properties"; - public final static String JMS_PRODUCER_MSG_BODY_KEY_STR = "msg_body"; - - ///// JMS Consumer - public final static String JMS_CONSUMER_DURABLE_KEY_STR = "jms_consumer_msg_durable"; - public final static String JMS_CONSUMER_SHARED_KEY_STR = "jms_consumer_msg_shared"; - public final static String JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR = "jms_consumer_subscription"; - public final static String JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR = "jms_consumer_msg_read_selector"; - public final static String JMS_CONSUMER_MSG_NOLOCAL_KEY_STR = "jms_consumer_msg_nolocal"; - public final static String JMS_CONSUMER_READ_TIMEOUT_KEY_STR = "jms_consumer_msg_read_timeout"; - - - // Only applicable to Pulsar JMS provider - public final static String PULSAR_JMS_TOPIC_URI_KEY_STR = "pulsar_topic_uri"; - - // Supported message operation types - public enum OP_TYPES { - MSG_SEND("msg_send"), - MSG_READ("msg_read"); - - public final String label; - OP_TYPES(String label) { - this.label = label; - } - } - public static boolean isValidClientType(String type) { - return Arrays.stream(OP_TYPES.values()).anyMatch(t -> t.label.equals(type)); - } - - // JMS Destination Types - public enum JMS_DESTINATION_TYPES { - QUEUE("queue"), - TOPIC("topic"); - - public final String label; - JMS_DESTINATION_TYPES(String label) { - this.label = label; - } - } - public static boolean isValidJmsDestinationType(String type) { - return Arrays.stream(JMS_DESTINATION_TYPES.values()).anyMatch(t -> t.label.equals(type)); - } -} - diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/PulsarConfig.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/PulsarConfig.java deleted file mode 100644 index 5a015cddf..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/PulsarConfig.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.util; - -import org.apache.commons.configuration2.Configuration; -import org.apache.commons.configuration2.FileBasedConfiguration; -import org.apache.commons.configuration2.PropertiesConfiguration; -import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder; -import org.apache.commons.configuration2.builder.fluent.Parameters; -import org.apache.commons.configuration2.ex.ConfigurationException; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -public class PulsarConfig { - private final static Logger logger = LogManager.getLogger(PulsarConfig.class); - - public static final String SCHEMA_CONF_PREFIX = "schema"; - public static final String CLIENT_CONF_PREFIX = "client"; - public static final String PRODUCER_CONF_PREFIX = "producer"; - public static final String CONSUMER_CONF_PREFIX = "consumer"; - - private final Map schemaConfMap = new HashMap<>(); - private final Map clientConfMap = new HashMap<>(); - private final Map producerConfMap = new HashMap<>(); - private final Map consumerConfMap = new HashMap<>(); - - public PulsarConfig(String fileName) { - File file = new File(fileName); - - try { - String canonicalFilePath = file.getCanonicalPath(); - - Parameters params = new Parameters(); - - FileBasedConfigurationBuilder builder = - new FileBasedConfigurationBuilder(PropertiesConfiguration.class) - .configure(params.properties() - .setFileName(fileName)); - - Configuration config = builder.getConfiguration(); - - // Get schema specific configuration settings - for (Iterator it = config.getKeys(SCHEMA_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - schemaConfMap.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get client connection specific configuration settings - for (Iterator it = config.getKeys(CLIENT_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - clientConfMap.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get producer specific configuration settings - for (Iterator it = config.getKeys(PRODUCER_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - producerConfMap.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get consumer specific configuration settings - for (Iterator it = config.getKeys(CONSUMER_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - consumerConfMap.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - } catch (IOException ioe) { - logger.error("Can't read the specified config properties file: " + fileName); - ioe.printStackTrace(); - } catch (ConfigurationException cex) { - logger.error("Error loading configuration items from the specified config properties file: " + fileName + ":" + cex.getMessage()); - cex.printStackTrace(); - } - } - - public Map getSchemaConfMap() { - return this.schemaConfMap; - } - public Map getClientConfMap() { - return this.clientConfMap; - } - public Map getProducerConfMap() { - return this.producerConfMap; - } - public Map getConsumerConfMap() { - return this.consumerConfMap; - } -} diff --git a/driver-jms/src/main/resources/jms.md b/driver-jms/src/main/resources/jms.md deleted file mode 100644 index 07dd0c5c7..000000000 --- a/driver-jms/src/main/resources/jms.md +++ /dev/null @@ -1 +0,0 @@ -# Overview diff --git a/driver-jms/src/main/resources/pulsar_config.properties b/driver-jms/src/main/resources/pulsar_config.properties deleted file mode 100644 index f711535ac..000000000 --- a/driver-jms/src/main/resources/pulsar_config.properties +++ /dev/null @@ -1,33 +0,0 @@ -### Schema related configurations - schema.xxx -# valid types: -# - primitive type (https://pulsar.apache.org/docs/en/schema-understand/#primitive-type) -# - keyvalue (https://pulsar.apache.org/docs/en/schema-understand/#keyvalue) -# - strut (complex type) (https://pulsar.apache.org/docs/en/schema-understand/#struct) -# avro, json, protobuf -# -# NOTE: for JMS client, Pulsar "schema" is NOT supported yet -schema.type= -schema.definition= - - -### Pulsar client related configurations - client.xxx -# http://pulsar.apache.org/docs/en/client-libraries-java/#client -client.connectionTimeoutMs=5000 -#client.authPluginClassName=org.apache.pulsar.client.impl.auth.AuthenticationToken -#client.authParams= -#client.tlsAllowInsecureConnection=true -client.numIoThreads=10 -client.numListenerThreads=10 - - -### Producer related configurations (global) - producer.xxx -# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer -producer.sendTimeoutMs= -producer.blockIfQueueFull=true -producer.maxPendingMessages=10000 -producer.batchingMaxMessages=10000 - - -### Consumer related configurations (global) - consumer.xxx -# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer -consumer.receiverQueueSize=2000 diff --git a/driver-jms/src/main/resources/pulsar_jms.yaml b/driver-jms/src/main/resources/pulsar_jms.yaml deleted file mode 100644 index 755f05ed6..000000000 --- a/driver-jms/src/main/resources/pulsar_jms.yaml +++ /dev/null @@ -1,89 +0,0 @@ -bindings: - payload: NumberNameToString() #AlphaNumericString(20) - tenant: Mod(10000); Div(10L); ToString(); Prefix("tnt") - namespace: Mod(10); Div(5L); ToString(); Prefix("ns") - core_topic_name: Mod(5); ToString(); Prefix("t") - -# document level parameters that apply to all Pulsar client types: -params: - ### static only - async_api: "true" - - ### Static only - # Valid values: queue (point-to-point) or topic (pub-sub) - jms_desitation_type: "topic" - - ### Static Only - # NOTE: ONLY relevant when the JMS provider is Pulsar - #pulsar_topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}" - #pulsar_topic_uri: "persistent://public/default/pt100" - #pulsar_topic_uri: "persistent://public/default/t0" - pulsar_topic_uri: "persistent://public/default/pt100_10" - #pulsar_topic_uri: "persistent://public/default/pt200_10" - #pulsar_topic_uri: "persistent://public/default/pt300_10" - #pulsar_topic_uri: "persistent://public/default/pt400_10" - -blocks: - - name: "producer-block" - tags: - phase: "jms_producer" - statements: - - name: "s1" - optype: "msg_send" - - ### JMS PRODUCER message header - ### https://docs.oracle.com/javaee/7/api/constant-values.html#javax.jms.DeliveryMode.NON_PERSISTENT - # - static or dynamic - # - Producer only - # Valid values: non-persistent(1), or persistent(2) - default - jms_producer_header_msg_delivery_mode: "2" - # Valid values: 0~9 (4 as default) - jms_producer_header_msg_priority: "4" - # Valid values: non-negative long; default 0 (never expires) - jms_producer_header_msg_ttl: "0" - # Valid values: non-negative long; default 0 (no delay) - jms_producer_header_msg_delivery_delay: "0" - # Valid values: true/false; default false (message timestamp is enabled) - jms_producer_header_disable_msg_timestamp: "false" - # Valid values: true/false; default false (message ID is enabled) - jms_producer_header_disable_msg_id: "false" - - ### JMS PRODUCER message properties - # - static only - # - Producer only - # - In format: "key1=value1;key2=value2;..." - jms_producer_msg_properties: "key1=value1;key2=value2" - - ### JMS PRODUCER message body - msg_body: "{payload}" - - - name: "consumer-block" - tags: - phase: "jms_consumer" - statements: - - name: "s1" - optype: "msg_read" - - ### JMS CONSUMER durable and shared - jms_consumer_msg_durable: "true" - jms_consumer_msg_shared: "true" - - ### JMS CONSUMER subscription name - # - only relevant for durable consumer - jms_consumer_subscription: "mysub" - - ### JMS CONSUMER subscription name - # - only relevant for unshared consumer - jms_consumer_nolocal: "false" - - ### JMS CONSUMER message read timeout - # - unit: milliseconds - # - 0 means call blocks indefinitely - # - FIXME: 0 supposes to wait indefinitly; but - # it actually behaves like no wait at all - jms_consumer_msg_read_timeout: "10000" - - ### JMS CONSUMER message selector - # - empty string means no message selector - # - https://docs.oracle.com/cd/E19798-01/821-1841/bncer/index.html - jms_consumer_msg_read_selector: "" diff --git a/nb/pom.xml b/nb/pom.xml index 3cbffe028..43e93653e 100644 --- a/nb/pom.xml +++ b/nb/pom.xml @@ -67,6 +67,12 @@ 4.17.22-SNAPSHOT + + io.nosqlbench + adapter-pulsar + 4.17.22-SNAPSHOT + + diff --git a/nb5/pom.xml b/nb5/pom.xml index f2fda67a0..f801d40cd 100644 --- a/nb5/pom.xml +++ b/nb5/pom.xml @@ -88,6 +88,12 @@ 4.17.31-SNAPSHOT + + io.nosqlbench + adapter-pulsar + 4.17.31-SNAPSHOT + + From a05c511de7d34ef63502df7189d1a78621a9e956 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 14 Nov 2022 09:50:20 -0600 Subject: [PATCH 04/14] fix: upgrade org.eclipse.jetty:jetty-server from 11.0.11 to 11.0.12 (#763) Snyk has created this PR to upgrade org.eclipse.jetty:jetty-server from 11.0.11 to 11.0.12. See this package in Maven Repository: https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-server/ See this project in Snyk: https://app.snyk.io/org/jshook/project/03cbee46-d5d2-41d3-89cc-a2ad77ab807a?utm_source=github&utm_medium=referral&page=upgrade-pr Co-authored-by: snyk-bot --- docsys/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsys/pom.xml b/docsys/pom.xml index 17538bfee..5ea4dcbf4 100644 --- a/docsys/pom.xml +++ b/docsys/pom.xml @@ -50,7 +50,7 @@ org.eclipse.jetty jetty-server - 11.0.11 + 11.0.12 org.eclipse.jetty From eb1de8f1d5c40421f72c69a064e89a42bc9e3b71 Mon Sep 17 00:00:00 2001 From: jeffbanks Date: Mon, 14 Nov 2022 10:39:10 -0600 Subject: [PATCH 05/14] Timing changes for github actions to properly perform build/test step. --- .../scripts/examples/extension_histostatslogger.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nbr-examples/src/test/resources/scripts/examples/extension_histostatslogger.js b/nbr-examples/src/test/resources/scripts/examples/extension_histostatslogger.js index 377c1b357..50c99758b 100644 --- a/nbr-examples/src/test/resources/scripts/examples/extension_histostatslogger.js +++ b/nbr-examples/src/test/resources/scripts/examples/extension_histostatslogger.js @@ -18,16 +18,15 @@ activitydef = { "alias" : "testhistostatslogger", "driver" : "diag", "cycles" : "50000", - "threads" : "20", - "interval" : "2000", - "targetrate" : "10000.0", + "threads" : "5", + "rate" : "100.0", "op" : "noop" }; histostatslogger.logHistoStats("testing extention histostatslogger", ".*", "logs/histostats.csv", "0.5s"); print("started logging to logs/histostats.csv for all metrics at 1/2" + " second intervals."); - scenario.start(activitydef); -scenario.waitMillis(2000); +scenario.waitMillis(4000); scenario.stop(activitydef); + From ca6f2b052bd711971c1edadcb95d9e5d8bd186f1 Mon Sep 17 00:00:00 2001 From: yabinmeng Date: Mon, 14 Nov 2022 15:27:57 -0600 Subject: [PATCH 06/14] Add PulsarAdapter specific metrics --- .../adapter/pulsar/PulsarDriverAdapter.java | 2 +- .../adapter/pulsar/PulsarOpMapper.java | 36 +- .../adapter/pulsar/PulsarOpType.java | 37 +- .../adapter/pulsar/PulsarSpace.java | 14 +- .../dispensers/AdminNamespaceOpDispenser.java | 5 +- .../dispensers/AdminTenantOpDispenser.java | 5 +- .../dispensers/AdminTopicOpDispenser.java | 5 +- .../MessageConsumerOpDispenser.java | 6 +- .../MessageProducerOpDispenser.java | 6 +- .../dispensers/MessageReaderOpDispenser.java | 6 +- .../dispensers/PulsarAdminOpDispenser.java | 13 +- .../dispensers/PulsarBaseOpDispenser.java | 467 +++++++++++++++++- .../dispensers/PulsarClientOpDispenser.java | 27 +- .../PulsarAdapterInvalidParamException.java | 4 + .../pulsar/util/PulsarAdapterMetrics.java | 196 ++++++++ .../pulsar/util/PulsarAdapterUtil.java | 30 +- .../api/activityimpl/BaseOpDispenser.java | 12 +- .../api/engine/metrics/ActivityMetrics.java | 18 + pom.xml | 3 +- 19 files changed, 800 insertions(+), 92 deletions(-) create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java index b121ca5e3..4d0c2c7fb 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java @@ -30,7 +30,7 @@ import org.apache.logging.log4j.Logger; import java.util.function.Function; -@Service(value = DriverAdapter.class, selector = "pulsar-nb5", maturity = Maturity.Experimental) +@Service(value = DriverAdapter.class, selector = "pulsar") public class PulsarDriverAdapter extends BaseDriverAdapter { private final static Logger logger = LogManager.getLogger(PulsarDriverAdapter.class); diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java index babcb0349..8620eecba 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java @@ -17,6 +17,7 @@ package io.nosqlbench.adapter.pulsar; import io.nosqlbench.adapter.pulsar.dispensers.*; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnsupportedOpException; import io.nosqlbench.adapter.pulsar.ops.PulsarOp; import io.nosqlbench.api.config.standard.NBConfiguration; import io.nosqlbench.engine.api.activityimpl.OpDispenser; @@ -36,24 +37,23 @@ public class PulsarOpMapper implements OpMapper { private final static Logger logger = LogManager.getLogger(PulsarOpMapper.class); private final NBConfiguration cfg; - private final DriverSpaceCache cache; + private final DriverSpaceCache spaceCache; private final DriverAdapter adapter; - public PulsarOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache cache) { + public PulsarOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache spaceCache) { this.cfg = cfg; - this.cache = cache; + this.spaceCache = spaceCache; this.adapter = adapter; } @Override public OpDispenser apply(ParsedOp op) { - String space = op.getStaticConfigOr("space", "default"); - - PulsarClient pulsarClient = cache.get(space).getPulsarClient(); - PulsarAdmin pulsarAdmin = cache.get(space).getPulsarAdmin(); - Schema pulsarSchema = cache.get(space).getPulsarSchema(); - + String spaceName = op.getStaticConfigOr("space", "default"); + PulsarSpace pulsarSpace = spaceCache.get(spaceName); + PulsarClient pulsarClient = pulsarSpace.getPulsarClient(); + PulsarAdmin pulsarAdmin = pulsarSpace.getPulsarAdmin(); + Schema pulsarSchema = pulsarSpace.getPulsarSchema(); /* * If the user provides a body element, then they want to provide the JSON or @@ -66,19 +66,25 @@ public class PulsarOpMapper implements OpMapper { else { TypeAndTarget opType = op.getTypeAndTarget(PulsarOpType.class, String.class); + if (PulsarOpType.isValidPulsarOpType(opType.enumId.label)) { + throw new PulsarAdapterUnsupportedOpException( + "Unrecognized Pulsar Adapter Op Type -- must be one of the following values: \"" + + PulsarOpType.getValidPulsarOpTypeList() + "\"!"); + } + return switch (opType.enumId) { case AdminTenant -> - new AdminTenantOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin); + new AdminTenantOpDispenser(adapter, op, opType.targetFunction, pulsarSpace); case AdminNamespace -> - new AdminNamespaceOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin); + new AdminNamespaceOpDispenser(adapter, op, opType.targetFunction, pulsarSpace); case AdminTopic -> - new AdminTopicOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin); + new AdminTopicOpDispenser(adapter, op, opType.targetFunction, pulsarSpace); case MessageProduce -> - new MessageProducerOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema); + new MessageProducerOpDispenser(adapter, op, opType.targetFunction, pulsarSpace); case MessageConsume -> - new MessageConsumerOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema); + new MessageConsumerOpDispenser(adapter, op, opType.targetFunction, pulsarSpace); case MessageRead -> - new MessageReaderOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema); + new MessageReaderOpDispenser(adapter, op, opType.targetFunction, pulsarSpace); }; } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java index c4bf6c3ee..278137169 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java @@ -16,11 +16,36 @@ package io.nosqlbench.adapter.pulsar; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; + +import java.util.Arrays; +import java.util.stream.Collectors; + public enum PulsarOpType { - AdminTenant, - AdminNamespace, - AdminTopic, - MessageProduce, - MessageConsume, - MessageRead + AdminTenant("admin-tenant"), + AdminNamespace("admin-namespace"), + AdminTopic("admin-topic"), + MessageProduce("msg-send"), + MessageConsume("msg-consume"), + MessageRead("msg-read"); + + public final String label; + + PulsarOpType(String label) { + this.label = label; + } + + + public static boolean isValidPulsarOpType(String type) { + return Arrays.stream(values()) + .anyMatch(t -> t.label.equalsIgnoreCase(type)); + } + + public static String getValidPulsarOpTypeList() { + return Arrays.stream(values()) + .map(t -> t.label) + .collect(Collectors.joining(", ")); + } } + + diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java index 708007527..487295c1f 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java @@ -16,25 +16,28 @@ package io.nosqlbench.adapter.pulsar; +import com.codahale.metrics.Gauge; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; import io.nosqlbench.adapter.pulsar.util.PulsarNBClientConf; import io.nosqlbench.api.config.standard.ConfigModel; import io.nosqlbench.api.config.standard.NBConfigModel; import io.nosqlbench.api.config.standard.NBConfiguration; import io.nosqlbench.api.config.standard.Param; +import io.nosqlbench.api.engine.metrics.ActivityMetrics; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; -import org.apache.pulsar.client.api.ClientBuilder; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.*; import org.apache.pulsar.common.schema.KeyValueEncodingType; +import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; public class PulsarSpace { @@ -57,7 +60,6 @@ public class PulsarSpace { this.pulsarSvcUrl = cfg.get("service_url"); this.webSvcUrl = cfg.get("web_url"); - this.pulsarNBClientConf = new PulsarNBClientConf(cfg.get("config")); initPulsarAdminAndClientObj(); @@ -178,7 +180,7 @@ public class PulsarSpace { // this is to allow KEY_VALUE schema if (pulsarNBClientConf.hasSchemaConfKey("schema.key.type")) { Schema pulsarKeySchema = buildSchemaFromDefinition("schema.key.type", "schema.key.definition"); - Object encodingType = pulsarNBClientConf.getSchemaConfValue("schema.keyvalue.encodingtype"); + Object encodingType = pulsarNBClientConf.getSchemaConfValue("schema.keyvalue.encodingtype"); KeyValueEncodingType keyValueEncodingType = KeyValueEncodingType.SEPARATED; if (encodingType != null) { keyValueEncodingType = KeyValueEncodingType.valueOf(encodingType.toString()); diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java index 89514d4e6..8b5fd31d1 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java @@ -16,6 +16,7 @@ package io.nosqlbench.adapter.pulsar.dispensers; +import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.AdminNamespaceOp; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; @@ -28,8 +29,8 @@ public class AdminNamespaceOpDispenser extends PulsarAdminOpDispenser { public AdminNamespaceOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, - PulsarAdmin pulsarAdmin) { - super(adapter, op, tgtNameFunc, pulsarAdmin); + PulsarSpace pulsarSpace) { + super(adapter, op, tgtNameFunc, pulsarSpace); } @Override diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java index 5bca4ec78..21e479283 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java @@ -16,6 +16,7 @@ package io.nosqlbench.adapter.pulsar.dispensers; +import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.AdminTenantOp; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; @@ -31,8 +32,8 @@ public class AdminTenantOpDispenser extends PulsarAdminOpDispenser { public AdminTenantOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, - PulsarAdmin pulsarAdmin) { - super(adapter, op, tgtNameFunc, pulsarAdmin); + PulsarSpace pulsarSpace) { + super(adapter, op, tgtNameFunc, pulsarSpace); adminRolesFunc = lookupStaticStrSetOpValueFunc("admin_roles"); allowedClustersFunc = lookupStaticStrSetOpValueFunc("allowed_clusters"); diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java index 70af09180..87e892fb7 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java @@ -16,6 +16,7 @@ package io.nosqlbench.adapter.pulsar.dispensers; +import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.AdminTopicOp; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; @@ -31,8 +32,8 @@ public class AdminTopicOpDispenser extends PulsarAdminOpDispenser { public AdminTopicOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, - PulsarAdmin pulsarAdmin) { - super(adapter, op, tgtNameFunc, pulsarAdmin); + PulsarSpace pulsarSpace) { + super(adapter, op, tgtNameFunc, pulsarSpace); // Non-partitioned topic is default enablePartFunc = lookupStaticBoolConfigValueFunc("enable_partition", false); diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java index 8a0cb6ba6..67e7dfabc 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java @@ -16,6 +16,7 @@ package io.nosqlbench.adapter.pulsar.dispensers; +import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.MessageConsumerOp; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; @@ -29,9 +30,8 @@ public class MessageConsumerOpDispenser extends PulsarClientOpDispenser { public MessageConsumerOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, - PulsarClient pulsarClient, - Schema pulsarSchema) { - super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema); + PulsarSpace pulsarSpace) { + super(adapter, op, tgtNameFunc, pulsarSpace); } @Override diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java index ef9b2547d..ecb765af1 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java @@ -16,6 +16,7 @@ package io.nosqlbench.adapter.pulsar.dispensers; +import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.MessageProducerOp; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; @@ -29,9 +30,8 @@ public class MessageProducerOpDispenser extends PulsarClientOpDispenser { public MessageProducerOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, - PulsarClient pulsarClient, - Schema pulsarSchema) { - super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema); + PulsarSpace pulsarSpace) { + super(adapter, op, tgtNameFunc, pulsarSpace); } @Override diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java index 3e9f819fd..53394f44b 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java @@ -16,6 +16,7 @@ package io.nosqlbench.adapter.pulsar.dispensers; +import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.MessageReaderOp; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; @@ -29,9 +30,8 @@ public class MessageReaderOpDispenser extends PulsarClientOpDispenser { public MessageReaderOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, - PulsarClient pulsarClient, - Schema pulsarSchema) { - super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema); + PulsarSpace pulsarSpace) { + super(adapter, op, tgtNameFunc, pulsarSpace); } @Override diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java index d18eee305..3bd2bd609 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java @@ -16,6 +16,8 @@ package io.nosqlbench.adapter.pulsar.dispensers; +import io.nosqlbench.adapter.pulsar.PulsarSpace; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -30,11 +32,12 @@ public abstract class PulsarAdminOpDispenser extends PulsarBaseOpDispenser { public PulsarAdminOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, - PulsarAdmin pulsarAdmin) { - super(adapter, op, tgtNameFunc); - this.pulsarAdmin = pulsarAdmin; + PulsarSpace pulsarSpace) { + super(adapter, op, tgtNameFunc, pulsarSpace); + this.pulsarAdmin = pulsarSpace.getPulsarAdmin(); - // Creating admin objects (tenant, namespace, topic) is the default - this.adminDelOpFunc = lookupStaticBoolConfigValueFunc("admin_delop", false); + // Doc-level parameter: admin_delop + this.adminDelOpFunc = lookupStaticBoolConfigValueFunc( + PulsarAdapterUtil.DOC_LEVEL_PARAMS.ADMIN_DELOP.label, false); } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java index 1a0350ac1..a2ff631e4 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java @@ -17,7 +17,11 @@ package io.nosqlbench.adapter.pulsar.dispensers; */ import io.nosqlbench.adapter.pulsar.PulsarSpace; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterInvalidParamException; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; import io.nosqlbench.adapter.pulsar.ops.PulsarOp; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; @@ -26,8 +30,10 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.api.*; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.LongFunction; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -35,39 +41,64 @@ import java.util.stream.Collectors; public abstract class PulsarBaseOpDispenser extends BaseOpDispenser { private final static Logger logger = LogManager.getLogger("PulsarBaseOpDispenser"); + protected final ParsedOp parsedOp; protected final LongFunction asyncApiFunc; protected final LongFunction tgtNameFunc; + protected final PulsarSpace pulsarSpace; - public PulsarBaseOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc) { + private final ConcurrentHashMap> producers = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> consumers = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> readers = new ConcurrentHashMap<>(); + protected final PulsarAdapterMetrics pulsarAdapterMetrics; + + public PulsarBaseOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarSpace pulsarSpace) { super(adapter, op); this.parsedOp = op; this.tgtNameFunc = tgtNameFunc; - // Async API is the default - this.asyncApiFunc = lookupStaticBoolConfigValueFunc("async_api", true); + this.pulsarSpace = pulsarSpace; + + // Doc-level parameter: async_api + this.asyncApiFunc = lookupStaticBoolConfigValueFunc( + PulsarAdapterUtil.DOC_LEVEL_PARAMS.ASYNC_API.label, true); + + this.pulsarAdapterMetrics = new PulsarAdapterMetrics(pulsarSpace, getDefaultMetricsPrefix(this.parsedOp)); + if (instrument) { + pulsarAdapterMetrics.initPulsarAdapterInstrumentation(); + } } protected LongFunction lookupStaticBoolConfigValueFunc(String paramName, boolean defaultValue) { - return (l) -> parsedOp.getOptionalStaticConfig(paramName, String.class) + LongFunction booleanLongFunction; + booleanLongFunction = (l) -> parsedOp.getOptionalStaticConfig(paramName, String.class) .filter(Predicate.not(String::isEmpty)) .map(value -> BooleanUtils.toBoolean(value)) .orElse(defaultValue); + logger.info("{}: {}", paramName, booleanLongFunction.apply(0)); + return booleanLongFunction; } protected LongFunction lookupStaticIntOpValueFunc(String paramName, int defaultValue) { - return (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) + LongFunction integerLongFunction; + integerLongFunction = (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) .filter(Predicate.not(String::isEmpty)) .map(value -> NumberUtils.toInt(value)) .map(value -> { if (value < 0) return 0; else return value; }).orElse(defaultValue); + logger.info("{}: {}", paramName, integerLongFunction.apply(0)); + return integerLongFunction; } protected LongFunction> lookupStaticStrSetOpValueFunc(String paramName) { - return (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) + LongFunction> setStringLongFunction; + setStringLongFunction = (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) .filter(Predicate.not(String::isEmpty)) .map(value -> { Set set = new HashSet<>(); @@ -81,5 +112,429 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser use just the topic name test + .replace("persistent://public/default/", "") + // always remove topic type + .replace("non-persistent://", "") + .replace("persistent://", "") + // persistent://tenant/namespace/topicname -> tenant_namespace_topicname + .replace("/", "_"); + } + + return apiMetricsPrefix; + } + + ////////////////////////////////////// + // Producer Processing --> start + ////////////////////////////////////// + // + + // Topic name IS mandatory for a producer + // - It must be set at either global level or cycle level + // - If set at both levels, cycle level setting takes precedence + private String getEffectiveProducerTopicName(String cycleTopicName) { + if (!StringUtils.isBlank(cycleTopicName)) { + return cycleTopicName; + } + + String globalTopicName = pulsarSpace.getPulsarNBClientConf().getProducerTopicName(); + if (!StringUtils.isBlank(globalTopicName)) { + return globalTopicName; + } + + throw new PulsarAdapterInvalidParamException( + "Effective topic name for a producer can't NOT be empty, " + + "it must be set either as a corresponding adapter Op parameter value or " + + "set in the global Pulsar conf file."); + } + + // Producer name is NOT mandatory + // - It can be set at either global level or cycle level + // - If set at both levels, cycle level setting takes precedence + private String getEffectiveProducerName(String cycleProducerName) { + if (!StringUtils.isBlank(cycleProducerName)) { + return cycleProducerName; + } + + String globalProducerName = pulsarSpace.getPulsarNBClientConf().getProducerName(); + if (!StringUtils.isBlank(globalProducerName)) { + return globalProducerName; + } + + return ""; + } + + public Producer getProducer(String cycleTopicName, String cycleProducerName) { + String topicName = getEffectiveProducerTopicName(cycleTopicName); + String producerName = getEffectiveProducerName(cycleProducerName); + + String producerCacheKey = PulsarAdapterUtil.buildCacheKey(producerName, topicName); + Producer producer = producers.get(producerCacheKey); + + if (producer == null) { + PulsarClient pulsarClient = pulsarSpace.getPulsarClient(); + + // Get other possible producer settings that are set at global level + Map producerConf = pulsarSpace.getPulsarNBClientConf().getProducerConfMap(); + + // Remove global level settings: "topicName" and "producerName" + producerConf.remove(PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.topicName.label); + producerConf.remove(PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.producerName.label); + + try { + ProducerBuilder producerBuilder = pulsarClient. + newProducer(pulsarSpace.getPulsarSchema()). + loadConf(producerConf). + topic(topicName); + + if (!StringUtils.isAnyBlank(producerName)) { + producerBuilder = producerBuilder.producerName(producerName); + } + + producer = producerBuilder.create(); + producers.put(producerCacheKey, producer); + + if (instrument) { + pulsarAdapterMetrics.registerProducerApiMetrics(producer, + getPulsarAPIMetricsPrefix( + PulsarAdapterUtil.PULSAR_API_TYPE.PRODUCER.label, + producerName, + topicName)); + } + } + catch (PulsarClientException ple) { + throw new PulsarAdapterUnexpectedException("Failed to create a Pulsar producer."); + } + } + + return producer; + } + + // + ////////////////////////////////////// + // Producer Processing <-- end + ////////////////////////////////////// + + + + ////////////////////////////////////// + // Consumer Processing --> start + ////////////////////////////////////// + // + + // Subscription name is NOT mandatory + // - It can be set at either global level or cycle level + // - If set at both levels, cycle level setting takes precedence + private String getEffectiveSubscriptionName(String cycleSubscriptionName) { + if (!StringUtils.isBlank(cycleSubscriptionName)) { + return cycleSubscriptionName; + } + + String globalSubscriptionName = pulsarSpace.getPulsarNBClientConf().getConsumerSubscriptionName(); + if (!StringUtils.isBlank(globalSubscriptionName)) { + return globalSubscriptionName; + } + + throw new PulsarAdapterInvalidParamException( + "Effective subscription name for a consumer can't NOT be empty, " + + "it must be set either as a corresponding adapter Op parameter value or " + + "set in the global Pulsar conf file."); + } + + private String getEffectiveSubscriptionTypeStr(String cycleSubscriptionType) { + String subscriptionTypeStr = ""; + + if (!StringUtils.isBlank(cycleSubscriptionType)) { + subscriptionTypeStr = cycleSubscriptionType; + } + else { + String globalSubscriptionType = pulsarSpace.getPulsarNBClientConf().getConsumerSubscriptionType(); + if (!StringUtils.isBlank(globalSubscriptionType)) { + subscriptionTypeStr = globalSubscriptionType; + } + } + + if (StringUtils.isNotBlank(subscriptionTypeStr) && + !PulsarAdapterUtil.isValidSubscriptionType(subscriptionTypeStr)) { + throw new PulsarAdapterInvalidParamException( + "Invalid effective subscription type for a consumer (\"" + subscriptionTypeStr + "\"). " + + "It must be one of the following values: " + PulsarAdapterUtil.getValidSubscriptionTypeList()); + } + + return subscriptionTypeStr; + } + private SubscriptionType getEffectiveSubscriptionType(String cycleSubscriptionType) { + String effectiveSubscriptionStr = getEffectiveSubscriptionTypeStr(cycleSubscriptionType); + SubscriptionType subscriptionType = SubscriptionType.Exclusive; // default subscription type + + if (!StringUtils.isBlank(effectiveSubscriptionStr)) { + subscriptionType = SubscriptionType.valueOf(effectiveSubscriptionStr); + } + + return subscriptionType; + } + + private String getEffectiveConsumerName(String cycleConsumerName) { + if (!StringUtils.isBlank(cycleConsumerName)) { + return cycleConsumerName; + } + + String globalConsumerName = pulsarSpace.getPulsarNBClientConf().getConsumerName(); + if (!StringUtils.isBlank(globalConsumerName)) { + return globalConsumerName; + } + + return ""; + } + + public Consumer getConsumer(String cycleTopicName, + String cycleSubscriptionName, + String cycleSubscriptionType, + String cycleConsumerName, + String cycleKeySharedSubscriptionRanges) { + String subscriptionName = getEffectiveSubscriptionName(cycleSubscriptionName); + SubscriptionType subscriptionType = getEffectiveSubscriptionType(cycleSubscriptionType); + String consumerName = getEffectiveConsumerName(cycleConsumerName); + + if (StringUtils.isAnyBlank(cycleTopicName, subscriptionName)) { + throw new PulsarAdapterInvalidParamException( + "Must specify a topic name and a subscription name when creating a consumer!"); + } + + String consumerCacheKey = PulsarAdapterUtil.buildCacheKey(consumerName, subscriptionName, cycleTopicName); + Consumer consumer = consumers.get(consumerCacheKey); + if (consumer == null) { + PulsarClient pulsarClient = pulsarSpace.getPulsarClient(); + + // Get other possible consumer settings that are set at global level + Map consumerConf = new HashMap<>(pulsarSpace.getPulsarNBClientConf().getConsumerConfMap()); + + // Remove global level settings: + // - "topicNames", "topicsPattern", "subscriptionName", "subscriptionType", "consumerName" + consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicNames.label); + consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label); + consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label); + consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label); + consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.consumerName.label); + // Remove non-standard consumer configuration properties + consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label); + + try { + ConsumerBuilder consumerBuilder = pulsarClient. + newConsumer(pulsarSpace.getPulsarSchema()). + loadConf(consumerConf). + topic(cycleTopicName). + subscriptionName(subscriptionName). + subscriptionType(subscriptionType); + + if (subscriptionType == SubscriptionType.Key_Shared) { + KeySharedPolicy keySharedPolicy = KeySharedPolicy.autoSplitHashRange(); + if (cycleKeySharedSubscriptionRanges != null && !cycleKeySharedSubscriptionRanges.isEmpty()) { + Range[] ranges = parseRanges(cycleKeySharedSubscriptionRanges); + logger.info("Configuring KeySharedPolicy#stickyHashRange with ranges {}", ranges); + keySharedPolicy = KeySharedPolicy.stickyHashRange().ranges(ranges); + } + consumerBuilder.keySharedPolicy(keySharedPolicy); + } + + if (!StringUtils.isBlank(consumerName)) { + consumerBuilder = consumerBuilder.consumerName(consumerName); + } + + consumer = consumerBuilder.subscribe(); + consumers.put(consumerCacheKey, consumer); + + if (instrument) { + pulsarAdapterMetrics.registerConsumerApiMetrics( + consumer, + getPulsarAPIMetricsPrefix( + PulsarAdapterUtil.PULSAR_API_TYPE.CONSUMER.label, + consumerName, + cycleTopicName)); + } + + } + catch (PulsarClientException ple) { + throw new PulsarAdapterUnexpectedException("Failed to create a Pulsar consumer!"); + } + } + + return consumer; + } + + private static Range[] parseRanges(String ranges) { + if (ranges == null || ranges.isEmpty()) { + return new Range[0]; + } + String[] split = ranges.split(","); + Range[] result = new Range[split.length]; + for (int i = 0; i < split.length; i++) { + String range = split[i]; + int pos = range.indexOf(".."); + if (pos <= 0) { + throw new IllegalArgumentException("Invalid range '" + range + "'"); + } + try { + int start = Integer.parseInt(range.substring(0, pos)); + int end = Integer.parseInt(range.substring(pos + 2)); + result[i] = Range.of(start, end); + } catch (NumberFormatException err) { + throw new IllegalArgumentException("Invalid range '" + range + "'"); + } + } + return result; + } + + // + ////////////////////////////////////// + // Consumer Processing <-- end + ////////////////////////////////////// + + + + ////////////////////////////////////// + // Reader Processing --> Start + ////////////////////////////////////// + // + + // Topic name IS mandatory for a reader + // - It must be set at either global level or cycle level + // - If set at both levels, cycle level setting takes precedence + private String getEffectiveReaderTopicName(String cycleReaderTopicName) { + if (!StringUtils.isBlank(cycleReaderTopicName)) { + return cycleReaderTopicName; + } + + String globalReaderTopicName = pulsarSpace.getPulsarNBClientConf().getReaderTopicName(); + if (!StringUtils.isBlank(globalReaderTopicName)) { + return globalReaderTopicName; + } + + throw new PulsarAdapterInvalidParamException( + "Effective topic name for a reader can't NOT be empty, " + + "it must be set either as a corresponding adapter Op parameter value or " + + "set in the global Pulsar conf file."); + } + + // Reader name is NOT mandatory + // - It can be set at either global level or cycle level + // - If set at both levels, cycle level setting takes precedence + private String getEffectiveReaderName(String cycleReaderName) { + if (!StringUtils.isBlank(cycleReaderName)) { + return cycleReaderName; + } + + String globalReaderName = pulsarSpace.getPulsarNBClientConf().getReaderName(); + if (!StringUtils.isBlank(globalReaderName)) { + return globalReaderName; + } + + return ""; + } + + private String getEffectiveStartMsgPosStr(String cycleStartMsgPosStr) { + if (!StringUtils.isBlank(cycleStartMsgPosStr)) { + return cycleStartMsgPosStr; + } + + String globalStartMsgPosStr = pulsarSpace.getPulsarNBClientConf().getStartMsgPosStr(); + if (!StringUtils.isBlank(globalStartMsgPosStr)) { + return globalStartMsgPosStr; + } + + return PulsarAdapterUtil.READER_MSG_POSITION_TYPE.latest.label; + } + + public Reader getReader(String cycleTopicName, + String cycleReaderName, + String cycleStartMsgPos) { + + String topicName = getEffectiveReaderTopicName(cycleTopicName); + String readerName = getEffectiveReaderName(cycleReaderName); + String startMsgPosStr = getEffectiveStartMsgPosStr(cycleStartMsgPos); + if (!PulsarAdapterUtil.isValideReaderStartPosition(startMsgPosStr)) { + throw new RuntimeException("Reader:: Invalid value for reader start message position!"); + } + + String readerCacheKey = PulsarAdapterUtil.buildCacheKey(topicName, readerName, startMsgPosStr); + Reader reader = readers.get(readerCacheKey); + + if (reader == null) { + PulsarClient pulsarClient = pulsarSpace.getPulsarClient();; + + Map readerConf = pulsarSpace.getPulsarNBClientConf().getReaderConfMap(); + + // Remove global level settings: "topicName" and "readerName" + readerConf.remove(PulsarAdapterUtil.READER_CONF_STD_KEY.topicName.label); + readerConf.remove(PulsarAdapterUtil.READER_CONF_STD_KEY.readerName.label); + // Remove non-standard reader configuration properties + readerConf.remove(PulsarAdapterUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label); + + try { + ReaderBuilder readerBuilder = pulsarClient. + newReader(pulsarSpace.getPulsarSchema()). + loadConf(readerConf). + topic(topicName). + readerName(readerName); + + MessageId startMsgId = MessageId.latest; + if (startMsgPosStr.equalsIgnoreCase(PulsarAdapterUtil.READER_MSG_POSITION_TYPE.earliest.label)) { + startMsgId = MessageId.earliest; + } + //TODO: custom start message position is NOT supported yet + //else if (startMsgPosStr.startsWith(PulsarAdapterUtil.READER_MSG_POSITION_TYPE.custom.label)) { + // startMsgId = MessageId.latest; + //} + + reader = readerBuilder.startMessageId(startMsgId).create(); + + } catch (PulsarClientException ple) { + ple.printStackTrace(); + throw new RuntimeException("Unable to create a Pulsar reader!"); + } + + readers.put(readerCacheKey, reader); + } + + return reader; + } + // + ////////////////////////////////////// + // Reader Processing <-- end + ////////////////////////////////////// } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java index 43acd0bb8..bacd1d7ed 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java @@ -16,6 +16,8 @@ package io.nosqlbench.adapter.pulsar.dispensers; +import io.nosqlbench.adapter.pulsar.PulsarSpace; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; import org.apache.pulsar.client.api.PulsarClient; @@ -28,13 +30,28 @@ public abstract class PulsarClientOpDispenser extends PulsarBaseOpDispenser { protected final PulsarClient pulsarClient; protected final Schema pulsarSchema; + protected final LongFunction useTransactFunc; + protected final LongFunction transactBatchNumFunc; + protected final LongFunction seqTrackingFunc; + public PulsarClientOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, - PulsarClient pulsarClient, - Schema pulsarSchema) { - super(adapter, op, tgtNameFunc); - this.pulsarClient = pulsarClient; - this.pulsarSchema = pulsarSchema; + PulsarSpace pulsarSpace) { + super(adapter, op, tgtNameFunc, pulsarSpace); + this.pulsarClient = pulsarSpace.getPulsarClient(); + this.pulsarSchema = pulsarSpace.getPulsarSchema(); + + // Doc-level parameter: use_transaction + this.useTransactFunc = lookupStaticBoolConfigValueFunc( + PulsarAdapterUtil.DOC_LEVEL_PARAMS.USE_TRANSACTION.label, false); + + // Doc-level parameter: transact_batch_num + this.transactBatchNumFunc = lookupStaticIntOpValueFunc( + PulsarAdapterUtil.DOC_LEVEL_PARAMS.TRANSACT_BATCH_NUM.label, 1); + + // Doc-level parameter: seq_tracking + this.seqTrackingFunc = lookupStaticBoolConfigValueFunc( + PulsarAdapterUtil.DOC_LEVEL_PARAMS.SEQ_TRACKING.label, false); } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java index 1004a7a72..5996e8573 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java @@ -22,4 +22,8 @@ public class PulsarAdapterInvalidParamException extends RuntimeException { public PulsarAdapterInvalidParamException(String paramName, String errDesc) { super("Invalid setting for parameter (" + paramName + "): " + errDesc); } + + public PulsarAdapterInvalidParamException(String fullErrDesc) { + super(fullErrDesc); + } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java new file mode 100644 index 000000000..f67bda339 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.util; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Timer; +import io.nosqlbench.adapter.pulsar.PulsarSpace; +import io.nosqlbench.api.engine.metrics.ActivityMetrics; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerStats; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerStats; + +import java.util.function.Function; + +public class PulsarAdapterMetrics { + + private final static Logger logger = LogManager.getLogger("PulsarAdapterMetrics"); + + private final PulsarSpace pulsarSpace; + private final String defaultAdapterMetricsPrefix; + + /** + * Pulsar adapter specific metrics + */ + protected Counter bytesCounter; + // - message out of sequence error counter + protected Counter msgErrOutOfSeqCounter; + // - message loss counter + protected Counter msgErrLossCounter; + // - message duplicate (when dedup is enabled) error counter + protected Counter msgErrDuplicateCounter; + + protected Histogram messageSizeHistogram; + // end-to-end latency + protected Histogram e2eMsgProcLatencyHistogram; + // A histogram that tracks payload round-trip-time, based on a user-defined field in some sender + // system which can be interpreted as millisecond epoch time in the system's local time zone. + // This is paired with a field name of the same type to be extracted and reported in a metric + // named 'payload-rtt'. + protected Histogram payloadRttHistogram; + + protected Timer bindTimer; + protected Timer executeTimer; + protected Timer createTransactionTimer; + protected Timer commitTransactionTimer; + + public PulsarAdapterMetrics(PulsarSpace pulsarSpace, String defaultMetricsPrefix) { + this.pulsarSpace = pulsarSpace; + this.defaultAdapterMetricsPrefix = defaultMetricsPrefix; + } + + public void initPulsarAdapterInstrumentation() { + // Counter metrics + this.bytesCounter = + ActivityMetrics.counter(this.defaultAdapterMetricsPrefix + "bytes"); + this.msgErrOutOfSeqCounter = + ActivityMetrics.counter(this.defaultAdapterMetricsPrefix + "err_msg_oos"); + this.msgErrLossCounter = + ActivityMetrics.counter(this.defaultAdapterMetricsPrefix + "err_msg_loss"); + this.msgErrDuplicateCounter = + ActivityMetrics.counter(this.defaultAdapterMetricsPrefix + "err_msg_dup"); + + // Histogram metrics + this.messageSizeHistogram = + ActivityMetrics.histogram(this.defaultAdapterMetricsPrefix + "message_size"); + this.e2eMsgProcLatencyHistogram = + ActivityMetrics.histogram(this.defaultAdapterMetricsPrefix + "e2e_msg_latency"); + this.payloadRttHistogram = + ActivityMetrics.histogram(this.defaultAdapterMetricsPrefix + "payload_rtt"); + + // Timer metrics + this.bindTimer = + ActivityMetrics.timer(this.defaultAdapterMetricsPrefix + "bind"); + this.executeTimer = + ActivityMetrics.timer(this.defaultAdapterMetricsPrefix + "execute"); + this.createTransactionTimer = + ActivityMetrics.timer(this.defaultAdapterMetricsPrefix + "create_transaction"); + this.commitTransactionTimer = + ActivityMetrics.timer(this.defaultAdapterMetricsPrefix + "commit_transaction"); + } + + + ////////////////////////////////////// + // Pulsar client producer API metrics + ////////////////////////////////////// + // + private static class ProducerGaugeImpl implements Gauge { + private final Producer producer; + private final Function valueExtractor; + + ProducerGaugeImpl(Producer producer, Function valueExtractor) { + this.producer = producer; + this.valueExtractor = valueExtractor; + } + + @Override + public Object getValue() { + // see Pulsar bug https://github.com/apache/pulsar/issues/10100 + // we need to synchronize on producer otherwise we could receive corrupted data + synchronized(producer) { + return valueExtractor.apply(producer.getStats()); + } + } + } + private static Gauge producerSafeExtractMetric(Producer producer, Function valueExtractor) { + return new ProducerGaugeImpl(producer, valueExtractor); + } + + public void registerProducerApiMetrics(Producer producer, String pulsarApiMetricsPrefix) { + String metricsPrefix = defaultAdapterMetricsPrefix; + if (!StringUtils.isBlank(pulsarApiMetricsPrefix)) { + metricsPrefix = pulsarApiMetricsPrefix; + } + + ActivityMetrics.gauge(metricsPrefix + "total_bytes_sent", + producerSafeExtractMetric(producer, (s -> s.getTotalBytesSent() + s.getNumBytesSent()))); + ActivityMetrics.gauge(metricsPrefix + "total_msg_sent", + producerSafeExtractMetric(producer, (s -> s.getTotalMsgsSent() + s.getNumMsgsSent()))); + ActivityMetrics.gauge(metricsPrefix + "total_send_failed", + producerSafeExtractMetric(producer, (s -> s.getTotalSendFailed() + s.getNumSendFailed()))); + ActivityMetrics.gauge(metricsPrefix + "total_ack_received", + producerSafeExtractMetric(producer,(s -> s.getTotalAcksReceived() + s.getNumAcksReceived()))); + ActivityMetrics.gauge(metricsPrefix + "send_bytes_rate", + producerSafeExtractMetric(producer, ProducerStats::getSendBytesRate)); + ActivityMetrics.gauge(metricsPrefix + "send_msg_rate", + producerSafeExtractMetric(producer, ProducerStats::getSendMsgsRate)); + } + + + ////////////////////////////////////// + // Pulsar client consumer API metrics + ////////////////////////////////////// + // + private static class ConsumerGaugeImpl implements Gauge { + private final Consumer consumer; + private final Function valueExtractor; + + ConsumerGaugeImpl(Consumer consumer, Function valueExtractor) { + this.consumer = consumer; + this.valueExtractor = valueExtractor; + } + + @Override + public Object getValue() { + // see Pulsar bug https://github.com/apache/pulsar/issues/10100 + // - this is a bug report for producer stats. + // - assume this also applies to consumer stats. + synchronized(consumer) { + return valueExtractor.apply(consumer.getStats()); + } + } + } + static Gauge consumerSafeExtractMetric(Consumer consumer, Function valueExtractor) { + return new ConsumerGaugeImpl(consumer, valueExtractor); + } + + public void registerConsumerApiMetrics(Consumer consumer, String pulsarApiMetricsPrefix) { + String metricsPrefix = defaultAdapterMetricsPrefix; + if (!StringUtils.isBlank(pulsarApiMetricsPrefix)) { + metricsPrefix = pulsarApiMetricsPrefix; + } + + ActivityMetrics.gauge(metricsPrefix + "total_bytes_recv", + consumerSafeExtractMetric(consumer, (s -> s.getTotalBytesReceived() + s.getNumBytesReceived()))); + ActivityMetrics.gauge(metricsPrefix + "total_msg_recv", + consumerSafeExtractMetric(consumer, (s -> s.getTotalMsgsReceived() + s.getNumMsgsReceived()))); + ActivityMetrics.gauge(metricsPrefix + "total_recv_failed", + consumerSafeExtractMetric(consumer, (s -> s.getTotalReceivedFailed() + s.getNumReceiveFailed()))); + ActivityMetrics.gauge(metricsPrefix + "total_acks_sent", + consumerSafeExtractMetric(consumer,(s -> s.getTotalAcksSent() + s.getNumAcksSent()))); + ActivityMetrics.gauge(metricsPrefix + "recv_bytes_rate", + consumerSafeExtractMetric(consumer, ConsumerStats::getRateBytesReceived)); + ActivityMetrics.gauge(metricsPrefix + "recv_msg_rate", + consumerSafeExtractMetric(consumer, ConsumerStats::getRateMsgsReceived)); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java index f2efbb8b6..66d346905 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java @@ -36,34 +36,6 @@ public class PulsarAdapterUtil { private final static Logger logger = LogManager.getLogger(PulsarAdapterUtil.class); - // Supported message operation types - // TODO: websocket-producer and managed-ledger - public enum OP_TYPES { - ADMIN_TENANT("admin-tenant"), - ADMIN_NAMESPACE("admin-namespace"), - ADMIN_TOPIC("admin-topic"), - E2E_MSG_PROC_SEND("e22-msg-proc-send"), - E2E_MSG_PROC_CONSUME("e22-msg-proc-consume"), -// BATCH_MSG_SEND_START("batch-msg-send-start"), -// BATCH_MSG_SEND("batch-msg-send"), -// BATCH_MSG_SEND_END("batch-msg-send-end"), - MSG_SEND("msg-send"), - MSG_CONSUME("msg-consume"), - MSG_READ("msg-read"), - MSG_MULTI_CONSUME("msg-mt-consume"); - - public final String label; - - OP_TYPES(String label) { - this.label = label; - } - } - - - public static boolean isValidClientType(String type) { - return Arrays.stream(OP_TYPES.values()).anyMatch(t -> t.label.equals(type)); - } - public static final String MSG_SEQUENCE_NUMBER = "sequence_number"; /////// @@ -72,8 +44,10 @@ public class PulsarAdapterUtil { TOPIC_URI("topic_uri"), ASYNC_API("async_api"), USE_TRANSACTION("use_transaction"), + TRANSACT_BATCH_NUM("transact_batch_num"), ADMIN_DELOP("admin_delop"), SEQ_TRACKING("seq_tracking"), + RTT_TRACKING_FIELD("payload_traking_field"), MSG_DEDUP_BROKER("msg_dedup_broker"), E2E_STARTING_TIME_SOURCE("e2e_starting_time_source"); diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java index 740ef3348..7bfc2d60a 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java @@ -39,7 +39,7 @@ public abstract class BaseOpDispenser implements OpDispenser private final String name; protected final DriverAdapter adapter; - private boolean instrument; + protected boolean instrument; private Histogram resultSizeHistogram; private Timer successTimer; private Timer errorTimer; @@ -83,12 +83,16 @@ public abstract class BaseOpDispenser implements OpDispenser @Override public abstract T apply(long cycle); + protected String getDefaultMetricsPrefix(ParsedOp pop) { + return pop.getStaticConfigOr("alias", "UNKNOWN") + "-" + pop.getName() + "--"; + } + private void configureInstrumentation(ParsedOp pop) { this.instrument = pop.takeStaticConfigOr("instrument", false); if (instrument) { - this.successTimer = ActivityMetrics.timer(pop.getStaticConfigOr("alias", "UNKNOWN") + "-" + pop.getName() + "--success"); - this.errorTimer = ActivityMetrics.timer(pop.getStaticConfigOr("alias", "UNKNOWN") + "-" + pop.getName() + "--error"); - this.resultSizeHistogram = ActivityMetrics.histogram(pop.getStaticConfigOr("alias", "UNKNOWN") + "-" + pop.getName() + "--resultset-size"); + this.successTimer = ActivityMetrics.timer(getDefaultMetricsPrefix(pop) + "success"); + this.errorTimer = ActivityMetrics.timer(getDefaultMetricsPrefix(pop) + "error"); + this.resultSizeHistogram = ActivityMetrics.histogram(getDefaultMetricsPrefix(pop) + "resultset-size"); } } diff --git a/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java b/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java index 31008d972..188d93dd1 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java +++ b/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java @@ -83,6 +83,15 @@ public class ActivityMetrics { return metric; } + private static Metric register(String fullMetricName, MetricProvider metricProvider) { + Metric metric = get().getMetrics().get(fullMetricName); + if (metric == null) { + metric = metricProvider.getMetric(); + return get().register(fullMetricName, metric); + } + return metric; + } + private static Metric register(ScriptContext context, String name, MetricProvider metricProvider) { Metric metric = get().getMetrics().get(name); if (metric == null) { @@ -185,6 +194,11 @@ public class ActivityMetrics { return (Counter) register(named, name, Counter::new); } + public static Counter counter(String fullName) { + Counter counter = get().register(fullName, new Counter()); + return counter; + } + /** *

Create a meter associated with an activity.

*

This method ensures that if multiple threads attempt to create the same-named metric on a given activity, @@ -215,6 +229,10 @@ public class ActivityMetrics { return (Gauge) register(named, name, () -> gauge); } + public static Gauge gauge(String fullMetricsName, Gauge gauge) { + return (Gauge) register(fullMetricsName, () -> gauge); + } + @SuppressWarnings("unchecked") public static Gauge gauge(ScriptContext scriptContext, String name, Gauge gauge) { return (Gauge) register(scriptContext, name, () -> gauge); diff --git a/pom.xml b/pom.xml index 9bb917de6..c8dd8eeff 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,7 @@ adapter-tcp adapter-dynamodb adapter-mongodb + adapter-pulsar @@ -91,7 +92,7 @@ driver-jmx driver-jdbc driver-cockroachdb - driver-pulsar + From 6999a675f7a6f2fa916221baa77a67f341e63197 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 14 Nov 2022 17:03:11 -0600 Subject: [PATCH 07/14] [Snyk] Upgrade io.dropwizard.metrics:metrics-core from 4.2.10 to 4.2.12 (#765) * fix: upgrade io.dropwizard.metrics:metrics-core from 4.2.10 to 4.2.12 Snyk has created this PR to upgrade io.dropwizard.metrics:metrics-core from 4.2.10 to 4.2.12. See this package in Maven Repository: https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core/ See this project in Snyk: https://app.snyk.io/org/jshook/project/fc9e1bd9-1d9a-474f-bde4-efb614c52ffe?utm_source=github&utm_medium=referral&page=upgrade-pr * Timing change for github actions Co-authored-by: snyk-bot Co-authored-by: jeffbanks --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index d48f66855..a408b78e4 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -141,7 +141,7 @@ io.dropwizard.metrics metrics-core - 4.2.10 + 4.2.12 From 5fc22bbaa5a35d9e9cc45ed126231f57130754eb Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 14 Nov 2022 17:16:14 -0600 Subject: [PATCH 08/14] fix: upgrade com.google.code.gson:gson from 2.9.0 to 2.9.1 (#766) Snyk has created this PR to upgrade com.google.code.gson:gson from 2.9.0 to 2.9.1. See this package in Maven Repository: https://mvnrepository.com/artifact/com.google.code.gson/gson/ See this project in Snyk: https://app.snyk.io/org/jshook/project/fc9e1bd9-1d9a-474f-bde4-efb614c52ffe?utm_source=github&utm_medium=referral&page=upgrade-pr Co-authored-by: snyk-bot --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index a408b78e4..a7cd31bad 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -271,7 +271,7 @@ com.google.code.gson gson - 2.9.0 + 2.9.1 From 2158dfa339088367bc1e4ae693c00e9d978a353e Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 14 Nov 2022 17:16:39 -0600 Subject: [PATCH 09/14] fix: upgrade io.netty:netty-handler from 4.1.81.Final to 4.1.82.Final (#767) Snyk has created this PR to upgrade io.netty:netty-handler from 4.1.81.Final to 4.1.82.Final. See this package in Maven Repository: https://mvnrepository.com/artifact/io.netty/netty-handler/ See this project in Snyk: https://app.snyk.io/org/jshook/project/faf9c31c-14e2-456a-9f66-9dda91c923d9?utm_source=github&utm_medium=referral&page=upgrade-pr Co-authored-by: snyk-bot --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index a7cd31bad..e0dcc4f7e 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -189,7 +189,7 @@ io.netty netty-handler - 4.1.81.Final + 4.1.82.Final From 33e330dc350f96c90590daa6a7cdf8a56d1a2344 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 14 Nov 2022 18:32:52 -0600 Subject: [PATCH 10/14] [Snyk] Security upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.325 to 1.12.330 (#768) * fix: nb-api/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMFASTERXMLJACKSONCORE-3038424 * Timing change for github actions; warnings cleanup/refactor Co-authored-by: snyk-bot Co-authored-by: jeffbanks --- nb-api/pom.xml | 2 +- .../testing/ExitStatusIntegrationTests.java | 25 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/nb-api/pom.xml b/nb-api/pom.xml index d2aea36ec..5b4546aea 100644 --- a/nb-api/pom.xml +++ b/nb-api/pom.xml @@ -97,7 +97,7 @@ com.amazonaws aws-java-sdk-s3 - 1.12.325 + 1.12.330 diff --git a/nbr/src/test/java/io/nosqlbench/cli/testing/ExitStatusIntegrationTests.java b/nbr/src/test/java/io/nosqlbench/cli/testing/ExitStatusIntegrationTests.java index f50704c62..3d0185c04 100644 --- a/nbr/src/test/java/io/nosqlbench/cli/testing/ExitStatusIntegrationTests.java +++ b/nbr/src/test/java/io/nosqlbench/cli/testing/ExitStatusIntegrationTests.java @@ -19,40 +19,40 @@ package io.nosqlbench.cli.testing; import org.junit.jupiter.api.Test; import java.util.Optional; -import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; -public class ExitStatusIntegrationTests { +class ExitStatusIntegrationTests { private final String java = Optional.ofNullable(System.getenv( - "JAVA_HOME")).map(v -> v+"/bin/java").orElse("java"); + "JAVA_HOME")).map(v -> v + "/bin/java").orElse("java"); private final static String JARNAME = "target/nbr.jar"; + @Test - public void testExitStatusOnBadParam() { + void testExitStatusOnBadParam() { ProcessInvoker invoker = new ProcessInvoker(); invoker.setLogDir("logs/test"); ProcessResult result = invoker.run("exitstatus_badparam", 15, java, "-jar", JARNAME, "--logs-dir", "logs/test/badparam/", - "badparam" + "badparam" ); assertThat(result.exception).isNull(); - String stderr = result.getStderrData().stream().collect(Collectors.joining("\n")); + String stderr = String.join("\n", result.getStderrData()); assertThat(stderr).contains("Scenario stopped due to error"); assertThat(result.exitStatus).isEqualTo(2); } @Test - public void testExitStatusOnActivityInitException() { + void testExitStatusOnActivityInitException() { ProcessInvoker invoker = new ProcessInvoker(); invoker.setLogDir("logs/test"); ProcessResult result = invoker.run("exitstatus_initexception", 15, java, "-jar", JARNAME, "--logs-dir", "logs/test/initerror", "run", - "driver=diag", "op=initdelay:initdelay=notanumber" + "driver=diag", "op=initdelay:initdelay=notanumber" ); assertThat(result.exception).isNull(); - String stderr = result.getStdoutData().stream().collect(Collectors.joining("\n")); + String stderr = String.join("\n", result.getStdoutData()); assertThat(stderr).contains("For input string: \"notanumber\""); assertThat(result.exitStatus).isEqualTo(2); } @@ -73,19 +73,18 @@ public class ExitStatusIntegrationTests { // } @Test - public void testExitStatusOnActivityOpException() { + void testExitStatusOnActivityOpException() { ProcessInvoker invoker = new ProcessInvoker(); invoker.setLogDir("logs/test"); ProcessResult result = invoker.run("exitstatus_asyncstoprequest", 30, java, "-jar", JARNAME, "--logs-dir", "logs/test/asyncstop", "run", - "driver=diag", "cyclerate=5", "op=erroroncycle:erroroncycle=10", "cycles=2000", "-vvv" + "driver=diag", "cyclerate=1", "op=erroroncycle:erroroncycle=10", "cycles=2000", "-vvv" ); assertThat(result.exception).isNull(); - String stdout = result.getStdoutData().stream().collect(Collectors.joining("\n")); + String stdout = String.join("\n", result.getStdoutData()); assertThat(stdout).contains("Diag was requested to stop on cycle 10"); assertThat(result.exitStatus).isEqualTo(2); } - } From 0c71696b15a00125ca866596f252d318ef4ab493 Mon Sep 17 00:00:00 2001 From: yabinmeng Date: Wed, 16 Nov 2022 15:37:26 -0600 Subject: [PATCH 11/14] Completed NB4 Pulsar driver code migration to NB5. But there are still a few lingering issues due to the difference of how NB4 vs NB5 driver/adapter interacts with the NB engine. --- .../adapter/pulsar/PulsarOpMapper.java | 6 +- .../adapter/pulsar/PulsarOpType.java | 1 + .../adapter/pulsar/PulsarSpace.java | 3 - .../dispensers/AdminNamespaceOpDispenser.java | 5 + .../dispensers/AdminTenantOpDispenser.java | 5 + .../dispensers/AdminTopicOpDispenser.java | 5 + .../MessageConsumerOpDispenser.java | 70 ++++- .../MessageProducerOpDispenser.java | 40 ++- .../dispensers/MessageReaderOpDispenser.java | 26 +- .../dispensers/PulsarAdminOpDispenser.java | 6 + .../dispensers/PulsarBaseOpDispenser.java | 210 +++++++++++-- .../dispensers/PulsarClientOpDispenser.java | 73 ++++- .../adapter/pulsar/ops/AdminNamespaceOp.java | 6 +- .../adapter/pulsar/ops/AdminTenantOp.java | 7 +- .../adapter/pulsar/ops/AdminTopicOp.java | 7 +- .../adapter/pulsar/ops/MessageConsumerOp.java | 277 +++++++++++++++++- .../adapter/pulsar/ops/MessageProducerOp.java | 270 ++++++++++++++++- .../adapter/pulsar/ops/MessageReaderOp.java | 20 +- .../adapter/pulsar/ops/PulsarAdminOp.java | 10 +- .../adapter/pulsar/ops/PulsarClientOp.java | 68 ++++- .../adapter/pulsar/ops/PulsarOp.java | 8 + .../util/EndToEndStartingTimeSource.java | 26 ++ .../MessageSequenceNumberSendingHandler.java | 109 +++++++ .../pulsar/util/PulsarAdapterMetrics.java | 113 ++++--- .../pulsar/util/PulsarAdapterUtil.java | 2 +- ...vroUtil.java => PulsarAvroSchemaUtil.java} | 2 +- .../util/ReceivedMessageSequenceTracker.java | 169 +++++++++++ .../api/engine/metrics/ActivityMetrics.java | 9 - 28 files changed, 1435 insertions(+), 118 deletions(-) create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/EndToEndStartingTimeSource.java create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/MessageSequenceNumberSendingHandler.java rename adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/{AvroUtil.java => PulsarAvroSchemaUtil.java} (99%) create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/ReceivedMessageSequenceTracker.java diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java index 8620eecba..a2b33b9a7 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java @@ -83,7 +83,11 @@ public class PulsarOpMapper implements OpMapper { new MessageProducerOpDispenser(adapter, op, opType.targetFunction, pulsarSpace); case MessageConsume -> new MessageConsumerOpDispenser(adapter, op, opType.targetFunction, pulsarSpace); - case MessageRead -> + ////////////////////////// + // NOTE: not sure how useful to have Pulsar message reader API in the NB performance testing + // currently, the reader API in NB Pulsar driver is no-op (see TDOD in MessageReaderOp) + ////////////////////////// + case MessageRead -> new MessageReaderOpDispenser(adapter, op, opType.targetFunction, pulsarSpace); }; } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java index 278137169..f6f9be451 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java @@ -26,6 +26,7 @@ public enum PulsarOpType { AdminNamespace("admin-namespace"), AdminTopic("admin-topic"), MessageProduce("msg-send"), + // This also supports multi-topic message consumption MessageConsume("msg-consume"), MessageRead("msg-read"); diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java index 487295c1f..93a92c70b 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java @@ -34,10 +34,7 @@ import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.api.*; import org.apache.pulsar.common.schema.KeyValueEncodingType; -import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; public class PulsarSpace { diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java index 8b5fd31d1..eb41ce1ed 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java @@ -20,12 +20,16 @@ import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.AdminNamespaceOp; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.pulsar.client.admin.PulsarAdmin; import java.util.function.LongFunction; public class AdminNamespaceOpDispenser extends PulsarAdminOpDispenser { + private final static Logger logger = LogManager.getLogger("AdminNamespaceOpDispenser"); + public AdminNamespaceOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, @@ -36,6 +40,7 @@ public class AdminNamespaceOpDispenser extends PulsarAdminOpDispenser { @Override public AdminNamespaceOp apply(long cycle) { return new AdminNamespaceOp( + pulsarAdapterMetrics, pulsarAdmin, asyncApiFunc.apply(cycle), adminDelOpFunc.apply(cycle), diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java index 21e479283..e19897ed2 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java @@ -20,6 +20,8 @@ import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.AdminTenantOp; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.pulsar.client.admin.PulsarAdmin; import java.util.*; @@ -27,6 +29,8 @@ import java.util.function.LongFunction; public class AdminTenantOpDispenser extends PulsarAdminOpDispenser { + private final static Logger logger = LogManager.getLogger("AdminTenantOpDispenser"); + private final LongFunction> adminRolesFunc; private final LongFunction> allowedClustersFunc; public AdminTenantOpDispenser(DriverAdapter adapter, @@ -42,6 +46,7 @@ public class AdminTenantOpDispenser extends PulsarAdminOpDispenser { @Override public AdminTenantOp apply(long cycle) { return new AdminTenantOp( + pulsarAdapterMetrics, pulsarAdmin, asyncApiFunc.apply(cycle), adminDelOpFunc.apply(cycle), diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java index 87e892fb7..c92436128 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java @@ -20,12 +20,16 @@ import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.AdminTopicOp; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.pulsar.client.admin.PulsarAdmin; import java.util.function.LongFunction; public class AdminTopicOpDispenser extends PulsarAdminOpDispenser { + private final static Logger logger = LogManager.getLogger("AdminTopicOpDispenser"); + private final LongFunction enablePartFunc; private final LongFunction partNumFunc; @@ -44,6 +48,7 @@ public class AdminTopicOpDispenser extends PulsarAdminOpDispenser { public AdminTopicOp apply(long cycle) { return new AdminTopicOp( + pulsarAdapterMetrics, pulsarAdmin, asyncApiFunc.apply(cycle), adminDelOpFunc.apply(cycle), diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java index 67e7dfabc..82edc63a9 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java @@ -18,24 +18,88 @@ package io.nosqlbench.adapter.pulsar.dispensers; import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.MessageConsumerOp; +import io.nosqlbench.adapter.pulsar.util.EndToEndStartingTimeSource; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; +import io.nosqlbench.adapter.pulsar.util.ReceivedMessageSequenceTracker; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.Schema; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.api.Consumer; +import java.util.HashMap; +import java.util.Map; import java.util.function.LongFunction; public class MessageConsumerOpDispenser extends PulsarClientOpDispenser { + private final static Logger logger = LogManager.getLogger("MessageConsumerOpDispenser"); + + public static final String TOPIC_PATTERN_OP_PARAM = "topic_pattern"; + public static final String SUBSCRIPTION_NAME_OP_PARAM = "subscription_name"; + public static final String SUBSCRIPTION_TYPE_OP_PARAM = "subscription_type"; + public static final String CONSUMER_NAME_OP_PARAM = "consumer_name"; + public static final String RANGES_OP_PARAM = "ranges"; + + private final LongFunction topicPatternFunc; + private final LongFunction subscriptionNameFunc; + private final LongFunction subscriptionTypeFunc; + private final LongFunction cycleConsumerNameFunc; + private final LongFunction rangesFunc; + private final LongFunction e2eStartTimeSrcParamStrFunc; + private final LongFunction consumerFunction; + + private final ThreadLocal> receivedMessageSequenceTrackersForTopicThreadLocal = + ThreadLocal.withInitial(HashMap::new); + public MessageConsumerOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, PulsarSpace pulsarSpace) { super(adapter, op, tgtNameFunc, pulsarSpace); + + this.topicPatternFunc = lookupOptionalStrOpValueFunc(TOPIC_PATTERN_OP_PARAM); + this.subscriptionNameFunc = lookupMandtoryStrOpValueFunc(SUBSCRIPTION_NAME_OP_PARAM); + this.subscriptionTypeFunc = lookupOptionalStrOpValueFunc(SUBSCRIPTION_TYPE_OP_PARAM); + this.cycleConsumerNameFunc = lookupOptionalStrOpValueFunc(CONSUMER_NAME_OP_PARAM); + this.rangesFunc = lookupOptionalStrOpValueFunc(RANGES_OP_PARAM); + this.e2eStartTimeSrcParamStrFunc = lookupOptionalStrOpValueFunc( + PulsarAdapterUtil.DOC_LEVEL_PARAMS.E2E_STARTING_TIME_SOURCE.label, "none"); + this.consumerFunction = (l) -> getConsumer( + tgtNameFunc.apply(l), + topicPatternFunc.apply(l), + subscriptionNameFunc.apply(l), + subscriptionTypeFunc.apply(l), + cycleConsumerNameFunc.apply(l), + rangesFunc.apply(l)); } @Override public MessageConsumerOp apply(long cycle) { - return new MessageConsumerOp(pulsarClient, pulsarSchema); + return new MessageConsumerOp( + pulsarAdapterMetrics, + pulsarClient, + pulsarSchema, + asyncApiFunc.apply(cycle), + useTransactFunc.apply(cycle), + seqTrackingFunc.apply(cycle), + transactSupplierFunc.apply(cycle), + payloadRttFieldFunc.apply(cycle), + EndToEndStartingTimeSource.valueOf(e2eStartTimeSrcParamStrFunc.apply(cycle).toUpperCase()), + this::getReceivedMessageSequenceTracker, + consumerFunction.apply(cycle), + pulsarSpace.getPulsarNBClientConf().getConsumerTimeoutSeconds() + ); + } + + private ReceivedMessageSequenceTracker getReceivedMessageSequenceTracker(String topicName) { + return receivedMessageSequenceTrackersForTopicThreadLocal.get() + .computeIfAbsent(topicName, k -> createReceivedMessageSequenceTracker()); + } + + private ReceivedMessageSequenceTracker createReceivedMessageSequenceTracker() { + return new ReceivedMessageSequenceTracker(pulsarAdapterMetrics.getMsgErrOutOfSeqCounter(), + pulsarAdapterMetrics.getMsgErrDuplicateCounter(), + pulsarAdapterMetrics.getMsgErrLossCounter()); } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java index ecb765af1..9f250e151 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java @@ -20,22 +20,56 @@ import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.MessageProducerOp; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.Schema; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.api.Producer; +import java.util.Optional; import java.util.function.LongFunction; public class MessageProducerOpDispenser extends PulsarClientOpDispenser { + private final static Logger logger = LogManager.getLogger("MessageProducerOpDispenser"); + + public static final String PRODUCER_NAME_OP_PARAM = "producer_name"; + public static final String MSG_KEY_OP_PARAM = "msg_key"; + public static final String MSG_PROP_OP_PARAM = "msg_prop"; + public static final String MSG_VALUE_OP_PARAM = "msg_value"; + + private final LongFunction cycleProducerNameFunc; + private final LongFunction> producerFunc; + private final LongFunction msgKeyFunc; + private final LongFunction msgPropFunc; + private final LongFunction msgValueFunc; + public MessageProducerOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, PulsarSpace pulsarSpace) { super(adapter, op, tgtNameFunc, pulsarSpace); + + this.cycleProducerNameFunc = lookupOptionalStrOpValueFunc(PRODUCER_NAME_OP_PARAM); + this.producerFunc = (l) -> getProducer(tgtNameFunc.apply(l), cycleProducerNameFunc.apply(l)); + this.msgKeyFunc = lookupOptionalStrOpValueFunc(MSG_KEY_OP_PARAM); + this.msgPropFunc = lookupOptionalStrOpValueFunc(MSG_PROP_OP_PARAM); + this.msgValueFunc = lookupMandtoryStrOpValueFunc(MSG_VALUE_OP_PARAM); } @Override public MessageProducerOp apply(long cycle) { - return new MessageProducerOp(pulsarClient, pulsarSchema); + return new MessageProducerOp( + pulsarAdapterMetrics, + pulsarClient, + pulsarSchema, + asyncApiFunc.apply(cycle), + useTransactFunc.apply(cycle), + seqTrackingFunc.apply(cycle), + transactSupplierFunc.apply(cycle), + errSimuTypeSetFunc.apply(cycle), + producerFunc.apply(cycle), + msgKeyFunc.apply(cycle), + msgPropFunc.apply(cycle), + msgValueFunc.apply(cycle) + ); } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java index 53394f44b..f4e0a292a 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java @@ -18,24 +18,48 @@ package io.nosqlbench.adapter.pulsar.dispensers; import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.MessageReaderOp; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; import java.util.function.LongFunction; public class MessageReaderOpDispenser extends PulsarClientOpDispenser { + private final static Logger logger = LogManager.getLogger("MessageReaderOpDispenser"); + + private final LongFunction cycleReaderNameFunc; + private final LongFunction msgStartPosStrFunc; + private final LongFunction readerFunc; + public MessageReaderOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, PulsarSpace pulsarSpace) { super(adapter, op, tgtNameFunc, pulsarSpace); + + this.cycleReaderNameFunc = lookupMandtoryStrOpValueFunc("reader_name"); + this.msgStartPosStrFunc = lookupOptionalStrOpValueFunc( + "start_msg_position", PulsarAdapterUtil.READER_MSG_POSITION_TYPE.earliest.label); + this.readerFunc = (l) -> getReader( + tgtNameFunc.apply(l), + cycleReaderNameFunc.apply(l), + msgStartPosStrFunc.apply(l)); } @Override public MessageReaderOp apply(long cycle) { - return new MessageReaderOp(pulsarClient, pulsarSchema); + + return new MessageReaderOp( + pulsarAdapterMetrics, + pulsarClient, + pulsarSchema, + asyncApiFunc.apply(cycle), + readerFunc.apply(cycle)); } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java index 3bd2bd609..a56752e29 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java @@ -20,12 +20,17 @@ import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.pulsar.client.admin.PulsarAdmin; import java.util.function.LongFunction; public abstract class PulsarAdminOpDispenser extends PulsarBaseOpDispenser { + private final static Logger logger = LogManager.getLogger("PulsarAdminOpDispenser"); + + protected final PulsarAdmin pulsarAdmin; protected final LongFunction adminDelOpFunc; @@ -34,6 +39,7 @@ public abstract class PulsarAdminOpDispenser extends PulsarBaseOpDispenser { LongFunction tgtNameFunc, PulsarSpace pulsarSpace) { super(adapter, op, tgtNameFunc, pulsarSpace); + this.pulsarAdmin = pulsarSpace.getPulsarAdmin(); // Doc-level parameter: admin_delop diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java index a2ff631e4..844c8c57e 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java @@ -22,6 +22,10 @@ import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; import io.nosqlbench.adapter.pulsar.ops.PulsarOp; import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; +import io.nosqlbench.api.config.NBNamedElement; +import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter; +import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters; +import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec; import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; @@ -36,21 +40,29 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.function.LongFunction; import java.util.function.Predicate; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; -public abstract class PulsarBaseOpDispenser extends BaseOpDispenser { +public abstract class PulsarBaseOpDispenser extends BaseOpDispenser implements NBNamedElement { private final static Logger logger = LogManager.getLogger("PulsarBaseOpDispenser"); protected final ParsedOp parsedOp; - protected final LongFunction asyncApiFunc; - protected final LongFunction tgtNameFunc; protected final PulsarSpace pulsarSpace; - + protected final PulsarAdapterMetrics pulsarAdapterMetrics; private final ConcurrentHashMap> producers = new ConcurrentHashMap<>(); private final ConcurrentHashMap> consumers = new ConcurrentHashMap<>(); private final ConcurrentHashMap> readers = new ConcurrentHashMap<>(); - protected final PulsarAdapterMetrics pulsarAdapterMetrics; + + protected final LongFunction asyncApiFunc; + protected final LongFunction tgtNameFunc; + + protected final int totalThreadNum; + + protected final long totalCycleNum; + + protected RateLimiter per_thread_cyclelimiter; public PulsarBaseOpDispenser(DriverAdapter adapter, ParsedOp op, @@ -67,12 +79,28 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser per_thread_cyclelimiter = + RateLimiters.createOrUpdate(this, "cycles", per_thread_cyclelimiter, spec)); } + @Override + public String getName() { + return "PulsarBaseOpDispenser"; + } + + public PulsarSpace getPulsarSpace() { return pulsarSpace; } + protected LongFunction lookupStaticBoolConfigValueFunc(String paramName, boolean defaultValue) { LongFunction booleanLongFunction; booleanLongFunction = (l) -> parsedOp.getOptionalStaticConfig(paramName, String.class) @@ -83,19 +111,6 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser lookupStaticIntOpValueFunc(String paramName, int defaultValue) { - LongFunction integerLongFunction; - integerLongFunction = (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) - .filter(Predicate.not(String::isEmpty)) - .map(value -> NumberUtils.toInt(value)) - .map(value -> { - if (value < 0) return 0; - else return value; - }).orElse(defaultValue); - logger.info("{}: {}", paramName, integerLongFunction.apply(0)); - return integerLongFunction; - } - protected LongFunction> lookupStaticStrSetOpValueFunc(String paramName) { LongFunction> setStringLongFunction; setStringLongFunction = (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) @@ -116,6 +131,42 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser lookupStaticIntOpValueFunc(String paramName, int defaultValue) { + LongFunction integerLongFunction; + integerLongFunction = (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) + .filter(Predicate.not(String::isEmpty)) + .map(value -> NumberUtils.toInt(value)) + .map(value -> { + if (value < 0) return 0; + else return value; + }).orElse(defaultValue); + logger.info("{}: {}", paramName, integerLongFunction.apply(0)); + return integerLongFunction; + } + + // If the corresponding Op parameter is not provided, use the specified default value + protected LongFunction lookupOptionalStrOpValueFunc(String paramName, String defaultValue) { + LongFunction stringLongFunction; + stringLongFunction = parsedOp.getAsOptionalFunction(paramName, String.class) + .orElse((l) -> defaultValue); + logger.info("{}: {}", paramName, stringLongFunction.apply(0)); + + return stringLongFunction; + } + protected LongFunction lookupOptionalStrOpValueFunc(String paramName) { + return lookupOptionalStrOpValueFunc(paramName, ""); + } + + // Mandatory Op parameter. Throw an error if not specified or having empty value + protected LongFunction lookupMandtoryStrOpValueFunc(String paramName) { + LongFunction stringLongFunction; + stringLongFunction = parsedOp.getAsRequiredFunction(paramName, String.class); + logger.info("{}: {}", paramName, stringLongFunction.apply(0)); + + return stringLongFunction; + } + /** * Get a proper Pulsar API metrics prefix depending on the API type * @@ -154,6 +205,8 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser tenant_namespace_topicname .replace("/", "_"); + + apiMetricsPrefix += "--"; } return apiMetricsPrefix; @@ -257,6 +310,61 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser getEffectiveConsumerTopicNameList(String cycleTopicNameListStr) { + String effectiveTopicNamesStr = getEffectiveConsumerTopicNameListStr(cycleTopicNameListStr); + + String[] names = effectiveTopicNamesStr.split("[;,]"); + ArrayList effectiveTopicNameList = new ArrayList<>(); + + for (String name : names) { + if (!StringUtils.isBlank(name)) + effectiveTopicNameList.add(name.trim()); + } + + return effectiveTopicNameList; + } + + private String getEffectiveConsumerTopicPatternStr(String cycleTopicPatternStr) { + if (!StringUtils.isBlank(cycleTopicPatternStr)) { + return cycleTopicPatternStr; + } + + String globalTopicsPattern = pulsarSpace.getPulsarNBClientConf().getConsumerTopicPattern(); + if (!StringUtils.isBlank(globalTopicsPattern)) { + return globalTopicsPattern; + } + + return ""; + } + + private Pattern getEffectiveConsumerTopicPattern(String cycleTopicPatternStr) { + String effectiveTopicPatternStr = getEffectiveConsumerTopicPatternStr(cycleTopicPatternStr); + Pattern topicsPattern; + try { + if (!StringUtils.isBlank(effectiveTopicPatternStr)) + topicsPattern = Pattern.compile(effectiveTopicPatternStr); + else + topicsPattern = null; + } catch (PatternSyntaxException pse) { + topicsPattern = null; + } + return topicsPattern; + } + + // Subscription name is NOT mandatory // - It can be set at either global level or cycle level // - If set at both levels, cycle level setting takes precedence @@ -322,22 +430,47 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser getConsumer(String cycleTopicName, + public Consumer getConsumer(String cycleTopicNameListStr, + String cycleTopicPatternStr, String cycleSubscriptionName, String cycleSubscriptionType, String cycleConsumerName, String cycleKeySharedSubscriptionRanges) { + + List topicNameList = getEffectiveConsumerTopicNameList(cycleTopicNameListStr); + String topicPatternStr = getEffectiveConsumerTopicPatternStr(cycleTopicPatternStr); + Pattern topicPattern = getEffectiveConsumerTopicPattern(cycleTopicPatternStr); String subscriptionName = getEffectiveSubscriptionName(cycleSubscriptionName); SubscriptionType subscriptionType = getEffectiveSubscriptionType(cycleSubscriptionType); String consumerName = getEffectiveConsumerName(cycleConsumerName); - if (StringUtils.isAnyBlank(cycleTopicName, subscriptionName)) { + if ( subscriptionType.equals(SubscriptionType.Exclusive) && (totalThreadNum > 1) ) { throw new PulsarAdapterInvalidParamException( - "Must specify a topic name and a subscription name when creating a consumer!"); + MessageConsumerOpDispenser.SUBSCRIPTION_TYPE_OP_PARAM, + "creating multiple consumers of \"Exclusive\" subscription type under the same subscription name"); } - String consumerCacheKey = PulsarAdapterUtil.buildCacheKey(consumerName, subscriptionName, cycleTopicName); + if ( (topicNameList.isEmpty() && (topicPattern == null)) || + (!topicNameList.isEmpty() && (topicPattern != null)) ) { + throw new PulsarAdapterInvalidParamException( + "Invalid combination of topic name(s) and topic patterns; only specify one parameter!"); + } + + boolean multiTopicConsumer = (topicNameList.size() > 1 || (topicPattern != null)); + + String consumerTopicListString; + if (!topicNameList.isEmpty()) { + consumerTopicListString = String.join("|", topicNameList); + } else { + consumerTopicListString = topicPatternStr; + } + + String consumerCacheKey = PulsarAdapterUtil.buildCacheKey( + consumerName, + subscriptionName, + consumerTopicListString); Consumer consumer = consumers.get(consumerCacheKey); + if (consumer == null) { PulsarClient pulsarClient = pulsarSpace.getPulsarClient(); @@ -355,13 +488,32 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser consumerBuilder = pulsarClient. - newConsumer(pulsarSpace.getPulsarSchema()). + ConsumerBuilder consumerBuilder; + + if (!multiTopicConsumer) { + assert (topicNameList.size() == 1); + consumerBuilder = pulsarClient.newConsumer(pulsarSpace.getPulsarSchema()); + consumerBuilder.topic(topicNameList.get(0)); + } + else { + consumerBuilder = pulsarClient.newConsumer(); + if (!topicNameList.isEmpty()) { + assert (topicNameList.size() > 1); + consumerBuilder.topics(topicNameList); + } + else { + consumerBuilder.topicsPattern(topicPattern); + } + } + + consumerBuilder. loadConf(consumerConf). - topic(cycleTopicName). subscriptionName(subscriptionName). subscriptionType(subscriptionType); + if (!StringUtils.isBlank(consumerName)) + consumerBuilder.consumerName(consumerName); + if (subscriptionType == SubscriptionType.Key_Shared) { KeySharedPolicy keySharedPolicy = KeySharedPolicy.autoSplitHashRange(); if (cycleKeySharedSubscriptionRanges != null && !cycleKeySharedSubscriptionRanges.isEmpty()) { @@ -372,10 +524,6 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser pulsarSchema; protected final LongFunction useTransactFunc; - protected final LongFunction transactBatchNumFunc; + // TODO: add support for "operation number per transaction" + // protected final LongFunction transactBatchNumFunc; protected final LongFunction seqTrackingFunc; + protected final LongFunction payloadRttFieldFunc; + protected final LongFunction> transactSupplierFunc; + protected final LongFunction> errSimuTypeSetFunc; public PulsarClientOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc, PulsarSpace pulsarSpace) { super(adapter, op, tgtNameFunc, pulsarSpace); + this.pulsarClient = pulsarSpace.getPulsarClient(); this.pulsarSchema = pulsarSpace.getPulsarSchema(); @@ -46,12 +64,61 @@ public abstract class PulsarClientOpDispenser extends PulsarBaseOpDispenser { this.useTransactFunc = lookupStaticBoolConfigValueFunc( PulsarAdapterUtil.DOC_LEVEL_PARAMS.USE_TRANSACTION.label, false); + // TODO: add support for "operation number per transaction" // Doc-level parameter: transact_batch_num - this.transactBatchNumFunc = lookupStaticIntOpValueFunc( - PulsarAdapterUtil.DOC_LEVEL_PARAMS.TRANSACT_BATCH_NUM.label, 1); + // this.transactBatchNumFunc = lookupStaticIntOpValueFunc( + // PulsarAdapterUtil.DOC_LEVEL_PARAMS.TRANSACT_BATCH_NUM.label, 1); // Doc-level parameter: seq_tracking this.seqTrackingFunc = lookupStaticBoolConfigValueFunc( PulsarAdapterUtil.DOC_LEVEL_PARAMS.SEQ_TRACKING.label, false); + + // Doc-level parameter: payload-tracking-field + this.payloadRttFieldFunc = (l) -> parsedOp.getStaticConfigOr( + PulsarAdapterUtil.DOC_LEVEL_PARAMS.RTT_TRACKING_FIELD.label, ""); + + this.transactSupplierFunc = (l) -> getTransactionSupplier(); + + this.errSimuTypeSetFunc = getStaticErrSimuTypeSetOpValueFunc(); + } + + protected Supplier getTransactionSupplier() { + return () -> { + try (Timer.Context time = pulsarAdapterMetrics.getCommitTransactionTimer().time() ){ + return pulsarClient + .newTransaction() + .build() + .get(); + } catch (ExecutionException | InterruptedException err) { + if (logger.isWarnEnabled()) { + logger.warn("Error while starting a new transaction", err); + } + throw new RuntimeException(err); + } catch (PulsarClientException err) { + throw new RuntimeException("Transactions are not enabled on Pulsar Client, " + + "please set client.enableTransaction=true in your Pulsar Client configuration"); + } + }; + } + + protected LongFunction> getStaticErrSimuTypeSetOpValueFunc() { + LongFunction> setStringLongFunction; + setStringLongFunction = (l) -> parsedOp.getOptionalStaticValue("seqerr_simu", String.class) + .filter(Predicate.not(String::isEmpty)) + .map(value -> { + Set set = new HashSet<>(); + + if (StringUtils.contains(value,',')) { + set = Arrays.stream(value.split(",")) + .map(PulsarAdapterUtil.SEQ_ERROR_SIMU_TYPE::parseSimuType) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + return set; + }).orElse(Collections.emptySet()); + logger.info("seqerr_simu: {}", setStringLongFunction.apply(0)); + return setStringLongFunction; } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java index 71e6f79ee..2caca86bc 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java @@ -17,6 +17,7 @@ package io.nosqlbench.adapter.pulsar.ops; import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -33,11 +34,12 @@ public class AdminNamespaceOp extends PulsarAdminOp { // in format: / private final String nsName; - public AdminNamespaceOp(PulsarAdmin pulsarAdmin, + public AdminNamespaceOp(PulsarAdapterMetrics pulsarAdapterMetrics, + PulsarAdmin pulsarAdmin, boolean asyncApi, boolean adminDelOp, String nsName) { - super(pulsarAdmin, asyncApi, adminDelOp); + super(pulsarAdapterMetrics, pulsarAdmin, asyncApi, adminDelOp); this.nsName = nsName; } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java index b40013ff7..f3050e82a 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java @@ -18,6 +18,7 @@ package io.nosqlbench.adapter.pulsar.ops; import io.nosqlbench.adapter.pulsar.PulsarDriverAdapter; import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -37,13 +38,15 @@ public class AdminTenantOp extends PulsarAdminOp { private final Set allowedClusters; private final String tntName; - public AdminTenantOp(PulsarAdmin pulsarAdmin, + public AdminTenantOp(PulsarAdapterMetrics pulsarAdapterMetrics, + PulsarAdmin pulsarAdmin, boolean asyncApi, boolean adminDelOp, String tntName, Set adminRoles, Set allowedClusters) { - super(pulsarAdmin, asyncApi, adminDelOp); + super(pulsarAdapterMetrics, pulsarAdmin, asyncApi, adminDelOp); + this.tntName = tntName; this.adminRoles = adminRoles; this.allowedClusters = allowedClusters; diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java index 2c3735658..507c13199 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java @@ -17,6 +17,7 @@ package io.nosqlbench.adapter.pulsar.ops; import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -35,13 +36,15 @@ public class AdminTopicOp extends PulsarAdminOp { private final boolean enablePart; private final int partNum; - public AdminTopicOp(PulsarAdmin pulsarAdmin, + public AdminTopicOp(PulsarAdapterMetrics pulsarAdapterMetrics, + PulsarAdmin pulsarAdmin, boolean asyncApi, boolean adminDelOp, String topicName, boolean enablePart, int partNum) { - super(pulsarAdmin, asyncApi, adminDelOp); + super(pulsarAdapterMetrics, pulsarAdmin, asyncApi, adminDelOp); + this.topicName = topicName; this.enablePart = enablePart; this.partNum = partNum; diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java index d9f28c7d3..71d5f5886 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java @@ -16,16 +16,285 @@ package io.nosqlbench.adapter.pulsar.ops; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.Schema; +import com.codahale.metrics.Timer; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; +import io.nosqlbench.adapter.pulsar.util.*; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.api.*; +import org.apache.pulsar.client.api.schema.GenericObject; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.shade.org.apache.avro.AvroRuntimeException; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.function.Supplier; public class MessageConsumerOp extends PulsarClientOp { - public MessageConsumerOp(PulsarClient pulsarClient, Schema pulsarSchema) { - super(pulsarClient, pulsarSchema); + + private final static Logger logger = LogManager.getLogger(MessageConsumerOp.class); + + private final boolean useTransact; + private final boolean seqTracking; + private final Supplier transactSupplier; + private final String payloadRttField; + private final EndToEndStartingTimeSource e2eStartingTimeSrc; + private final Function receivedMessageSequenceTrackerForTopic; + private final Consumer consumer; + private final int consumerTimeoutInSec; + + public MessageConsumerOp(PulsarAdapterMetrics pulsarAdapterMetrics, + PulsarClient pulsarClient, + Schema pulsarSchema, + boolean asyncApi, + boolean useTransact, + boolean seqTracking, + Supplier transactSupplier, + String payloadRttField, + EndToEndStartingTimeSource e2eStartingTimeSrc, + Function receivedMessageSequenceTrackerForTopic, + Consumer consumer, + int consumerTimeoutInSec) { + super(pulsarAdapterMetrics, pulsarClient, pulsarSchema, asyncApi); + + this.useTransact = useTransact; + this.seqTracking = seqTracking; + this.transactSupplier = transactSupplier; + this.payloadRttField = payloadRttField; + this.e2eStartingTimeSrc = e2eStartingTimeSrc; + this.receivedMessageSequenceTrackerForTopic = receivedMessageSequenceTrackerForTopic; + this.consumer = consumer; + this.consumerTimeoutInSec = consumerTimeoutInSec; } @Override public Object apply(long value) { + final Transaction transaction; + if (useTransact) { + // if you are in a transaction you cannot set the schema per-message + transaction = transactSupplier.get(); + } + else { + transaction = null; + } + + if (!asyncApi) { + try { + Message message; + + if (consumerTimeoutInSec <= 0) { + // wait forever + message = consumer.receive(); + } + else { + message = consumer.receive(consumerTimeoutInSec, TimeUnit.SECONDS); + if (message == null) { + if ( logger.isDebugEnabled() ) { + logger.debug("Failed to sync-receive a message before time out ({} seconds)", consumerTimeoutInSec); + } + } + } + + handleMessage(transaction, message); + } + catch (Exception e) { + throw new PulsarAdapterUnexpectedException("" + + "Sync message receiving failed - timeout value: " + consumerTimeoutInSec + " seconds "); + } + } + else { + try { + CompletableFuture> msgRecvFuture = consumer.receiveAsync(); + if (useTransact) { + // add commit step + msgRecvFuture = msgRecvFuture.thenCompose(msg -> { + Timer.Context ctx = transactionCommitTimer.time(); + return transaction + .commit() + .whenComplete((m,e) -> ctx.close()) + .thenApply(v-> msg); + } + ); + } + + msgRecvFuture.thenAccept(message -> { + try { + handleMessage(transaction, message); + } catch (PulsarClientException | TimeoutException e) { + pulsarActivity.asyncOperationFailed(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + pulsarActivity.asyncOperationFailed(e.getCause()); + } + }).exceptionally(ex -> { + pulsarActivity.asyncOperationFailed(ex); + return null; + }); + } + catch (Exception e) { + throw new PulsarAdapterUnexpectedException(e); + } + } + return null; } + + private void handleMessage(Transaction transaction, Message message) + throws PulsarClientException, InterruptedException, ExecutionException, TimeoutException { + + // acknowledge the message as soon as possible + if (!useTransact) { + consumer.acknowledgeAsync(message.getMessageId()) + .get(consumerTimeoutInSec, TimeUnit.SECONDS); + } else { + consumer.acknowledgeAsync(message.getMessageId(), transaction) + .get(consumerTimeoutInSec, TimeUnit.SECONDS); + + // little problem: here we are counting the "commit" time + // inside the overall time spent for the execution of the consume operation + // we should refactor this operation as for PulsarProducerOp, and use the passed callback + // to track with precision the time spent for the operation and for the commit + try (Timer.Context ctx = transactionCommitTimer.time()) { + transaction.commit().get(); + } + } + + if (logger.isDebugEnabled()) { + Object decodedPayload = message.getValue(); + if (decodedPayload instanceof GenericObject) { + // GenericObject is a wrapper for Primitives, for AVRO/JSON structs and for KeyValu + // we fall here with a configured AVRO schema or with AUTO_CONSUME + GenericObject object = (GenericObject) decodedPayload; + logger.debug("({}) message received: msg-key={}; msg-properties={}; msg-payload={}", + consumer.getConsumerName(), + message.getKey(), + message.getProperties(), + object.getNativeObject() + ""); + } + else { + logger.debug("({}) message received: msg-key={}; msg-properties={}; msg-payload={}", + consumer.getConsumerName(), + message.getKey(), + message.getProperties(), + new String(message.getData())); + } + } + + if (!payloadRttField.isEmpty()) { + boolean done = false; + Object decodedPayload = message.getValue(); + Long extractedSendTime = null; + // if Pulsar is able to decode this it is better to let it do the work + // because Pulsar caches the Schema, handles Schema evolution + // as much efficiently as possible + if (decodedPayload instanceof GenericRecord) { // AVRO and AUTO_CONSUME + final GenericRecord pulsarGenericRecord = (GenericRecord) decodedPayload; + + Object field = null; + // KeyValue is a special wrapper in Pulsar to represent a pair of values + // a Key and a Value + Object nativeObject = pulsarGenericRecord.getNativeObject(); + if (nativeObject instanceof KeyValue) { + KeyValue keyValue = (KeyValue) nativeObject; + // look into the Key + if (keyValue.getKey() instanceof GenericRecord) { + GenericRecord keyPart = (GenericRecord) keyValue.getKey(); + try { + field = keyPart.getField(payloadRttField); + } catch (AvroRuntimeException err) { + // field is not in the key + logger.error("Cannot find {} in key {}: {}", payloadRttField, keyPart, err + ""); + } + } + // look into the Value + if (keyValue.getValue() instanceof GenericRecord && field == null) { + GenericRecord valuePart = (GenericRecord) keyValue.getValue(); + try { + field = valuePart.getField(payloadRttField); + } catch (AvroRuntimeException err) { + // field is not in the value + logger.error("Cannot find {} in value {}: {}", payloadRttField, valuePart, err + ""); + } + } + if (field == null) { + throw new RuntimeException("Cannot find field {}" + payloadRttField + " in " + keyValue.getKey() + " and " + keyValue.getValue()); + } + } else { + field = pulsarGenericRecord.getField(payloadRttField); + } + + if (field != null) { + if (field instanceof Number) { + extractedSendTime = ((Number) field).longValue(); + } else { + extractedSendTime = Long.valueOf(field.toString()); + } + } else { + logger.error("Cannot find {} in value {}", payloadRttField, pulsarGenericRecord); + } + done = true; + } + if (!done) { + org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration(); + org.apache.avro.generic.GenericRecord avroGenericRecord = + PulsarAvroSchemaUtil.GetGenericRecord_ApacheAvro(avroSchema, message.getData()); + if (avroGenericRecord.hasField(payloadRttField)) { + extractedSendTime = (Long) avroGenericRecord.get(payloadRttField); + } + } + if (extractedSendTime != null) { + // fallout expects latencies in "ns" and not in "ms" + long delta = TimeUnit.MILLISECONDS + .toNanos(System.currentTimeMillis() - extractedSendTime); + payloadRttHistogram.update(delta); + } + } + + // keep track end-to-end message processing latency + if (e2eStartingTimeSrc != EndToEndStartingTimeSource.NONE) { + long startTimeStamp = 0L; + + switch (e2eStartingTimeSrc) { + case MESSAGE_PUBLISH_TIME: + startTimeStamp = message.getPublishTime(); + break; + case MESSAGE_EVENT_TIME: + startTimeStamp = message.getEventTime(); + break; + case MESSAGE_PROPERTY_E2E_STARTING_TIME: + String startingTimeProperty = message.getProperty("e2e_starting_time"); + startTimeStamp = startingTimeProperty != null ? Long.parseLong(startingTimeProperty) : 0L; + break; + } + + if (startTimeStamp != 0L) { + long e2eMsgLatency = System.currentTimeMillis() - startTimeStamp; + e2eMsgProcLatencyHistogram.update(e2eMsgLatency); + } + } + + // keep track of message errors and update error counters + if (seqTracking) checkAndUpdateMessageErrorCounter(message); + + int messageSize = message.getData().length; + messageSizeHistogram.update(messageSize); + } + + private void checkAndUpdateMessageErrorCounter(Message message) { + String msgSeqIdStr = message.getProperty(PulsarAdapterUtil.MSG_SEQUENCE_NUMBER); + + if ( !StringUtils.isBlank(msgSeqIdStr) ) { + long sequenceNumber = Long.parseLong(msgSeqIdStr); + ReceivedMessageSequenceTracker receivedMessageSequenceTracker = + receivedMessageSequenceTrackerForTopic.apply(message.getTopicName()); + receivedMessageSequenceTracker.sequenceNumberReceived(sequenceNumber); + } + } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java index 664509ec8..b2995db96 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java @@ -16,17 +16,279 @@ package io.nosqlbench.adapter.pulsar.ops; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.Schema; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Timer; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; +import io.nosqlbench.adapter.pulsar.util.MessageSequenceNumberSendingHandler; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; +import io.nosqlbench.adapter.pulsar.util.PulsarAvroSchemaUtil; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.api.*; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.KeyValueSchema; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.SchemaType; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.function.Supplier; public class MessageProducerOp extends PulsarClientOp { - public MessageProducerOp(PulsarClient pulsarClient, Schema pulsarSchema) { - super(pulsarClient, pulsarSchema); + private final static Logger logger = LogManager.getLogger("MessageProducerOp"); + + private final boolean useTransact; + private final boolean seqTracking; + private final Supplier transactSupplier; + private final Set errSimuTypeSet; + private final Producer producer; + private final String msgKey; + private final String msgPropRawJsonStr; + private final String msgValue; + + private final Map msgProperties = new HashMap<>(); + private final ThreadLocal> MessageSequenceNumberSendingHandlersThreadLocal = + ThreadLocal.withInitial(HashMap::new); + + public MessageProducerOp(PulsarAdapterMetrics pulsarAdapterMetrics, + PulsarClient pulsarClient, + Schema pulsarSchema, + boolean asyncApi, + boolean useTransact, + boolean seqTracking, + Supplier transactSupplier, + Set errSimuTypeSet, + Producer producer, + String msgKey, + String msgProp, + String msgValue) { + super(pulsarAdapterMetrics, pulsarClient, pulsarSchema, asyncApi); + + this.useTransact = useTransact; + this.seqTracking = seqTracking; + this.transactSupplier = transactSupplier; + this.errSimuTypeSet = errSimuTypeSet; + this.producer = producer; + this.msgKey = msgKey; + this.msgPropRawJsonStr = msgProp; + this.msgValue = msgValue; + + getMsgPropMapFromRawJsonStr(); + getMsgPropMapFromRawJsonStr(); + } + + private MessageSequenceNumberSendingHandler getMessageSequenceNumberSendingHandler(String topicName) { + return MessageSequenceNumberSendingHandlersThreadLocal.get() + .computeIfAbsent(topicName, k -> new MessageSequenceNumberSendingHandler()); + } + + // Check if msgPropJonStr is valid JSON string with a collection of key/value pairs + // - if Yes, convert it to a map + // - otherwise, log an error message and ignore message properties without throwing a runtime exception + private void getMsgPropMapFromRawJsonStr() { + if (!StringUtils.isBlank(msgPropRawJsonStr)) { + try { + msgProperties.putAll(PulsarAdapterUtil.convertJsonToMap(msgPropRawJsonStr)); + } + catch (Exception e) { + logger.error( + "Error parsing message property JSON string {}, ignore message properties!", + msgPropRawJsonStr); + } + } + + if (seqTracking) { + long nextSequenceNumber = getMessageSequenceNumberSendingHandler(producer.getTopic()) + .getNextSequenceNumber(errSimuTypeSet); + msgProperties.put(PulsarAdapterUtil.MSG_SEQUENCE_NUMBER, String.valueOf(nextSequenceNumber)); + } } @Override public Object apply(long value) { + + TypedMessageBuilder typedMessageBuilder; + + final Transaction transaction; + if (useTransact) { + // if you are in a transaction you cannot set the schema per-message + transaction = transactSupplier.get(); + typedMessageBuilder = producer.newMessage(transaction); + } + else { + transaction = null; + typedMessageBuilder = producer.newMessage(pulsarSchema); + } + + // set message key + if (!StringUtils.isBlank(msgKey)) { + typedMessageBuilder = typedMessageBuilder.key(msgKey); + } + + // set message properties + if ( !msgPropRawJsonStr.isEmpty() ) { + typedMessageBuilder = typedMessageBuilder.properties(msgProperties); + } + + // set message payload + int messageSize; + SchemaType schemaType = pulsarSchema.getSchemaInfo().getType(); + if (pulsarSchema instanceof KeyValueSchema) { + + // {KEY IN JSON}||{VALUE IN JSON} + int separator = msgValue.indexOf("}||{"); + if (separator < 0) { + throw new IllegalArgumentException("KeyValue payload MUST be in form {KEY IN JSON}||{VALUE IN JSON} (with 2 pipes that separate the KEY part from the VALUE part)"); + } + String keyInput = msgValue.substring(0, separator + 1); + String valueInput = msgValue.substring(separator + 3); + + KeyValueSchema keyValueSchema = (KeyValueSchema) pulsarSchema; + org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration(); + GenericRecord payload = PulsarAvroSchemaUtil.GetGenericRecord_PulsarAvro( + (GenericAvroSchema) keyValueSchema.getValueSchema(), + avroSchema, + valueInput + ); + + org.apache.avro.Schema avroSchemaForKey = getKeyAvroSchemaFromConfiguration(); + GenericRecord key = PulsarAvroSchemaUtil.GetGenericRecord_PulsarAvro( + (GenericAvroSchema) keyValueSchema.getKeySchema(), + avroSchemaForKey, + keyInput + ); + typedMessageBuilder = typedMessageBuilder.value(new KeyValue(key, payload)); + // TODO: add a way to calculate the message size for KEY_VALUE messages + messageSize = msgValue.length(); + } + else if (PulsarAdapterUtil.isAvroSchemaTypeStr(schemaType.name())) { + GenericRecord payload = PulsarAvroSchemaUtil.GetGenericRecord_PulsarAvro( + (GenericAvroSchema) pulsarSchema, + pulsarSchema.getSchemaInfo().getSchemaDefinition(), + msgValue + ); + typedMessageBuilder = typedMessageBuilder.value(payload); + // TODO: add a way to calculate the message size for AVRO messages + messageSize = msgValue.length(); + } else { + byte[] array = msgValue.getBytes(StandardCharsets.UTF_8); + typedMessageBuilder = typedMessageBuilder.value(array); + messageSize = array.length; + } + + messageSizeHistogram.update(messageSize); + + //TODO: add error handling with failed message production + if (!asyncApi) { + try { + logger.trace("Sending message"); + typedMessageBuilder.send(); + + if (useTransact) { + try (Timer.Context ctx = transactionCommitTimer.time()) { + transaction.commit().get(); + } + } + + if (logger.isDebugEnabled()) { + if (PulsarAdapterUtil.isAvroSchemaTypeStr(schemaType.name())) { + org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration(); + org.apache.avro.generic.GenericRecord avroGenericRecord = + PulsarAvroSchemaUtil.GetGenericRecord_ApacheAvro(avroSchema, msgValue); + + logger.debug("({}) Sync message sent: msg-key={}; msg-properties={}; msg-payload={})", + producer.getProducerName(), + msgKey, + msgPropRawJsonStr, + avroGenericRecord.toString()); + } + else { + logger.debug("({}) Sync message sent; msg-key={}; msg-properties={}; msg-payload={}", + producer.getProducerName(), + msgKey, + msgPropRawJsonStr, + msgValue); + } + } + } + catch (PulsarClientException | ExecutionException | InterruptedException pce) { + String errMsg = + "Sync message sending failed: " + + "key - " + msgKey + "; " + + "properties - " + msgPropRawJsonStr + "; " + + "payload - " + msgValue; + + logger.trace(errMsg); + + throw new PulsarAdapterUnexpectedException(errMsg); + } + + timeTracker.run(); + } + else { + try { + // we rely on blockIfQueueIsFull in order to throttle the request in this case + CompletableFuture future = typedMessageBuilder.sendAsync(); + + if (useTransact) { + // add commit step + future = future.thenCompose(msg -> { + Timer.Context ctx = transactionCommitTimer.time(); + return transaction + .commit() + .whenComplete((m,e) -> ctx.close()) + .thenApply(v-> msg); + } + ); + } + + future.whenComplete((messageId, error) -> { + if (logger.isDebugEnabled()) { + if (PulsarAdapterUtil.isAvroSchemaTypeStr(schemaType.name())) { + org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration(); + org.apache.avro.generic.GenericRecord avroGenericRecord = + PulsarAvroSchemaUtil.GetGenericRecord_ApacheAvro(avroSchema, msgValue); + + logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={})", + producer.getProducerName(), + msgKey, + msgPropRawJsonStr, + avroGenericRecord.toString()); + } + else { + logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={}", + producer.getProducerName(), + msgKey, + msgPropRawJsonStr, + msgValue); + } + } + + timeTracker.run(); + }).exceptionally(ex -> { + logger.error("Async message sending failed: " + + "key - " + msgKey + "; " + + "properties - " + msgPropRawJsonStr + "; " + + "payload - " + msgValue); + + pulsarActivity.asyncOperationFailed(ex); + return null; + }); + } + catch (Exception e) { + throw new PulsarAdapterUnexpectedException(e); + } + } + return null; } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java index c9d47ec30..18ba4ac3e 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java @@ -16,17 +16,33 @@ package io.nosqlbench.adapter.pulsar.ops; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; public class MessageReaderOp extends PulsarClientOp { - public MessageReaderOp(PulsarClient pulsarClient, Schema pulsarSchema) { - super(pulsarClient, pulsarSchema); + private final static Logger logger = LogManager.getLogger(MessageReaderOp.class); + + private final Reader reader; + + public MessageReaderOp(PulsarAdapterMetrics pulsarAdapterMetrics, + PulsarClient pulsarClient, + Schema pulsarSchema, + boolean asyncApi, + Reader reader) { + super(pulsarAdapterMetrics, pulsarClient, pulsarSchema, asyncApi); + + this.reader = reader; } @Override public Object apply(long value) { + // TODO: implement the Pulsar reader logic when needed + // at the moment, the reader API support from the NB Pulsar driver is disabled return null; } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java index 4729a8cc5..b1be645d2 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java @@ -16,17 +16,21 @@ package io.nosqlbench.adapter.pulsar.ops; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp; import org.apache.pulsar.client.admin.PulsarAdmin; public abstract class PulsarAdminOp extends PulsarOp { protected PulsarAdmin pulsarAdmin; - protected boolean asyncApi; protected boolean adminDelOp; - public PulsarAdminOp(PulsarAdmin pulsarAdmin, boolean asyncApi, boolean adminDelOp) { + public PulsarAdminOp(PulsarAdapterMetrics pulsarAdapterMetrics, + PulsarAdmin pulsarAdmin, + boolean asyncApi, + boolean adminDelOp) { + super(pulsarAdapterMetrics, asyncApi); + this.pulsarAdmin = pulsarAdmin; - this.asyncApi = asyncApi; this.adminDelOp = adminDelOp; } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java index 957616681..15ea44b15 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java @@ -16,16 +16,74 @@ package io.nosqlbench.adapter.pulsar.ops; -import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Timer; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; +import io.nosqlbench.adapter.pulsar.util.PulsarAvroSchemaUtil; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.KeyValueSchema; +import org.apache.pulsar.common.schema.SchemaType; public abstract class PulsarClientOp extends PulsarOp { - protected PulsarClient pulsarClient; - protected Schema pulsarScheam; + protected final PulsarClient pulsarClient; + protected final Schema pulsarSchema; + + // Pulsar KeyValue schema + private org.apache.avro.Schema avroSchema; + private org.apache.avro.Schema avroKeySchema; + + protected final Histogram messageSizeHistogram; + protected final Histogram payloadRttHistogram; + protected final Histogram e2eMsgProcLatencyHistogram; + + protected final Timer transactionCommitTimer; + + public PulsarClientOp(PulsarAdapterMetrics pulsarAdapterMetrics, + PulsarClient pulsarClient, + Schema pulsarScheam, + boolean asyncApi) { + super (pulsarAdapterMetrics, asyncApi); - public PulsarClientOp(PulsarClient pulsarClient, Schema pulsarScheam) { this.pulsarClient = pulsarClient; - this.pulsarScheam = pulsarScheam; + this.pulsarSchema = pulsarScheam; + + this.messageSizeHistogram = pulsarAdapterMetrics.getMessageSizeHistogram(); + this.payloadRttHistogram = pulsarAdapterMetrics.getPayloadRttHistogram(); + this.e2eMsgProcLatencyHistogram = pulsarAdapterMetrics.getE2eMsgProcLatencyHistogram(); + this.transactionCommitTimer = pulsarAdapterMetrics.getCommitTransactionTimer(); + } + + protected org.apache.avro.Schema getAvroSchemaFromConfiguration() { + // no need for synchronization, this is only a cache + // in case of the race we will parse the string twice, not a big + if (avroSchema == null) { + if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { + KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema; + Schema valueSchema = kvSchema.getValueSchema(); + String avroDefStr = valueSchema.getSchemaInfo().getSchemaDefinition(); + avroSchema = PulsarAvroSchemaUtil.GetSchema_ApacheAvro(avroDefStr); + } else { + String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition(); + avroSchema = PulsarAvroSchemaUtil.GetSchema_ApacheAvro(avroDefStr); + } + } + return avroSchema; + } + + protected org.apache.avro.Schema getKeyAvroSchemaFromConfiguration() { + // no need for synchronization, this is only a cache + // in case of the race we will parse the string twice, not a big + if (avroKeySchema == null) { + if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { + KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema; + Schema keySchema = kvSchema.getKeySchema(); + String avroDefStr = keySchema.getSchemaInfo().getSchemaDefinition(); + avroKeySchema = PulsarAvroSchemaUtil.GetSchema_ApacheAvro(avroDefStr); + } else { + throw new RuntimeException("We are not using KEY_VALUE schema, so no Schema for the Key!"); + } + } + return avroKeySchema; } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java index 0c68c52d7..a3e0b87b1 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java @@ -17,7 +17,15 @@ package io.nosqlbench.adapter.pulsar.ops; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp; public abstract class PulsarOp implements CycleOp { + protected final boolean asyncApi; + protected final PulsarAdapterMetrics pulsarAdapterMetrics; + + public PulsarOp(PulsarAdapterMetrics pulsarAdapterMetrics, boolean asyncApi) { + this.pulsarAdapterMetrics = pulsarAdapterMetrics; + this.asyncApi = asyncApi; + } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/EndToEndStartingTimeSource.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/EndToEndStartingTimeSource.java new file mode 100644 index 000000000..7a9f39680 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/EndToEndStartingTimeSource.java @@ -0,0 +1,26 @@ +package io.nosqlbench.adapter.pulsar.util; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +public enum EndToEndStartingTimeSource { + NONE, // no end-to-end latency calculation + MESSAGE_PUBLISH_TIME, // use message publish timestamp + MESSAGE_EVENT_TIME, // use message event timestamp + MESSAGE_PROPERTY_E2E_STARTING_TIME // use message property called "e2e_starting_time" as the timestamp +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/MessageSequenceNumberSendingHandler.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/MessageSequenceNumberSendingHandler.java new file mode 100644 index 000000000..9ff2aad5f --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/MessageSequenceNumberSendingHandler.java @@ -0,0 +1,109 @@ +package io.nosqlbench.adapter.pulsar.util; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; +import org.apache.commons.lang3.RandomUtils; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Queue; +import java.util.Set; + +/** + * Handles adding a monotonic sequence number to message properties of sent messages + */ +public class MessageSequenceNumberSendingHandler { + static final int SIMULATED_ERROR_PROBABILITY_PERCENTAGE = 10; + long number = 1; + Queue outOfOrderNumbers; + + public long getNextSequenceNumber(Set simulatedErrorTypes) { + return getNextSequenceNumber(simulatedErrorTypes, SIMULATED_ERROR_PROBABILITY_PERCENTAGE); + } + + long getNextSequenceNumber(Set simulatedErrorTypes, int errorProbabilityPercentage) { + simulateError(simulatedErrorTypes, errorProbabilityPercentage); + return nextNumber(); + } + + private void simulateError(Set simulatedErrorTypes, int errorProbabilityPercentage) { + if (!simulatedErrorTypes.isEmpty() && shouldSimulateError(errorProbabilityPercentage)) { + int selectIndex = 0; + int numberOfErrorTypes = simulatedErrorTypes.size(); + if (numberOfErrorTypes > 1) { + // pick one of the simulated error type randomly + selectIndex = RandomUtils.nextInt(0, numberOfErrorTypes); + } + PulsarAdapterUtil.SEQ_ERROR_SIMU_TYPE errorType = simulatedErrorTypes.stream() + .skip(selectIndex) + .findFirst() + .get(); + switch (errorType) { + case OutOfOrder: + // simulate message out of order + injectMessagesOutOfOrder(); + break; + case MsgDup: + // simulate message duplication + injectMessageDuplication(); + break; + case MsgLoss: + // simulate message loss + injectMessageLoss(); + break; + } + } + } + + private boolean shouldSimulateError(int errorProbabilityPercentage) { + // Simulate error with the specified probability + return RandomUtils.nextInt(0, 100) < errorProbabilityPercentage; + } + + long nextNumber() { + if (outOfOrderNumbers != null) { + long nextNumber = outOfOrderNumbers.poll(); + if (outOfOrderNumbers.isEmpty()) { + outOfOrderNumbers = null; + } + return nextNumber; + } + return number++; + } + + void injectMessagesOutOfOrder() { + if (outOfOrderNumbers == null) { + outOfOrderNumbers = new ArrayDeque<>(Arrays.asList(number + 2, number, number + 1)); + number += 3; + } + } + + void injectMessageDuplication() { + if (outOfOrderNumbers == null) { + number--; + } + } + + void injectMessageLoss() { + if (outOfOrderNumbers == null) { + number++; + } + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java index f67bda339..7243d4c19 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java @@ -21,6 +21,8 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Histogram; import com.codahale.metrics.Timer; import io.nosqlbench.adapter.pulsar.PulsarSpace; +import io.nosqlbench.adapter.pulsar.dispensers.PulsarBaseOpDispenser; +import io.nosqlbench.api.config.NBNamedElement; import io.nosqlbench.api.engine.metrics.ActivityMetrics; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -36,69 +38,104 @@ public class PulsarAdapterMetrics { private final static Logger logger = LogManager.getLogger("PulsarAdapterMetrics"); - private final PulsarSpace pulsarSpace; + private final PulsarBaseOpDispenser pulsarBaseOpDispenser; private final String defaultAdapterMetricsPrefix; /** * Pulsar adapter specific metrics */ - protected Counter bytesCounter; // - message out of sequence error counter - protected Counter msgErrOutOfSeqCounter; + private Counter msgErrOutOfSeqCounter; // - message loss counter - protected Counter msgErrLossCounter; + private Counter msgErrLossCounter; // - message duplicate (when dedup is enabled) error counter - protected Counter msgErrDuplicateCounter; + private Counter msgErrDuplicateCounter; - protected Histogram messageSizeHistogram; + private Histogram messageSizeHistogram; // end-to-end latency - protected Histogram e2eMsgProcLatencyHistogram; + private Histogram e2eMsgProcLatencyHistogram; // A histogram that tracks payload round-trip-time, based on a user-defined field in some sender // system which can be interpreted as millisecond epoch time in the system's local time zone. // This is paired with a field name of the same type to be extracted and reported in a metric // named 'payload-rtt'. - protected Histogram payloadRttHistogram; + private Histogram payloadRttHistogram; - protected Timer bindTimer; - protected Timer executeTimer; - protected Timer createTransactionTimer; - protected Timer commitTransactionTimer; + private Timer bindTimer; + private Timer executeTimer; + private Timer createTransactionTimer; + private Timer commitTransactionTimer; - public PulsarAdapterMetrics(PulsarSpace pulsarSpace, String defaultMetricsPrefix) { - this.pulsarSpace = pulsarSpace; + public PulsarAdapterMetrics(PulsarBaseOpDispenser pulsarBaseOpDispenser, String defaultMetricsPrefix) { + this.pulsarBaseOpDispenser = pulsarBaseOpDispenser; this.defaultAdapterMetricsPrefix = defaultMetricsPrefix; } public void initPulsarAdapterInstrumentation() { // Counter metrics - this.bytesCounter = - ActivityMetrics.counter(this.defaultAdapterMetricsPrefix + "bytes"); this.msgErrOutOfSeqCounter = - ActivityMetrics.counter(this.defaultAdapterMetricsPrefix + "err_msg_oos"); + ActivityMetrics.counter( + pulsarBaseOpDispenser, + defaultAdapterMetricsPrefix + "err_msg_oos"); this.msgErrLossCounter = - ActivityMetrics.counter(this.defaultAdapterMetricsPrefix + "err_msg_loss"); + ActivityMetrics.counter( + pulsarBaseOpDispenser, + defaultAdapterMetricsPrefix + "err_msg_loss"); this.msgErrDuplicateCounter = - ActivityMetrics.counter(this.defaultAdapterMetricsPrefix + "err_msg_dup"); + ActivityMetrics.counter( + pulsarBaseOpDispenser, + defaultAdapterMetricsPrefix + "err_msg_dup"); // Histogram metrics this.messageSizeHistogram = - ActivityMetrics.histogram(this.defaultAdapterMetricsPrefix + "message_size"); + ActivityMetrics.histogram( + pulsarBaseOpDispenser, + defaultAdapterMetricsPrefix + "message_size", + ActivityMetrics.DEFAULT_HDRDIGITS); this.e2eMsgProcLatencyHistogram = - ActivityMetrics.histogram(this.defaultAdapterMetricsPrefix + "e2e_msg_latency"); + ActivityMetrics.histogram( + pulsarBaseOpDispenser, + defaultAdapterMetricsPrefix + "e2e_msg_latency", + ActivityMetrics.DEFAULT_HDRDIGITS); this.payloadRttHistogram = - ActivityMetrics.histogram(this.defaultAdapterMetricsPrefix + "payload_rtt"); + ActivityMetrics.histogram( + pulsarBaseOpDispenser, + defaultAdapterMetricsPrefix + "payload_rtt", + ActivityMetrics.DEFAULT_HDRDIGITS); // Timer metrics this.bindTimer = - ActivityMetrics.timer(this.defaultAdapterMetricsPrefix + "bind"); + ActivityMetrics.timer( + pulsarBaseOpDispenser, + defaultAdapterMetricsPrefix + "bind", + ActivityMetrics.DEFAULT_HDRDIGITS); this.executeTimer = - ActivityMetrics.timer(this.defaultAdapterMetricsPrefix + "execute"); + ActivityMetrics.timer( + pulsarBaseOpDispenser, + defaultAdapterMetricsPrefix + "execute", + ActivityMetrics.DEFAULT_HDRDIGITS); this.createTransactionTimer = - ActivityMetrics.timer(this.defaultAdapterMetricsPrefix + "create_transaction"); + ActivityMetrics.timer( + pulsarBaseOpDispenser, + defaultAdapterMetricsPrefix + "create_transaction", + ActivityMetrics.DEFAULT_HDRDIGITS); this.commitTransactionTimer = - ActivityMetrics.timer(this.defaultAdapterMetricsPrefix + "commit_transaction"); + ActivityMetrics.timer( + pulsarBaseOpDispenser, + defaultAdapterMetricsPrefix + "commit_transaction", + ActivityMetrics.DEFAULT_HDRDIGITS); } + public Counter getMsgErrOutOfSeqCounter() { return this.msgErrOutOfSeqCounter; } + public Counter getMsgErrLossCounter() { return this.msgErrLossCounter; } + public Counter getMsgErrDuplicateCounter() { return this.msgErrDuplicateCounter; } + public Histogram getMessageSizeHistogram() { return this.messageSizeHistogram; } + public Histogram getE2eMsgProcLatencyHistogram() { return this.e2eMsgProcLatencyHistogram; } + public Histogram getPayloadRttHistogram() { return payloadRttHistogram; } + public Timer getBindTimer() { return bindTimer; } + public Timer getExecuteTimer() { return executeTimer; } + public Timer getCreateTransactionTimer() { return createTransactionTimer; } + public Timer getCommitTransactionTimer() { return commitTransactionTimer; } + ////////////////////////////////////// // Pulsar client producer API metrics @@ -132,17 +169,17 @@ public class PulsarAdapterMetrics { metricsPrefix = pulsarApiMetricsPrefix; } - ActivityMetrics.gauge(metricsPrefix + "total_bytes_sent", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "total_bytes_sent", producerSafeExtractMetric(producer, (s -> s.getTotalBytesSent() + s.getNumBytesSent()))); - ActivityMetrics.gauge(metricsPrefix + "total_msg_sent", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "total_msg_sent", producerSafeExtractMetric(producer, (s -> s.getTotalMsgsSent() + s.getNumMsgsSent()))); - ActivityMetrics.gauge(metricsPrefix + "total_send_failed", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "total_send_failed", producerSafeExtractMetric(producer, (s -> s.getTotalSendFailed() + s.getNumSendFailed()))); - ActivityMetrics.gauge(metricsPrefix + "total_ack_received", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "total_ack_received", producerSafeExtractMetric(producer,(s -> s.getTotalAcksReceived() + s.getNumAcksReceived()))); - ActivityMetrics.gauge(metricsPrefix + "send_bytes_rate", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "send_bytes_rate", producerSafeExtractMetric(producer, ProducerStats::getSendBytesRate)); - ActivityMetrics.gauge(metricsPrefix + "send_msg_rate", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "send_msg_rate", producerSafeExtractMetric(producer, ProducerStats::getSendMsgsRate)); } @@ -180,17 +217,17 @@ public class PulsarAdapterMetrics { metricsPrefix = pulsarApiMetricsPrefix; } - ActivityMetrics.gauge(metricsPrefix + "total_bytes_recv", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "total_bytes_recv", consumerSafeExtractMetric(consumer, (s -> s.getTotalBytesReceived() + s.getNumBytesReceived()))); - ActivityMetrics.gauge(metricsPrefix + "total_msg_recv", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "total_msg_recv", consumerSafeExtractMetric(consumer, (s -> s.getTotalMsgsReceived() + s.getNumMsgsReceived()))); - ActivityMetrics.gauge(metricsPrefix + "total_recv_failed", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "total_recv_failed", consumerSafeExtractMetric(consumer, (s -> s.getTotalReceivedFailed() + s.getNumReceiveFailed()))); - ActivityMetrics.gauge(metricsPrefix + "total_acks_sent", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "total_acks_sent", consumerSafeExtractMetric(consumer,(s -> s.getTotalAcksSent() + s.getNumAcksSent()))); - ActivityMetrics.gauge(metricsPrefix + "recv_bytes_rate", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "recv_bytes_rate", consumerSafeExtractMetric(consumer, ConsumerStats::getRateBytesReceived)); - ActivityMetrics.gauge(metricsPrefix + "recv_msg_rate", + ActivityMetrics.gauge(pulsarBaseOpDispenser, metricsPrefix + "recv_msg_rate", consumerSafeExtractMetric(consumer, ConsumerStats::getRateMsgsReceived)); } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java index 66d346905..130ce032f 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java @@ -470,7 +470,7 @@ public class PulsarAdapterUtil { } } - schema = AvroUtil.GetSchema_PulsarAvro("NBAvro", schemaDefinitionStr); + schema = PulsarAvroSchemaUtil.GetSchema_PulsarAvro("NBAvro", schemaDefinitionStr); } else { throw new RuntimeException("Trying to create a \"Avro\" schema for a non-Avro schema type string: " + typeStr); } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/AvroUtil.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAvroSchemaUtil.java similarity index 99% rename from adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/AvroUtil.java rename to adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAvroSchemaUtil.java index 882f060f9..a60b1f0ff 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/AvroUtil.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAvroSchemaUtil.java @@ -31,7 +31,7 @@ import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; -public class AvroUtil { +public class PulsarAvroSchemaUtil { //////////////////////// // Get an OSS Apache Avro schema from a string definition public static org.apache.avro.Schema GetSchema_ApacheAvro(String avroSchemDef) { diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/ReceivedMessageSequenceTracker.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/ReceivedMessageSequenceTracker.java new file mode 100644 index 000000000..f929ab25a --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/ReceivedMessageSequenceTracker.java @@ -0,0 +1,169 @@ +package io.nosqlbench.adapter.pulsar.util; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import com.codahale.metrics.Counter; + +import java.util.Iterator; +import java.util.SortedSet; +import java.util.TreeSet; + +/** + * Detects message loss, message duplication and out-of-order message delivery + * based on a monotonic sequence number that each received message contains. + *

+ * Out-of-order messages are detected with a maximum look behind of 1000 sequence number entries. + * This is currently defined as a constant, {@link ReceivedMessageSequenceTracker#DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS}. + */ +public class ReceivedMessageSequenceTracker implements AutoCloseable { + private static final int DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS = 1000; + private static final int DEFAULT_MAX_TRACK_SKIPPED_SEQUENCE_NUMBERS = 1000; + // message out-of-sequence error counter + private final Counter msgErrOutOfSeqCounter; + // duplicate message error counter + private final Counter msgErrDuplicateCounter; + // message loss error counter + private final Counter msgErrLossCounter; + private final SortedSet pendingOutOfSeqNumbers; + private final int maxTrackOutOfOrderSequenceNumbers; + private final SortedSet skippedSeqNumbers; + private final int maxTrackSkippedSequenceNumbers; + private long expectedNumber = -1; + + public ReceivedMessageSequenceTracker(Counter msgErrOutOfSeqCounter, Counter msgErrDuplicateCounter, Counter msgErrLossCounter) { + this(msgErrOutOfSeqCounter, msgErrDuplicateCounter, msgErrLossCounter, + DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS, DEFAULT_MAX_TRACK_SKIPPED_SEQUENCE_NUMBERS); + } + + public ReceivedMessageSequenceTracker(Counter msgErrOutOfSeqCounter, Counter msgErrDuplicateCounter, Counter msgErrLossCounter, + int maxTrackOutOfOrderSequenceNumbers, int maxTrackSkippedSequenceNumbers) { + this.msgErrOutOfSeqCounter = msgErrOutOfSeqCounter; + this.msgErrDuplicateCounter = msgErrDuplicateCounter; + this.msgErrLossCounter = msgErrLossCounter; + this.maxTrackOutOfOrderSequenceNumbers = maxTrackOutOfOrderSequenceNumbers; + this.maxTrackSkippedSequenceNumbers = maxTrackSkippedSequenceNumbers; + this.pendingOutOfSeqNumbers = new TreeSet<>(); + this.skippedSeqNumbers = new TreeSet<>(); + } + + /** + * Notifies the tracker about a received sequence number + * + * @param sequenceNumber the sequence number of the received message + */ + public void sequenceNumberReceived(long sequenceNumber) { + if (expectedNumber == -1) { + expectedNumber = sequenceNumber + 1; + return; + } + + if (sequenceNumber < expectedNumber) { + if (skippedSeqNumbers.remove(sequenceNumber)) { + // late out-of-order delivery was detected + // decrease the loss counter + msgErrLossCounter.dec(); + // increment the out-of-order counter + msgErrOutOfSeqCounter.inc(); + } else { + msgErrDuplicateCounter.inc(); + } + return; + } + + boolean messagesSkipped = false; + if (sequenceNumber > expectedNumber) { + if (pendingOutOfSeqNumbers.size() == maxTrackOutOfOrderSequenceNumbers) { + messagesSkipped = processLowestPendingOutOfSequenceNumber(); + } + if (!pendingOutOfSeqNumbers.add(sequenceNumber)) { + msgErrDuplicateCounter.inc(); + } + } else { + // sequenceNumber == expectedNumber + expectedNumber++; + } + processPendingOutOfSequenceNumbers(messagesSkipped); + cleanUpTooFarBehindOutOfSequenceNumbers(); + } + + private boolean processLowestPendingOutOfSequenceNumber() { + // remove the lowest pending out of sequence number + Long lowestOutOfSeqNumber = pendingOutOfSeqNumbers.first(); + pendingOutOfSeqNumbers.remove(lowestOutOfSeqNumber); + if (lowestOutOfSeqNumber > expectedNumber) { + // skip the expected number ahead to the number after the lowest sequence number + // increment the counter with the amount of sequence numbers that got skipped + // keep track of the skipped sequence numbers to detect late out-of-order message delivery + for (long l = expectedNumber; l < lowestOutOfSeqNumber; l++) { + msgErrLossCounter.inc(); + skippedSeqNumbers.add(l); + if (skippedSeqNumbers.size() > maxTrackSkippedSequenceNumbers) { + skippedSeqNumbers.remove(skippedSeqNumbers.first()); + } + } + expectedNumber = lowestOutOfSeqNumber + 1; + return true; + } else { + msgErrLossCounter.inc(); + } + return false; + } + + private void processPendingOutOfSequenceNumbers(boolean messagesSkipped) { + // check if there are previously received out-of-order sequence number that have been received + while (pendingOutOfSeqNumbers.remove(expectedNumber)) { + expectedNumber++; + if (!messagesSkipped) { + msgErrOutOfSeqCounter.inc(); + } + } + } + + private void cleanUpTooFarBehindOutOfSequenceNumbers() { + // remove sequence numbers that are too far behind + for (Iterator iterator = pendingOutOfSeqNumbers.iterator(); iterator.hasNext(); ) { + Long number = iterator.next(); + if (number < expectedNumber - maxTrackOutOfOrderSequenceNumbers) { + msgErrLossCounter.inc(); + iterator.remove(); + } else { + break; + } + } + } + + /** + * Handles the possible pending out of sequence numbers. Mainly needed in unit tests to assert the + * counter values. + */ + @Override + public void close() { + while (!pendingOutOfSeqNumbers.isEmpty()) { + processPendingOutOfSequenceNumbers(processLowestPendingOutOfSequenceNumber()); + } + } + + public int getMaxTrackOutOfOrderSequenceNumbers() { + return maxTrackOutOfOrderSequenceNumbers; + } + + public int getMaxTrackSkippedSequenceNumbers() { + return maxTrackSkippedSequenceNumbers; + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java b/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java index 188d93dd1..bff438d72 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java +++ b/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java @@ -194,11 +194,6 @@ public class ActivityMetrics { return (Counter) register(named, name, Counter::new); } - public static Counter counter(String fullName) { - Counter counter = get().register(fullName, new Counter()); - return counter; - } - /** *

Create a meter associated with an activity.

*

This method ensures that if multiple threads attempt to create the same-named metric on a given activity, @@ -229,10 +224,6 @@ public class ActivityMetrics { return (Gauge) register(named, name, () -> gauge); } - public static Gauge gauge(String fullMetricsName, Gauge gauge) { - return (Gauge) register(fullMetricsName, () -> gauge); - } - @SuppressWarnings("unchecked") public static Gauge gauge(ScriptContext scriptContext, String name, Gauge gauge) { return (Gauge) register(scriptContext, name, () -> gauge); From 12f8697e0e27754074eca413451c0707d0663d70 Mon Sep 17 00:00:00 2001 From: yabinmeng Date: Thu, 17 Nov 2022 10:39:37 -0600 Subject: [PATCH 12/14] Complete the first draft of NB5 Pulsar adapter code, except the per-thread rate limiter. --- .../adapter/pulsar/PulsarSpace.java | 52 ++++++++++++++++--- .../dispensers/PulsarBaseOpDispenser.java | 37 ++++--------- ...rAdapterAsyncOperationFailedException.java | 26 ++++++++++ .../adapter/pulsar/ops/MessageConsumerOp.java | 8 +-- .../adapter/pulsar/ops/MessageProducerOp.java | 8 +-- .../pulsar/util/PulsarAdapterMetrics.java | 2 - 6 files changed, 88 insertions(+), 45 deletions(-) create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterAsyncOperationFailedException.java diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java index 93a92c70b..e2e5bf07f 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java @@ -16,7 +16,6 @@ package io.nosqlbench.adapter.pulsar; -import com.codahale.metrics.Gauge; import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; import io.nosqlbench.adapter.pulsar.util.PulsarNBClientConf; @@ -24,7 +23,6 @@ import io.nosqlbench.api.config.standard.ConfigModel; import io.nosqlbench.api.config.standard.NBConfigModel; import io.nosqlbench.api.config.standard.NBConfiguration; import io.nosqlbench.api.config.standard.Param; -import io.nosqlbench.api.engine.metrics.ActivityMetrics; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -35,12 +33,13 @@ import org.apache.pulsar.client.api.*; import org.apache.pulsar.common.schema.KeyValueEncodingType; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; -public class PulsarSpace { +public class PulsarSpace implements AutoCloseable { private final static Logger logger = LogManager.getLogger(PulsarSpace.class); - private final String name; + private final String spaceName; private final NBConfiguration cfg; private final String pulsarSvcUrl; @@ -51,8 +50,13 @@ public class PulsarSpace { private PulsarAdmin pulsarAdmin; private Schema pulsarSchema; - public PulsarSpace(String name, NBConfiguration cfg) { - this.name = name; + private final ConcurrentHashMap> producers = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> consumers = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> readers = new ConcurrentHashMap<>(); + + + public PulsarSpace(String spaceName, NBConfiguration cfg) { + this.spaceName = spaceName; this.cfg = cfg; this.pulsarSvcUrl = cfg.get("service_url"); @@ -82,6 +86,17 @@ public class PulsarSpace { public PulsarClient getPulsarClient() { return pulsarClient; } public PulsarAdmin getPulsarAdmin() { return pulsarAdmin; } public Schema getPulsarSchema() { return pulsarSchema; } + public int getProducerSetCnt() { return producers.size(); } + public int getConsumerSetCnt() { return consumers.size(); } + public int getReaderSetCnt() { return readers.size(); } + public Producer getProducer(String name) { return producers.get(name); } + public void setProducer(String name, Producer producer) { producers.put(name, producer); } + public Consumer getConsumer(String name) { return consumers.get(name); } + public void setConsumer(String name, Consumer consumer) { consumers.put(name, consumer); } + + public Reader getReader(String name) { return readers.get(name); } + public void setReader(String name, Reader reader) { readers.put(name, reader); } + /** * Initialize @@ -147,6 +162,26 @@ public class PulsarSpace { } } + public void shutdownSpace() { + try { + for (Producer producer : producers.values()) { + if (producer != null) producer.close(); + } + for (Consumer consumer : consumers.values()) { + if (consumer != null) consumer.close(); + } + for (Reader reader : readers.values()) { + if (reader != null) reader.close(); + } + if (pulsarAdmin != null) pulsarAdmin.close(); + if (pulsarClient != null) pulsarClient.close(); + } + catch (Exception e) { + throw new PulsarAdapterUnexpectedException( + "Unexpected error when shutting down the Pulsar space \"" + spaceName + "\"!"); + } + } + /** * Get Pulsar schema from the definition string */ @@ -185,6 +220,11 @@ public class PulsarSpace { pulsarSchema = Schema.KeyValue(pulsarKeySchema, pulsarSchema, keyValueEncodingType); } } + + @Override + public void close() { + shutdownSpace(); + } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java index 844c8c57e..74c9e0ce2 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java @@ -23,9 +23,6 @@ import io.nosqlbench.adapter.pulsar.ops.PulsarOp; import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; import io.nosqlbench.api.config.NBNamedElement; -import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter; -import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters; -import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec; import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; @@ -37,7 +34,6 @@ import org.apache.logging.log4j.Logger; import org.apache.pulsar.client.api.*; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.LongFunction; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -51,10 +47,6 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser> producers = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> consumers = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> readers = new ConcurrentHashMap<>(); - protected final LongFunction asyncApiFunc; protected final LongFunction tgtNameFunc; @@ -62,8 +54,6 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser tgtNameFunc, @@ -81,17 +71,10 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser per_thread_cyclelimiter = - RateLimiters.createOrUpdate(this, "cycles", per_thread_cyclelimiter, spec)); } @Override @@ -187,11 +170,11 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser producer = producers.get(producerCacheKey); + Producer producer = pulsarSpace.getProducer(producerCacheKey); if (producer == null) { PulsarClient pulsarClient = pulsarSpace.getPulsarClient(); @@ -280,7 +263,7 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser consumer = consumers.get(consumerCacheKey); + Consumer consumer = pulsarSpace.getConsumer(consumerCacheKey); if (consumer == null) { PulsarClient pulsarClient = pulsarSpace.getPulsarClient(); @@ -525,7 +508,7 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser reader = readers.get(readerCacheKey); + Reader reader = pulsarSpace.getReader(readerCacheKey); if (reader == null) { PulsarClient pulsarClient = pulsarSpace.getPulsarClient();; @@ -676,7 +659,7 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser { - pulsarActivity.asyncOperationFailed(ex); - return null; + throw new PulsarAdapterAsyncOperationFailedException(ex); }); } catch (Exception e) { diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java index b2995db96..965b323b0 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java @@ -18,6 +18,7 @@ package io.nosqlbench.adapter.pulsar.ops; import com.codahale.metrics.Histogram; import com.codahale.metrics.Timer; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterAsyncOperationFailedException; import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; import io.nosqlbench.adapter.pulsar.util.MessageSequenceNumberSendingHandler; import io.nosqlbench.adapter.pulsar.util.PulsarAdapterMetrics; @@ -231,8 +232,6 @@ public class MessageProducerOp extends PulsarClientOp { throw new PulsarAdapterUnexpectedException(errMsg); } - - timeTracker.run(); } else { try { @@ -272,16 +271,13 @@ public class MessageProducerOp extends PulsarClientOp { msgValue); } } - - timeTracker.run(); }).exceptionally(ex -> { logger.error("Async message sending failed: " + "key - " + msgKey + "; " + "properties - " + msgPropRawJsonStr + "; " + "payload - " + msgValue); - pulsarActivity.asyncOperationFailed(ex); - return null; + throw new PulsarAdapterAsyncOperationFailedException(ex); }); } catch (Exception e) { diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java index 7243d4c19..ae48803f4 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterMetrics.java @@ -20,9 +20,7 @@ import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; import com.codahale.metrics.Histogram; import com.codahale.metrics.Timer; -import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.dispensers.PulsarBaseOpDispenser; -import io.nosqlbench.api.config.NBNamedElement; import io.nosqlbench.api.engine.metrics.ActivityMetrics; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; From 0de71028847f6cda80d248938e1710b39168f961 Mon Sep 17 00:00:00 2001 From: yabinmeng Date: Thu, 17 Nov 2022 18:47:37 -0600 Subject: [PATCH 13/14] Add support for 1) producer message compression 2) consumer DLT/negAck/ackTimeout policy Add sample NB scenario yaml files --- .../adapter/pulsar/PulsarOpMapper.java | 16 +- .../adapter/pulsar/PulsarOpType.java | 35 +- .../adapter/pulsar/PulsarSpace.java | 34 +- .../dispensers/PulsarBaseOpDispenser.java | 18 +- .../adapter/pulsar/ops/MessageProducerOp.java | 37 +- .../pulsar/util/PulsarAdapterUtil.java | 20 +- ...BClientConf.java => PulsarClientConf.java} | 229 ++++++----- .../pulsar/util/PulsarConfConverter.java | 367 ++++++++++++++++++ ...in-namespace.yaml => admin_namespace.yaml} | 2 +- .../{admin-tenant.yaml => admin_tenant.yaml} | 4 +- .../{admin-topic.yaml => admin_topic.yaml} | 4 +- .../src/main/resources/config.properties | 8 +- .../src/main/resources/iot-key-example.avsc | 9 + .../src/main/resources/iot-value-example.avsc | 11 + .../src/main/resources/msg_proc_avro.yaml | 41 ++ .../src/main/resources/msg_proc_kvraw.yaml | 34 ++ adapter-pulsar/src/main/resources/pulsar.md | 1 + 17 files changed, 643 insertions(+), 227 deletions(-) rename adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/{PulsarNBClientConf.java => PulsarClientConf.java} (54%) create mode 100644 adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarConfConverter.java rename adapter-pulsar/src/main/resources/{admin-namespace.yaml => admin_namespace.yaml} (93%) rename adapter-pulsar/src/main/resources/{admin-tenant.yaml => admin_tenant.yaml} (81%) rename adapter-pulsar/src/main/resources/{admin-topic.yaml => admin_topic.yaml} (88%) create mode 100644 adapter-pulsar/src/main/resources/iot-key-example.avsc create mode 100644 adapter-pulsar/src/main/resources/iot-value-example.avsc create mode 100644 adapter-pulsar/src/main/resources/msg_proc_avro.yaml create mode 100644 adapter-pulsar/src/main/resources/msg_proc_kvraw.yaml create mode 100644 adapter-pulsar/src/main/resources/pulsar.md diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java index a2b33b9a7..a3db586ee 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java @@ -17,7 +17,6 @@ package io.nosqlbench.adapter.pulsar; import io.nosqlbench.adapter.pulsar.dispensers.*; -import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnsupportedOpException; import io.nosqlbench.adapter.pulsar.ops.PulsarOp; import io.nosqlbench.api.config.standard.NBConfiguration; import io.nosqlbench.engine.api.activityimpl.OpDispenser; @@ -28,9 +27,6 @@ import io.nosqlbench.engine.api.templating.ParsedOp; import io.nosqlbench.engine.api.templating.TypeAndTarget; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.Schema; public class PulsarOpMapper implements OpMapper { @@ -51,10 +47,6 @@ public class PulsarOpMapper implements OpMapper { String spaceName = op.getStaticConfigOr("space", "default"); PulsarSpace pulsarSpace = spaceCache.get(spaceName); - PulsarClient pulsarClient = pulsarSpace.getPulsarClient(); - PulsarAdmin pulsarAdmin = pulsarSpace.getPulsarAdmin(); - Schema pulsarSchema = pulsarSpace.getPulsarSchema(); - /* * If the user provides a body element, then they want to provide the JSON or * a data structure that can be converted into JSON, bypassing any further @@ -66,12 +58,6 @@ public class PulsarOpMapper implements OpMapper { else { TypeAndTarget opType = op.getTypeAndTarget(PulsarOpType.class, String.class); - if (PulsarOpType.isValidPulsarOpType(opType.enumId.label)) { - throw new PulsarAdapterUnsupportedOpException( - "Unrecognized Pulsar Adapter Op Type -- must be one of the following values: \"" + - PulsarOpType.getValidPulsarOpTypeList() + "\"!"); - } - return switch (opType.enumId) { case AdminTenant -> new AdminTenantOpDispenser(adapter, op, opType.targetFunction, pulsarSpace); @@ -87,7 +73,7 @@ public class PulsarOpMapper implements OpMapper { // NOTE: not sure how useful to have Pulsar message reader API in the NB performance testing // currently, the reader API in NB Pulsar driver is no-op (see TDOD in MessageReaderOp) ////////////////////////// - case MessageRead -> + case MessageRead -> new MessageReaderOpDispenser(adapter, op, opType.targetFunction, pulsarSpace); }; } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java index f6f9be451..d63185121 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java @@ -16,37 +16,14 @@ package io.nosqlbench.adapter.pulsar; -import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; - -import java.util.Arrays; -import java.util.stream.Collectors; - public enum PulsarOpType { - AdminTenant("admin-tenant"), - AdminNamespace("admin-namespace"), - AdminTopic("admin-topic"), - MessageProduce("msg-send"), + AdminTenant, + AdminNamespace, + AdminTopic, + MessageProduce, // This also supports multi-topic message consumption - MessageConsume("msg-consume"), - MessageRead("msg-read"); - - public final String label; - - PulsarOpType(String label) { - this.label = label; - } - - - public static boolean isValidPulsarOpType(String type) { - return Arrays.stream(values()) - .anyMatch(t -> t.label.equalsIgnoreCase(type)); - } - - public static String getValidPulsarOpTypeList() { - return Arrays.stream(values()) - .map(t -> t.label) - .collect(Collectors.joining(", ")); - } + MessageConsume, + MessageRead; } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java index e2e5bf07f..708e26a96 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java @@ -18,7 +18,7 @@ package io.nosqlbench.adapter.pulsar; import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; -import io.nosqlbench.adapter.pulsar.util.PulsarNBClientConf; +import io.nosqlbench.adapter.pulsar.util.PulsarClientConf; import io.nosqlbench.api.config.standard.ConfigModel; import io.nosqlbench.api.config.standard.NBConfigModel; import io.nosqlbench.api.config.standard.NBConfiguration; @@ -45,7 +45,7 @@ public class PulsarSpace implements AutoCloseable { private final String pulsarSvcUrl; private final String webSvcUrl; - private PulsarNBClientConf pulsarNBClientConf; + private PulsarClientConf pulsarClientConf; private PulsarClient pulsarClient; private PulsarAdmin pulsarAdmin; private Schema pulsarSchema; @@ -61,7 +61,7 @@ public class PulsarSpace implements AutoCloseable { this.pulsarSvcUrl = cfg.get("service_url"); this.webSvcUrl = cfg.get("web_url"); - this.pulsarNBClientConf = new PulsarNBClientConf(cfg.get("config")); + this.pulsarClientConf = new PulsarClientConf(cfg.get("config")); initPulsarAdminAndClientObj(); createPulsarSchemaFromConf(); @@ -82,7 +82,7 @@ public class PulsarSpace implements AutoCloseable { public String getPulsarSvcUrl() { return pulsarSvcUrl; } public String getWebSvcUrl() { return webSvcUrl; } - public PulsarNBClientConf getPulsarNBClientConf() { return pulsarNBClientConf; } + public PulsarClientConf getPulsarNBClientConf() { return pulsarClientConf; } public PulsarClient getPulsarClient() { return pulsarClient; } public PulsarAdmin getPulsarAdmin() { return pulsarAdmin; } public Schema getPulsarSchema() { return pulsarSchema; } @@ -111,7 +111,7 @@ public class PulsarSpace implements AutoCloseable { ClientBuilder clientBuilder = PulsarClient.builder(); try { - Map clientConfMap = pulsarNBClientConf.getClientConfMap(); + Map clientConfMap = pulsarClientConf.getClientConfMapRaw(); // Override "client.serviceUrl" setting in config.properties clientConfMap.remove("serviceUrl"); @@ -119,9 +119,9 @@ public class PulsarSpace implements AutoCloseable { // Pulsar Authentication String authPluginClassName = - (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authPulginClassName.label); + pulsarClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authPulginClassName.label); String authParams = - (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authParams.label); + pulsarClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authParams.label); if ( !StringUtils.isAnyBlank(authPluginClassName, authParams) ) { adminBuilder.authentication(authPluginClassName, authParams); @@ -131,7 +131,7 @@ public class PulsarSpace implements AutoCloseable { boolean useTls = StringUtils.contains(pulsarSvcUrl, "pulsar+ssl"); if ( useTls ) { String tlsHostnameVerificationEnableStr = - (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsHostnameVerificationEnable.label); + pulsarClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsHostnameVerificationEnable.label); boolean tlsHostnameVerificationEnable = BooleanUtils.toBoolean(tlsHostnameVerificationEnableStr); adminBuilder @@ -140,14 +140,14 @@ public class PulsarSpace implements AutoCloseable { .enableTlsHostnameVerification(tlsHostnameVerificationEnable); String tlsTrustCertsFilePath = - (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsTrustCertsFilePath.label); + pulsarClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsTrustCertsFilePath.label); if (!StringUtils.isBlank(tlsTrustCertsFilePath)) { adminBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath); clientBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath); } String tlsAllowInsecureConnectionStr = - (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsAllowInsecureConnection.label); + pulsarClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsAllowInsecureConnection.label); boolean tlsAllowInsecureConnection = BooleanUtils.toBoolean(tlsAllowInsecureConnectionStr); adminBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection); clientBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection); @@ -188,14 +188,12 @@ public class PulsarSpace implements AutoCloseable { private Schema buildSchemaFromDefinition(String schemaTypeConfEntry, String schemaDefinitionConfEntry) { - Object value = pulsarNBClientConf.getSchemaConfValue(schemaTypeConfEntry); - Object schemaDefinition = pulsarNBClientConf.getSchemaConfValue(schemaDefinitionConfEntry); - String schemaType = (value != null) ? value.toString() : ""; + String schemaType = pulsarClientConf.getSchemaConfValue(schemaTypeConfEntry); + String schemaDef = pulsarClientConf.getSchemaConfValue(schemaDefinitionConfEntry); Schema result; if (PulsarAdapterUtil.isAvroSchemaTypeStr(schemaType)) { - String schemaDefStr = (schemaDefinition != null) ? schemaDefinition.toString() : ""; - result = PulsarAdapterUtil.getAvroSchema(schemaType, schemaDefStr); + result = PulsarAdapterUtil.getAvroSchema(schemaType, schemaDef); } else if (PulsarAdapterUtil.isPrimitiveSchemaTypeStr(schemaType)) { result = PulsarAdapterUtil.getPrimitiveTypeSchema(schemaType); } else if (PulsarAdapterUtil.isAutoConsumeSchemaTypeStr(schemaType)) { @@ -210,12 +208,12 @@ public class PulsarSpace implements AutoCloseable { pulsarSchema = buildSchemaFromDefinition("schema.type", "schema.definition"); // this is to allow KEY_VALUE schema - if (pulsarNBClientConf.hasSchemaConfKey("schema.key.type")) { + if (pulsarClientConf.hasSchemaConfKey("schema.key.type")) { Schema pulsarKeySchema = buildSchemaFromDefinition("schema.key.type", "schema.key.definition"); - Object encodingType = pulsarNBClientConf.getSchemaConfValue("schema.keyvalue.encodingtype"); + String encodingType = pulsarClientConf.getSchemaConfValue("schema.keyvalue.encodingtype"); KeyValueEncodingType keyValueEncodingType = KeyValueEncodingType.SEPARATED; if (encodingType != null) { - keyValueEncodingType = KeyValueEncodingType.valueOf(encodingType.toString()); + keyValueEncodingType = KeyValueEncodingType.valueOf(encodingType); } pulsarSchema = Schema.KeyValue(pulsarKeySchema, pulsarSchema, keyValueEncodingType); } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java index 74c9e0ce2..cfe9231ef 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java @@ -246,7 +246,7 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser producerConf = pulsarSpace.getPulsarNBClientConf().getProducerConfMap(); + Map producerConf = pulsarSpace.getPulsarNBClientConf().getProducerConfMapTgt(); // Remove global level settings: "topicName" and "producerName" producerConf.remove(PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.topicName.label); @@ -265,13 +265,11 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser consumerConf = new HashMap<>(pulsarSpace.getPulsarNBClientConf().getConsumerConfMap()); + Map consumerConf = new HashMap<>(pulsarSpace.getPulsarNBClientConf().getConsumerConfMapTgt()); // Remove global level settings: // - "topicNames", "topicsPattern", "subscriptionName", "subscriptionType", "consumerName" @@ -628,7 +626,7 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser readerConf = pulsarSpace.getPulsarNBClientConf().getReaderConfMap(); + Map readerConf = pulsarSpace.getPulsarNBClientConf().getReaderConfMapTgt(); // Remove global level settings: "topicName" and "readerName" readerConf.remove(PulsarAdapterUtil.READER_CONF_STD_KEY.topicName.label); diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java index 965b323b0..39c346473 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java @@ -16,7 +16,6 @@ package io.nosqlbench.adapter.pulsar.ops; -import com.codahale.metrics.Histogram; import com.codahale.metrics.Timer; import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterAsyncOperationFailedException; import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; @@ -84,7 +83,6 @@ public class MessageProducerOp extends PulsarClientOp { this.msgValue = msgValue; getMsgPropMapFromRawJsonStr(); - getMsgPropMapFromRawJsonStr(); } private MessageSequenceNumberSendingHandler getMessageSequenceNumberSendingHandler(String topicName) { @@ -131,7 +129,7 @@ public class MessageProducerOp extends PulsarClientOp { } // set message key - if (!StringUtils.isBlank(msgKey)) { + if ( !StringUtils.isBlank(msgKey) && !(pulsarSchema instanceof KeyValueSchema) ) { typedMessageBuilder = typedMessageBuilder.key(msgKey); } @@ -145,31 +143,32 @@ public class MessageProducerOp extends PulsarClientOp { SchemaType schemaType = pulsarSchema.getSchemaInfo().getType(); if (pulsarSchema instanceof KeyValueSchema) { - // {KEY IN JSON}||{VALUE IN JSON} - int separator = msgValue.indexOf("}||{"); - if (separator < 0) { - throw new IllegalArgumentException("KeyValue payload MUST be in form {KEY IN JSON}||{VALUE IN JSON} (with 2 pipes that separate the KEY part from the VALUE part)"); - } - String keyInput = msgValue.substring(0, separator + 1); - String valueInput = msgValue.substring(separator + 3); +// // {KEY IN JSON}||{VALUE IN JSON} +// int separator = msgValue.indexOf("}||{"); +// if (separator < 0) { +// throw new IllegalArgumentException("KeyValue payload MUST be in form {KEY IN JSON}||{VALUE IN JSON} (with 2 pipes that separate the KEY part from the VALUE part)"); +// } +// String keyInput = msgValue.substring(0, separator + 1); +// String valueInput = msgValue.substring(separator + 3); KeyValueSchema keyValueSchema = (KeyValueSchema) pulsarSchema; org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration(); GenericRecord payload = PulsarAvroSchemaUtil.GetGenericRecord_PulsarAvro( (GenericAvroSchema) keyValueSchema.getValueSchema(), avroSchema, - valueInput + msgValue ); org.apache.avro.Schema avroSchemaForKey = getKeyAvroSchemaFromConfiguration(); GenericRecord key = PulsarAvroSchemaUtil.GetGenericRecord_PulsarAvro( (GenericAvroSchema) keyValueSchema.getKeySchema(), avroSchemaForKey, - keyInput + msgKey ); + typedMessageBuilder = typedMessageBuilder.value(new KeyValue(key, payload)); // TODO: add a way to calculate the message size for KEY_VALUE messages - messageSize = msgValue.length(); + messageSize = msgKey.length() + msgValue.length(); } else if (PulsarAdapterUtil.isAvroSchemaTypeStr(schemaType.name())) { GenericRecord payload = PulsarAvroSchemaUtil.GetGenericRecord_PulsarAvro( @@ -209,14 +208,14 @@ public class MessageProducerOp extends PulsarClientOp { logger.debug("({}) Sync message sent: msg-key={}; msg-properties={}; msg-payload={})", producer.getProducerName(), msgKey, - msgPropRawJsonStr, + msgProperties, avroGenericRecord.toString()); } else { logger.debug("({}) Sync message sent; msg-key={}; msg-properties={}; msg-payload={}", producer.getProducerName(), msgKey, - msgPropRawJsonStr, + msgProperties, msgValue); } } @@ -225,7 +224,7 @@ public class MessageProducerOp extends PulsarClientOp { String errMsg = "Sync message sending failed: " + "key - " + msgKey + "; " + - "properties - " + msgPropRawJsonStr + "; " + + "properties - " + msgProperties + "; " + "payload - " + msgValue; logger.trace(errMsg); @@ -260,21 +259,21 @@ public class MessageProducerOp extends PulsarClientOp { logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={})", producer.getProducerName(), msgKey, - msgPropRawJsonStr, + msgProperties, avroGenericRecord.toString()); } else { logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={}", producer.getProducerName(), msgKey, - msgPropRawJsonStr, + msgProperties, msgValue); } } }).exceptionally(ex -> { logger.error("Async message sending failed: " + "key - " + msgKey + "; " + - "properties - " + msgPropRawJsonStr + "; " + + "properties - " + msgProperties + "; " + "payload - " + msgValue); throw new PulsarAdapterAsyncOperationFailedException(ex); diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java index 130ce032f..2150c84bf 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java @@ -17,6 +17,8 @@ package io.nosqlbench.adapter.pulsar.util; import com.fasterxml.jackson.databind.ObjectMapper; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterInvalidParamException; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -460,19 +462,25 @@ public class PulsarAdapterUtil { // Check if payloadStr points to a file (e.g. "file:///path/to/a/file") if (isAvroSchemaTypeStr(typeStr)) { if (StringUtils.isBlank(schemaDefinitionStr)) { - throw new RuntimeException("Schema definition must be provided for \"Avro\" schema type!"); - } else if (schemaDefinitionStr.startsWith(filePrefix)) { + throw new PulsarAdapterInvalidParamException( + "Schema definition must be provided for \"Avro\" schema type!"); + } + else if (schemaDefinitionStr.startsWith(filePrefix)) { try { Path filePath = Paths.get(URI.create(schemaDefinitionStr)); schemaDefinitionStr = Files.readString(filePath, StandardCharsets.US_ASCII); - } catch (IOException ioe) { - throw new RuntimeException("Error reading the specified \"Avro\" schema definition file: " + definitionStr + ": " + ioe.getMessage()); + } + catch (IOException ioe) { + throw new PulsarAdapterUnexpectedException( + "Error reading the specified \"Avro\" schema definition file: " + definitionStr + ": " + ioe.getMessage()); } } schema = PulsarAvroSchemaUtil.GetSchema_PulsarAvro("NBAvro", schemaDefinitionStr); - } else { - throw new RuntimeException("Trying to create a \"Avro\" schema for a non-Avro schema type string: " + typeStr); + } + else { + throw new PulsarAdapterInvalidParamException( + "Trying to create a \"Avro\" schema for a non-Avro schema type string: " + typeStr); } return schema; diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarNBClientConf.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarClientConf.java similarity index 54% rename from adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarNBClientConf.java rename to adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarClientConf.java index a434bc10d..9fd56be8f 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarNBClientConf.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarClientConf.java @@ -23,6 +23,7 @@ import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder; import org.apache.commons.configuration2.builder.fluent.Parameters; import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -32,9 +33,9 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; -public class PulsarNBClientConf { +public class PulsarClientConf { - private final static Logger logger = LogManager.getLogger(PulsarNBClientConf.class); + private final static Logger logger = LogManager.getLogger(PulsarClientConf.class); private String canonicalFilePath = ""; @@ -43,14 +44,44 @@ public class PulsarNBClientConf { public static final String PRODUCER_CONF_PREFIX = "producer"; public static final String CONSUMER_CONF_PREFIX = "consumer"; public static final String READER_CONF_PREFIX = "reader"; - private final HashMap schemaConfMap = new HashMap<>(); - private final HashMap clientConfMap = new HashMap<>(); - private final HashMap producerConfMap = new HashMap<>(); - private final HashMap consumerConfMap = new HashMap<>(); - private final HashMap readerConfMap = new HashMap<>(); - // TODO: add support for other operation types: websocket-producer, managed-ledger + private final Map schemaConfMapRaw = new HashMap<>(); + private final Map clientConfMapRaw = new HashMap<>(); - public PulsarNBClientConf(String fileName) { + // "Raw" map is what is read from the config properties file + // "Tgt" map is what is really needed in the Pulsar producer/consumer/reader API + private final Map producerConfMapRaw = new HashMap<>(); + private final Map producerConfMapTgt = new HashMap<>(); + + private final Map consumerConfMapRaw = new HashMap<>(); + private final Map consumerConfMapTgt = new HashMap<>(); + + private final Map readerConfMapRaw = new HashMap<>(); + private final Map readerConfMapTgt = new HashMap<>(); + + public PulsarClientConf(String fileName) { + + ////////////////// + // Read related Pulsar client configuration settings from a file + readRawConfFromFile(fileName); + + + ////////////////// + // Ignores the following Pulsar client/producer/consumer configurations since + // they need to be specified either as the NB CLI parameters or as the NB yaml + // OpTemplate parameters. + clientConfMapRaw.remove("brokerServiceUrl"); + clientConfMapRaw.remove("webServiceUrl"); + + + ////////////////// + // Convert the raw configuration map () to the required map () + producerConfMapTgt.putAll(PulsarConfConverter.convertRawProducerConf(producerConfMapRaw)); + consumerConfMapTgt.putAll(PulsarConfConverter.convertRawConsumerConf(consumerConfMapRaw)); + // TODO: Reader API is not disabled at the moment. Revisit when needed + } + + + public void readRawConfFromFile(String fileName) { File file = new File(fileName); try { @@ -65,44 +96,37 @@ public class PulsarNBClientConf { Configuration config = builder.getConfiguration(); - // Get schema specific configuration settings - for (Iterator it = config.getKeys(SCHEMA_CONF_PREFIX); it.hasNext(); ) { + for (Iterator it = config.getKeys(); it.hasNext(); ) { String confKey = it.next(); String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - schemaConfMap.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - // Get client connection specific configuration settings - for (Iterator it = config.getKeys(CLIENT_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - clientConfMap.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } + if (!StringUtils.isBlank(confVal)) { - // Get producer specific configuration settings - for (Iterator it = config.getKeys(PRODUCER_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - producerConfMap.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get consumer specific configuration settings - for (Iterator it = config.getKeys(CONSUMER_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - consumerConfMap.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get reader specific configuration settings - for (Iterator it = config.getKeys(READER_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - readerConfMap.put(confKey.substring(READER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); + // Get schema specific configuration settings, removing "schema." prefix + if (StringUtils.startsWith(confKey, SCHEMA_CONF_PREFIX)) { + schemaConfMapRaw.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), confVal); + } + // Get client connection specific configuration settings, removing "client." prefix + // <<< https://pulsar.apache.org/docs/reference-configuration/#client >>> + else if (StringUtils.startsWith(confKey, CLIENT_CONF_PREFIX)) { + clientConfMapRaw.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), confVal); + } + // Get producer specific configuration settings, removing "producer." prefix + // <<< https://pulsar.apache.org/docs/client-libraries-java/#configure-producer >>> + else if (StringUtils.startsWith(confKey, PRODUCER_CONF_PREFIX)) { + producerConfMapRaw.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), confVal); + } + // Get consumer specific configuration settings, removing "consumer." prefix + // <<< https://pulsar.apache.org/docs/client-libraries-java/#configure-consumer >>> + else if (StringUtils.startsWith(confKey, CONSUMER_CONF_PREFIX)) { + consumerConfMapRaw.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), confVal); + } + // Get reader specific configuration settings, removing "reader." prefix + // <<< https://pulsar.apache.org/docs/2.10.x/client-libraries-java/#configure-reader >>> + else if (StringUtils.startsWith(confKey, READER_CONF_PREFIX)) { + readerConfMapRaw.put(confKey.substring(READER_CONF_PREFIX.length() + 1), confVal); + } + } } } catch (IOException ioe) { logger.error("Can't read the specified config properties file!"); @@ -114,78 +138,59 @@ public class PulsarNBClientConf { } + public Map getSchemaConfMapRaw() { return this.schemaConfMapRaw; } + public Map getClientConfMapRaw() { return this.clientConfMapRaw; } + public Map getProducerConfMapRaw() { return this.producerConfMapRaw; } + public Map getProducerConfMapTgt() { return this.producerConfMapTgt; } + public Map getConsumerConfMapRaw() { return this.consumerConfMapRaw; } + public Map getConsumerConfMapTgt() { return this.consumerConfMapTgt; } + public Map getReaderConfMapRaw() { return this.readerConfMapRaw; } + public Map getReaderConfMapTgt() { return this.readerConfMapTgt; } + + + public String toString() { + return new ToStringBuilder(this). + append("schemaConfMapRaw", schemaConfMapRaw.toString()). + append("clientConfMapRaw", clientConfMapRaw.toString()). + append("producerConfMapRaw", producerConfMapRaw.toString()). + append("consumerConfMapRaw", consumerConfMapRaw.toString()). + append("readerConfMapRaw", readerConfMapRaw.toString()). + toString(); + } + ////////////////// // Get Schema related config - public Map getSchemaConfMap() { - return this.schemaConfMap; - } public boolean hasSchemaConfKey(String key) { if (key.contains(SCHEMA_CONF_PREFIX)) - return schemaConfMap.containsKey(key.substring(SCHEMA_CONF_PREFIX.length() + 1)); + return schemaConfMapRaw.containsKey(key.substring(SCHEMA_CONF_PREFIX.length() + 1)); else - return schemaConfMap.containsKey(key); + return schemaConfMapRaw.containsKey(key); } - public Object getSchemaConfValue(String key) { + public String getSchemaConfValue(String key) { if (key.contains(SCHEMA_CONF_PREFIX)) - return schemaConfMap.get(key.substring(SCHEMA_CONF_PREFIX.length()+1)); + return schemaConfMapRaw.get(key.substring(SCHEMA_CONF_PREFIX.length()+1)); else - return schemaConfMap.get(key); - } - public void setSchemaConfValue(String key, Object value) { - if (key.contains(SCHEMA_CONF_PREFIX)) - schemaConfMap.put(key.substring(SCHEMA_CONF_PREFIX.length() + 1), value); - else - schemaConfMap.put(key, value); + return schemaConfMapRaw.get(key); } ////////////////// // Get Pulsar client related config - public Map getClientConfMap() { - return this.clientConfMap; - } - public boolean hasClientConfKey(String key) { + public String getClientConfValue(String key) { if (key.contains(CLIENT_CONF_PREFIX)) - return clientConfMap.containsKey(key.substring(CLIENT_CONF_PREFIX.length() + 1)); + return clientConfMapRaw.get(key.substring(CLIENT_CONF_PREFIX.length()+1)); else - return clientConfMap.containsKey(key); - } - public Object getClientConfValue(String key) { - if (key.contains(CLIENT_CONF_PREFIX)) - return clientConfMap.get(key.substring(CLIENT_CONF_PREFIX.length()+1)); - else - return clientConfMap.get(key); - } - public void setClientConfValue(String key, Object value) { - if (key.contains(CLIENT_CONF_PREFIX)) - clientConfMap.put(key.substring(CLIENT_CONF_PREFIX.length() + 1), value); - else - clientConfMap.put(key, value); + return clientConfMapRaw.get(key); } ////////////////// // Get Pulsar producer related config - public Map getProducerConfMap() { - return this.producerConfMap; - } - public boolean hasProducerConfKey(String key) { - if (key.contains(PRODUCER_CONF_PREFIX)) - return producerConfMap.containsKey(key.substring(PRODUCER_CONF_PREFIX.length() + 1)); - else - return producerConfMap.containsKey(key); - } public Object getProducerConfValue(String key) { if (key.contains(PRODUCER_CONF_PREFIX)) - return producerConfMap.get(key.substring(PRODUCER_CONF_PREFIX.length()+1)); + return producerConfMapTgt.get(key.substring(PRODUCER_CONF_PREFIX.length()+1)); else - return producerConfMap.get(key); - } - public void setProducerConfValue(String key, Object value) { - if (key.contains(PRODUCER_CONF_PREFIX)) - producerConfMap.put(key.substring(PRODUCER_CONF_PREFIX.length()+1), value); - else - producerConfMap.put(key, value); + return producerConfMapTgt.get(key); } // other producer helper functions ... public String getProducerName() { @@ -208,30 +213,15 @@ public class PulsarNBClientConf { ////////////////// // Get Pulsar consumer related config - public Map getConsumerConfMap() { - return this.consumerConfMap; - } - public boolean hasConsumerConfKey(String key) { + public String getConsumerConfValue(String key) { if (key.contains(CONSUMER_CONF_PREFIX)) - return consumerConfMap.containsKey(key.substring(CONSUMER_CONF_PREFIX.length() + 1)); + return consumerConfMapRaw.get(key.substring(CONSUMER_CONF_PREFIX.length() + 1)); else - return consumerConfMap.containsKey(key); - } - public Object getConsumerConfValue(String key) { - if (key.contains(CONSUMER_CONF_PREFIX)) - return consumerConfMap.get(key.substring(CONSUMER_CONF_PREFIX.length() + 1)); - else - return consumerConfMap.get(key); - } - public void setConsumerConfValue(String key, Object value) { - if (key.contains(CONSUMER_CONF_PREFIX)) - consumerConfMap.put(key.substring(CONSUMER_CONF_PREFIX.length() + 1), value); - else - consumerConfMap.put(key, value); + return consumerConfMapRaw.get(key); } // Other consumer helper functions ... public String getConsumerTopicNames() { - Object confValue = getConsumerConfValue( + String confValue = getConsumerConfValue( "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicNames.label); if (confValue == null) return ""; @@ -284,26 +274,23 @@ public class PulsarNBClientConf { ////////////////// // Get Pulsar reader related config - public Map getReaderConfMap() { - return this.readerConfMap; - } public boolean hasReaderConfKey(String key) { if (key.contains(READER_CONF_PREFIX)) - return readerConfMap.containsKey(key.substring(READER_CONF_PREFIX.length() + 1)); + return readerConfMapRaw.containsKey(key.substring(READER_CONF_PREFIX.length() + 1)); else - return readerConfMap.containsKey(key); + return readerConfMapRaw.containsKey(key); } public Object getReaderConfValue(String key) { if (key.contains(READER_CONF_PREFIX)) - return readerConfMap.get(key.substring(READER_CONF_PREFIX.length() + 1)); + return readerConfMapRaw.get(key.substring(READER_CONF_PREFIX.length() + 1)); else - return readerConfMap.get(key); + return readerConfMapRaw.get(key); } - public void setReaderConfValue(String key, Object value) { + public void setReaderConfValue(String key, String value) { if (key.contains(READER_CONF_PREFIX)) - readerConfMap.put(key.substring(READER_CONF_PREFIX.length() + 1), value); + readerConfMapRaw.put(key.substring(READER_CONF_PREFIX.length() + 1), value); else - readerConfMap.put(key, value); + readerConfMapRaw.put(key, value); } // Other reader helper functions ... public String getReaderTopicName() { diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarConfConverter.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarConfConverter.java new file mode 100644 index 000000000..8024cb39e --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarConfConverter.java @@ -0,0 +1,367 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterInvalidParamException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.DeadLetterPolicy; +import org.apache.pulsar.client.api.RedeliveryBackoff; +import org.apache.pulsar.client.impl.MultiplierRedeliveryBackoff; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PulsarConfConverter { + + // <<< https://pulsar.apache.org/docs/client-libraries-java/#configure-producer >>> + private final static Map validPulsarProducerConfKeyTypeMap = Map.ofEntries( + Map.entry("topicName", "String"), + Map.entry("producerName","String"), + Map.entry("sendTimeoutMs","long"), + Map.entry("blockIfQueueFull","boolean"), + Map.entry("maxPendingMessages","int"), + Map.entry("maxPendingMessagesAcrossPartitions","int"), + Map.entry("messageRoutingMode","MessageRoutingMode"), + Map.entry("hashingScheme","HashingScheme"), + Map.entry("cryptoFailureAction","ProducerCryptoFailureAction"), + Map.entry("batchingMaxPublishDelayMicros","long"), + Map.entry("batchingMaxMessages","int"), + Map.entry("batchingEnabled","boolean"), + Map.entry("chunkingEnabled","boolean"), + Map.entry("compressionType","CompressionType"), + Map.entry("initialSubscriptionName","string") + ); + public static Map convertRawProducerConf(Map pulsarProducerConfMapRaw) { + Map producerConfObjMap = new HashMap<>(); + setConfObjMapForPrimitives(producerConfObjMap, pulsarProducerConfMapRaw, validPulsarProducerConfKeyTypeMap); + + /** + * Non-primitive type processing for Pulsar producer configuration items + */ + // TODO: Skip the following Pulsar configuration items for now because they're not really + // needed in the NB S4J testing at the moment. Add support for them when needed. + // * messageRoutingMode + // * hashingScheme + // * cryptoFailureAction + + // "compressionType" has value type "CompressionType" + // - expecting the following values: 'LZ4', 'ZLIB', 'ZSTD', 'SNAPPY' + String confKeyName = "compressionType"; + String confVal = pulsarProducerConfMapRaw.get(confKeyName); + String expectedVal = "(LZ4|ZLIB|ZSTD|SNAPPY)"; + + if (StringUtils.isNotBlank(confVal)) { + if (StringUtils.equalsAnyIgnoreCase(confVal, "LZ4", "ZLIB", "ZSTD", "SNAPPY")) { + CompressionType compressionType = CompressionType.NONE; + + switch (StringUtils.upperCase(confVal)) { + case "LZ4": + compressionType = CompressionType.LZ4; + case "ZLIB": + compressionType = CompressionType.ZLIB; + case "ZSTD": + compressionType = CompressionType.ZSTD; + case "SNAPPY": + compressionType = CompressionType.SNAPPY; + } + + producerConfObjMap.put(confKeyName, compressionType); + } else { + throw new PulsarAdapterInvalidParamException( + getInvalidConfValStr(confKeyName, confVal, "producer", expectedVal)); + } + } + + return producerConfObjMap; + } + + + // https://pulsar.apache.org/docs/client-libraries-java/#configure-consumer + private final static Map validPulsarConsumerConfKeyTypeMap = Map.ofEntries( + Map.entry("topicNames", "Set"), + Map.entry("topicsPattern","Pattern"), + Map.entry("subscriptionName","String"), + Map.entry("subscriptionType","SubscriptionType"), + Map.entry("receiverQueueSize","int"), + Map.entry("acknowledgementsGroupTimeMicros","long"), + Map.entry("negativeAckRedeliveryDelayMicros","long"), + Map.entry("maxTotalReceiverQueueSizeAcrossPartitions","int"), + Map.entry("consumerName","String"), + Map.entry("ackTimeoutMillis","long"), + Map.entry("tickDurationMillis","long"), + Map.entry("priorityLevel","int"), + Map.entry("cryptoFailureAction","ConsumerCryptoFailureAction"), + Map.entry("properties","SortedMap"), + Map.entry("readCompacted","boolean"), + Map.entry("subscriptionInitialPosition", "SubscriptionInitialPosition"), + Map.entry("patternAutoDiscoveryPeriod", "int"), + Map.entry("regexSubscriptionMode", "RegexSubscriptionMode"), + Map.entry("deadLetterPolicy", "DeadLetterPolicy"), + Map.entry("autoUpdatePartitions", "boolean"), + Map.entry("replicateSubscriptionState", "boolean"), + Map.entry("negativeAckRedeliveryBackoff", "RedeliveryBackoff"), + Map.entry("ackTimeoutRedeliveryBackoff", "RedeliveryBackoff"), + Map.entry("autoAckOldestChunkedMessageOnQueueFull", "boolean"), + Map.entry("maxPendingChunkedMessage", "int"), + Map.entry("expireTimeOfIncompleteChunkedMessageMillis", "long") + ); + public static Map convertRawConsumerConf(Map pulsarConsumerConfMapRaw) { + Map consumerConfObjMap = new HashMap<>(); + setConfObjMapForPrimitives(consumerConfObjMap, pulsarConsumerConfMapRaw, validPulsarConsumerConfKeyTypeMap); + + /** + * Non-primitive type processing for Pulsar consumer configuration items + */ + // NOTE: The following non-primitive type configuration items are excluded since + // they'll be handled in PulsarBasedOpDispenser.getConsumer() method directly + // * topicNames + // * topicPattern + // * subscriptionType + + + // TODO: Skip the following Pulsar configuration items for now because they're not really + // needed in the NB S4J testing right now. Add the support for them when needed. + // * subscriptionInitialPosition + // * regexSubscriptionMode + // * cryptoFailureAction + + + // "properties" has value type "SortedMap" + // - expecting the value string has the format: a JSON string that includes a set of key/value pairs + String confKeyName = "properties"; + String confVal = pulsarConsumerConfMapRaw.get(confKeyName); + String expectedVal = "{\"property1\":\"value1\", \"property2\":\"value2\"}, ..."; + + ObjectMapper mapper = new ObjectMapper(); + + if (StringUtils.isNotBlank(confVal)) { + try { + Map consumerProperties = mapper.readValue(confVal, Map.class); + + // Empty map value is considered as no value + if (!consumerProperties.isEmpty()) { + consumerConfObjMap.put(confKeyName, consumerProperties); + } + + } catch (Exception e) { + throw new PulsarAdapterInvalidParamException( + getInvalidConfValStr(confKeyName, confVal, "consumer", expectedVal)); + } + } + + // "deadLetterPolicy" + // - expecting the value is a JSON string has the format: + // {"maxRedeliverCount":"","deadLetterTopic":"","initialSubscriptionName":""} + confKeyName = "deadLetterPolicy"; + confVal = pulsarConsumerConfMapRaw.get(confKeyName); + expectedVal = "{" + + "\"maxRedeliverCount\":\"\"," + + "\"deadLetterTopic\":\"\"," + + "\"initialSubscriptionName\":\"\"}"; + + if (StringUtils.isNotBlank(confVal)) { + try { + Map dlqPolicyMap = mapper.readValue(confVal, Map.class); + + // Empty map value is considered as no value + if (!dlqPolicyMap.isEmpty()) { + boolean valid = true; + + // The JSON key must be one of "maxRedeliverCount", "deadLetterTopic", "initialSubscriptionName" + for (String key : dlqPolicyMap.keySet()) { + if (!StringUtils.equalsAnyIgnoreCase(key, + "maxRedeliverCount", "deadLetterTopic", "initialSubscriptionName")) { + valid = false; + break; + } + } + + // DLQ.maxRedeliverCount is mandatory + if (valid && !dlqPolicyMap.containsKey("maxRedeliverCount")) { + valid = false; + } + + String maxRedeliverCountStr = dlqPolicyMap.get("maxRedeliverCount"); + if (!NumberUtils.isCreatable(maxRedeliverCountStr)) { + valid = false; + } + + if (valid) { + DeadLetterPolicy deadLetterPolicy = DeadLetterPolicy.builder() + .maxRedeliverCount(NumberUtils.toInt(maxRedeliverCountStr)) + .deadLetterTopic(dlqPolicyMap.get("deadLetterTopic")) + .initialSubscriptionName(dlqPolicyMap.get("initialSubscriptionName")) + .build(); + + consumerConfObjMap.put(confKeyName, deadLetterPolicy); + } else { + throw new PulsarAdapterInvalidParamException( + getInvalidConfValStr(confKeyName, confVal, "consumer", expectedVal)); + } + } + } catch (Exception e) { + throw new PulsarAdapterInvalidParamException( + getInvalidConfValStr(confKeyName, confVal, "consumer", expectedVal)); + } + } + + // "negativeAckRedeliveryBackoff" or "ackTimeoutRedeliveryBackoff" + // - expecting the value is a JSON string has the format: + // {"minDelayMs":"", "maxDelayMs":"", "multiplier":""} + String[] redeliveryBackoffConfigSet = {"negativeAckRedeliveryBackoff", "ackTimeoutRedeliveryBackoff"}; + expectedVal = "{" + + "\"minDelayMs\":\"\"," + + "\"maxDelayMs\":\"\"," + + "\"multiplier\":\"\"}"; + + for (String confKey : redeliveryBackoffConfigSet) { + confVal = pulsarConsumerConfMapRaw.get(confKey); + + if (StringUtils.isNotBlank(confVal)) { + try { + Map redliveryBackoffMap = mapper.readValue(confVal, Map.class); + + // Empty map value is considered as no value + if (! redliveryBackoffMap.isEmpty()) { + boolean valid = true; + + // The JSON key must be one of "maxRedeliverCount", "deadLetterTopic", "initialSubscriptionName" + for (String key : redliveryBackoffMap.keySet()) { + if (!StringUtils.equalsAnyIgnoreCase(key, + "minDelayMs", "maxDelayMs", "multiplier")) { + valid = false; + break; + } + } + + String minDelayMsStr = redliveryBackoffMap.get("minDelayMs"); + String maxDelayMsStr = redliveryBackoffMap.get("maxDelayMs"); + String multiplierStr = redliveryBackoffMap.get("multiplier"); + + if ((StringUtils.isNotBlank(minDelayMsStr) && !NumberUtils.isCreatable(minDelayMsStr)) || + (StringUtils.isNotBlank(maxDelayMsStr) && !NumberUtils.isCreatable(maxDelayMsStr)) || + (StringUtils.isNotBlank(multiplierStr) && !NumberUtils.isCreatable(multiplierStr))) { + valid = false; + } + + if (valid) { + RedeliveryBackoff redeliveryBackoff = MultiplierRedeliveryBackoff.builder() + .minDelayMs(NumberUtils.toLong(minDelayMsStr)) + .maxDelayMs(NumberUtils.toLong(maxDelayMsStr)) + .multiplier(NumberUtils.toDouble(multiplierStr)) + .build(); + + consumerConfObjMap.put(confKey, redeliveryBackoff); + + } else { + throw new PulsarAdapterInvalidParamException( + getInvalidConfValStr(confKey, confVal, "consumer", expectedVal)); + } + } + + } catch (Exception e) { + throw new PulsarAdapterInvalidParamException( + getInvalidConfValStr(confKey, confVal, "consumer", expectedVal)); + } + } + } + + return consumerConfObjMap; + } + + + // Utility function + // - get configuration key names by the value type + private static List getConfKeyNameByValueType(Map confKeyTypeMap, String tgtValType) { + ArrayList confKeyNames = new ArrayList<>(); + + for (Map.Entry entry: confKeyTypeMap.entrySet()) { + if (StringUtils.equalsIgnoreCase(entry.getValue().toString(), tgtValType)) { + confKeyNames.add(entry.getKey().toString()); + } + } + + return confKeyNames; + } + + // Conversion from Map to Map for configuration items with primitive + // value types + private static void setConfObjMapForPrimitives( + Map tgtConfObjMap, + Map srcConfMapRaw, + Map validConfKeyTypeMap) + { + List confKeyList = new ArrayList<>(); + + // All configuration items with "String" as the value type + confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "String"); + for (String confKey : confKeyList) { + if (srcConfMapRaw.containsKey(confKey)) { + String confVal = srcConfMapRaw.get(confKey); + if (StringUtils.isNotBlank(confVal)) { + tgtConfObjMap.put(confKey, confVal); + } + } + } + + // All configuration items with "long" as the value type + confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "long"); + for (String confKey : confKeyList) { + if (srcConfMapRaw.containsKey(confKey)) { + String confVal = srcConfMapRaw.get(confKey); + if (StringUtils.isNotBlank(confVal)) { + tgtConfObjMap.put(confKey, Long.valueOf(confVal)); + } + } + } + + // All configuration items with "int" as the value type + confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "int"); + for (String confKey : confKeyList) { + if (srcConfMapRaw.containsKey(confKey)) { + String confVal = srcConfMapRaw.get(confKey); + if (StringUtils.isNotBlank(confVal)) { + tgtConfObjMap.put(confKey, Integer.valueOf(confVal)); + } + } + } + + // All configuration items with "boolean" as the value type + confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "boolean"); + for (String confKey : confKeyList) { + if (srcConfMapRaw.containsKey(confKey)) { + String confVal = srcConfMapRaw.get(confKey); + if (StringUtils.isNotBlank(confVal)) { + tgtConfObjMap.put(confKey, Boolean.valueOf(confVal)); + } + } + } + + // TODO: So far the above primitive types should be good enough. + // Add support for other types when needed + } + + private static String getInvalidConfValStr(String confKey, String confVal, String configCategory, String expectedVal) { + return "Incorrect value \"" + confVal + "\" for Pulsar " + configCategory + + " configuration item of \"" + confKey + "\". Expecting the following value (format): " + expectedVal; + } +} diff --git a/adapter-pulsar/src/main/resources/admin-namespace.yaml b/adapter-pulsar/src/main/resources/admin_namespace.yaml similarity index 93% rename from adapter-pulsar/src/main/resources/admin-namespace.yaml rename to adapter-pulsar/src/main/resources/admin_namespace.yaml index b80e9cff2..4fd9b18ac 100644 --- a/adapter-pulsar/src/main/resources/admin-namespace.yaml +++ b/adapter-pulsar/src/main/resources/admin_namespace.yaml @@ -5,7 +5,7 @@ bindings: params: async_api: "false" - admin_delop: "true" + admin_delop: "false" blocks: admin-namespace-block: diff --git a/adapter-pulsar/src/main/resources/admin-tenant.yaml b/adapter-pulsar/src/main/resources/admin_tenant.yaml similarity index 81% rename from adapter-pulsar/src/main/resources/admin-tenant.yaml rename to adapter-pulsar/src/main/resources/admin_tenant.yaml index 2278e8c41..865643fde 100644 --- a/adapter-pulsar/src/main/resources/admin-tenant.yaml +++ b/adapter-pulsar/src/main/resources/admin_tenant.yaml @@ -4,7 +4,7 @@ bindings: params: async_api: "false" - admin_delop: "true" + admin_delop: "false" blocks: admin-tenant-block: @@ -12,6 +12,6 @@ blocks: phase: admin-tenant ops: op1: - AdminTenant: "{tenant}" + AdminTopic: "{tenant}" admin_roles: "" allowed_clusters: "" diff --git a/adapter-pulsar/src/main/resources/admin-topic.yaml b/adapter-pulsar/src/main/resources/admin_topic.yaml similarity index 88% rename from adapter-pulsar/src/main/resources/admin-topic.yaml rename to adapter-pulsar/src/main/resources/admin_topic.yaml index a5f81d0dd..6e30ad2bf 100644 --- a/adapter-pulsar/src/main/resources/admin-topic.yaml +++ b/adapter-pulsar/src/main/resources/admin_topic.yaml @@ -6,7 +6,7 @@ bindings: params: async_api: "false" - admin_delop: "true" + admin_delop: "false" blocks: admin-topic-block: @@ -15,5 +15,5 @@ blocks: ops: op1: AdminTopic: "{tenant}/{namespace}/{topic}" - enable_partition: "true" + enable_partition: "false" partition_num: "5" diff --git a/adapter-pulsar/src/main/resources/config.properties b/adapter-pulsar/src/main/resources/config.properties index 37be64384..9cc804c1f 100644 --- a/adapter-pulsar/src/main/resources/config.properties +++ b/adapter-pulsar/src/main/resources/config.properties @@ -1,4 +1,4 @@ -### Schema related configurations - schema.xxx +9### Schema related configurations - schema.xxx # valid types: # - primitive type (https://pulsar.apache.org/docs/en/schema-understand/#primitive-type) # - keyvalue (https://pulsar.apache.org/docs/en/schema-understand/#keyvalue) @@ -8,8 +8,10 @@ # TODO: as a starting point, only supports the following types # 1) primitive types, including bytearray (byte[]) which is default, for messages without schema # 2) Avro for messages with schema +#schema.key.type=avro +#schema.key.definition=file:///Users/yabinmeng/DataStax/MyNBMain/nosqlbench/adapter-pulsar/src/main/resources/iot-key-example.avsc #schema.type=avro -#schema.definition=file:///Users/yabinmeng/DataStax/MyNoSQLBench/nosqlbench/driver-pulsar/src/main/resources/activities/iot-example.avsc +#schema.definition=file:///Users/yabinmeng/DataStax/MyNBMain/nosqlbench/adapter-pulsar/src/main/resources/iot-value-example.avsc schema.type= schema.definition= @@ -29,8 +31,6 @@ producer.producerName= producer.topicName= producer.sendTimeoutMs= producer.blockIfQueueFull=true -producer.maxPendingMessages=5000 -producer.batchingMaxMessages=5000 ### Consumer related configurations (global) - consumer.xxx diff --git a/adapter-pulsar/src/main/resources/iot-key-example.avsc b/adapter-pulsar/src/main/resources/iot-key-example.avsc new file mode 100644 index 000000000..f36b52bc3 --- /dev/null +++ b/adapter-pulsar/src/main/resources/iot-key-example.avsc @@ -0,0 +1,9 @@ +{ + "type": "record", + "name": "IotSensorKey", + "namespace": "TestNS", + "fields" : [ + {"name": "Location", "type": "string"}, + {"name": "WellID", "type": "string"} + ] +} diff --git a/adapter-pulsar/src/main/resources/iot-value-example.avsc b/adapter-pulsar/src/main/resources/iot-value-example.avsc new file mode 100644 index 000000000..20bb894fd --- /dev/null +++ b/adapter-pulsar/src/main/resources/iot-value-example.avsc @@ -0,0 +1,11 @@ +{ + "type": "record", + "name": "IotSensor", + "namespace": "TestNS", + "fields" : [ + {"name": "SensorID", "type": "string"}, + {"name": "SensorType", "type": "string"}, + {"name": "ReadingTime", "type": "string"}, + {"name": "ReadingValue", "type": "float"} + ] +} diff --git a/adapter-pulsar/src/main/resources/msg_proc_avro.yaml b/adapter-pulsar/src/main/resources/msg_proc_avro.yaml new file mode 100644 index 000000000..bfc07a176 --- /dev/null +++ b/adapter-pulsar/src/main/resources/msg_proc_avro.yaml @@ -0,0 +1,41 @@ +bindings: + # message key and value + mykey: NumberNameToString() + location: Cities(); + well_id: ToUUID();ToString(); + sensor_id: ToUUID();ToString(); + reading_time: ToDateTime(); + reading_value: ToFloat(100); + +# document level parameters that apply to all Pulsar client types: +params: + async_api: "true" + +blocks: + msg-produce-block: + tags: + phase: msg-send + ops: + op1: + MessageProduce: "tnt0/ns0/tp1" + msg_key: | + { + "Location": "{location}", + "WellID": "{well_id}" + } + msg_value: | + { + "SensorID": "{sensor_id}", + "SensorType": "Temperature", + "ReadingTime": "{reading_time}", + "ReadingValue": {reading_value} + } + + msg-consume-block: + tags: + phase: msg-recv + ops: + op1: + MessageConsume: "tnt0/ns0/tp0" + subscription_name: "mynbsub" +# subscription_type: "shared" diff --git a/adapter-pulsar/src/main/resources/msg_proc_kvraw.yaml b/adapter-pulsar/src/main/resources/msg_proc_kvraw.yaml new file mode 100644 index 000000000..4aea8f44a --- /dev/null +++ b/adapter-pulsar/src/main/resources/msg_proc_kvraw.yaml @@ -0,0 +1,34 @@ +bindings: + # message key, property and value + mykey: NumberNameToString() + int_prop_val: ToString(); Prefix("IntProp_") + text_prop_val: AlphaNumericString(5); Prefix("TextProp_") + myvalue: AlphaNumericString(20) + +# document level parameters that apply to all Pulsar client types: +params: + async_api: "true" + +blocks: + msg-produce-block: + tags: + phase: msg-send + ops: + op1: + MessageProduce: "tnt0/ns0/tp0" + msg_key: "{mykey}" + msg_prop: | + { + "prop1": "{int_prop_val}", + "prop2": "{text_prop_val}" + } + msg_value: "{myvalue}" + + msg-consume-block: + tags: + phase: msg-recv + ops: + op1: + MessageConsume: "tnt0/ns0/tp0" + subscription_name: "mynbsub" +# subscription_type: "shared" diff --git a/adapter-pulsar/src/main/resources/pulsar.md b/adapter-pulsar/src/main/resources/pulsar.md new file mode 100644 index 000000000..586fecbb2 --- /dev/null +++ b/adapter-pulsar/src/main/resources/pulsar.md @@ -0,0 +1 @@ +<< to be added ... >> From bd8aff90816e7a39857c40ebfaba2dd95653611b Mon Sep 17 00:00:00 2001 From: yabinmeng Date: Fri, 18 Nov 2022 18:22:04 -0600 Subject: [PATCH 14/14] Added support for 1) subscription initial position and 2) regex subscription mode Also fixed a consumer configuration loading issue with dlq/negAck/ackTimeout policy using Pulsar client's loadConf() API. Addressed the review comments in PR #791; reverted the changes in ActivityMetrics and BasedOpDispenser --- .../adapter/pulsar/PulsarSpace.java | 20 +- .../MessageConsumerOpDispenser.java | 25 +- .../MessageProducerOpDispenser.java | 8 +- .../dispensers/MessageReaderOpDispenser.java | 3 +- .../dispensers/PulsarBaseOpDispenser.java | 354 +++++++----------- .../dispensers/PulsarClientOpDispenser.java | 12 +- .../adapter/pulsar/ops/MessageProducerOp.java | 13 +- .../MessageSequenceNumberSendingHandler.java | 9 +- .../pulsar/util/PulsarAdapterUtil.java | 153 +++++--- .../adapter/pulsar/util/PulsarClientConf.java | 214 ++++------- .../pulsar/util/PulsarConfConverter.java | 151 +++++--- .../src/main/resources/config.properties | 17 +- .../src/main/resources/msg_proc_kvraw.yaml | 4 +- .../api/activityimpl/BaseOpDispenser.java | 2 +- .../api/engine/metrics/ActivityMetrics.java | 9 - pom.xml | 1 - 16 files changed, 471 insertions(+), 524 deletions(-) diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java index 708e26a96..4aba161f1 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java @@ -119,9 +119,9 @@ public class PulsarSpace implements AutoCloseable { // Pulsar Authentication String authPluginClassName = - pulsarClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authPulginClassName.label); + pulsarClientConf.getClientConfValueRaw(PulsarAdapterUtil.CLNT_CONF_KEY.authPulginClassName.label); String authParams = - pulsarClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authParams.label); + pulsarClientConf.getClientConfValueRaw(PulsarAdapterUtil.CLNT_CONF_KEY.authParams.label); if ( !StringUtils.isAnyBlank(authPluginClassName, authParams) ) { adminBuilder.authentication(authPluginClassName, authParams); @@ -131,7 +131,7 @@ public class PulsarSpace implements AutoCloseable { boolean useTls = StringUtils.contains(pulsarSvcUrl, "pulsar+ssl"); if ( useTls ) { String tlsHostnameVerificationEnableStr = - pulsarClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsHostnameVerificationEnable.label); + pulsarClientConf.getClientConfValueRaw(PulsarAdapterUtil.CLNT_CONF_KEY.tlsHostnameVerificationEnable.label); boolean tlsHostnameVerificationEnable = BooleanUtils.toBoolean(tlsHostnameVerificationEnableStr); adminBuilder @@ -140,14 +140,14 @@ public class PulsarSpace implements AutoCloseable { .enableTlsHostnameVerification(tlsHostnameVerificationEnable); String tlsTrustCertsFilePath = - pulsarClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsTrustCertsFilePath.label); + pulsarClientConf.getClientConfValueRaw(PulsarAdapterUtil.CLNT_CONF_KEY.tlsTrustCertsFilePath.label); if (!StringUtils.isBlank(tlsTrustCertsFilePath)) { adminBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath); clientBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath); } String tlsAllowInsecureConnectionStr = - pulsarClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsAllowInsecureConnection.label); + pulsarClientConf.getClientConfValueRaw(PulsarAdapterUtil.CLNT_CONF_KEY.tlsAllowInsecureConnection.label); boolean tlsAllowInsecureConnection = BooleanUtils.toBoolean(tlsAllowInsecureConnectionStr); adminBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection); clientBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection); @@ -188,8 +188,8 @@ public class PulsarSpace implements AutoCloseable { private Schema buildSchemaFromDefinition(String schemaTypeConfEntry, String schemaDefinitionConfEntry) { - String schemaType = pulsarClientConf.getSchemaConfValue(schemaTypeConfEntry); - String schemaDef = pulsarClientConf.getSchemaConfValue(schemaDefinitionConfEntry); + String schemaType = pulsarClientConf.getSchemaConfValueRaw(schemaTypeConfEntry); + String schemaDef = pulsarClientConf.getSchemaConfValueRaw(schemaDefinitionConfEntry); Schema result; if (PulsarAdapterUtil.isAvroSchemaTypeStr(schemaType)) { @@ -210,11 +210,13 @@ public class PulsarSpace implements AutoCloseable { // this is to allow KEY_VALUE schema if (pulsarClientConf.hasSchemaConfKey("schema.key.type")) { Schema pulsarKeySchema = buildSchemaFromDefinition("schema.key.type", "schema.key.definition"); - String encodingType = pulsarClientConf.getSchemaConfValue("schema.keyvalue.encodingtype"); KeyValueEncodingType keyValueEncodingType = KeyValueEncodingType.SEPARATED; - if (encodingType != null) { + + String encodingType = pulsarClientConf.getSchemaConfValueRaw("schema.keyvalue.encodingtype"); + if (StringUtils.isNotBlank(encodingType)) { keyValueEncodingType = KeyValueEncodingType.valueOf(encodingType); } + pulsarSchema = Schema.KeyValue(pulsarKeySchema, pulsarSchema, keyValueEncodingType); } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java index 82edc63a9..4a60adeb8 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java @@ -35,12 +35,6 @@ public class MessageConsumerOpDispenser extends PulsarClientOpDispenser { private final static Logger logger = LogManager.getLogger("MessageConsumerOpDispenser"); - public static final String TOPIC_PATTERN_OP_PARAM = "topic_pattern"; - public static final String SUBSCRIPTION_NAME_OP_PARAM = "subscription_name"; - public static final String SUBSCRIPTION_TYPE_OP_PARAM = "subscription_type"; - public static final String CONSUMER_NAME_OP_PARAM = "consumer_name"; - public static final String RANGES_OP_PARAM = "ranges"; - private final LongFunction topicPatternFunc; private final LongFunction subscriptionNameFunc; private final LongFunction subscriptionTypeFunc; @@ -49,8 +43,8 @@ public class MessageConsumerOpDispenser extends PulsarClientOpDispenser { private final LongFunction e2eStartTimeSrcParamStrFunc; private final LongFunction consumerFunction; - private final ThreadLocal> receivedMessageSequenceTrackersForTopicThreadLocal = - ThreadLocal.withInitial(HashMap::new); + private final ThreadLocal> + receivedMessageSequenceTrackersForTopicThreadLocal = ThreadLocal.withInitial(HashMap::new); public MessageConsumerOpDispenser(DriverAdapter adapter, ParsedOp op, @@ -58,11 +52,16 @@ public class MessageConsumerOpDispenser extends PulsarClientOpDispenser { PulsarSpace pulsarSpace) { super(adapter, op, tgtNameFunc, pulsarSpace); - this.topicPatternFunc = lookupOptionalStrOpValueFunc(TOPIC_PATTERN_OP_PARAM); - this.subscriptionNameFunc = lookupMandtoryStrOpValueFunc(SUBSCRIPTION_NAME_OP_PARAM); - this.subscriptionTypeFunc = lookupOptionalStrOpValueFunc(SUBSCRIPTION_TYPE_OP_PARAM); - this.cycleConsumerNameFunc = lookupOptionalStrOpValueFunc(CONSUMER_NAME_OP_PARAM); - this.rangesFunc = lookupOptionalStrOpValueFunc(RANGES_OP_PARAM); + this.topicPatternFunc = + lookupOptionalStrOpValueFunc(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label); + this.subscriptionNameFunc = + lookupMandtoryStrOpValueFunc(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label); + this.subscriptionTypeFunc = + lookupOptionalStrOpValueFunc(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label); + this.cycleConsumerNameFunc = + lookupOptionalStrOpValueFunc(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.consumerName.label); + this.rangesFunc = + lookupOptionalStrOpValueFunc(PulsarAdapterUtil.CONSUMER_CONF_CUSTOM_KEY.ranges.label); this.e2eStartTimeSrcParamStrFunc = lookupOptionalStrOpValueFunc( PulsarAdapterUtil.DOC_LEVEL_PARAMS.E2E_STARTING_TIME_SOURCE.label, "none"); this.consumerFunction = (l) -> getConsumer( diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java index 9f250e151..72703c44c 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java @@ -18,20 +18,19 @@ package io.nosqlbench.adapter.pulsar.dispensers; import io.nosqlbench.adapter.pulsar.PulsarSpace; import io.nosqlbench.adapter.pulsar.ops.MessageProducerOp; +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.pulsar.client.api.Producer; -import java.util.Optional; import java.util.function.LongFunction; public class MessageProducerOpDispenser extends PulsarClientOpDispenser { private final static Logger logger = LogManager.getLogger("MessageProducerOpDispenser"); - public static final String PRODUCER_NAME_OP_PARAM = "producer_name"; public static final String MSG_KEY_OP_PARAM = "msg_key"; public static final String MSG_PROP_OP_PARAM = "msg_prop"; public static final String MSG_VALUE_OP_PARAM = "msg_value"; @@ -48,7 +47,8 @@ public class MessageProducerOpDispenser extends PulsarClientOpDispenser { PulsarSpace pulsarSpace) { super(adapter, op, tgtNameFunc, pulsarSpace); - this.cycleProducerNameFunc = lookupOptionalStrOpValueFunc(PRODUCER_NAME_OP_PARAM); + this.cycleProducerNameFunc = + lookupOptionalStrOpValueFunc(PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.producerName.label); this.producerFunc = (l) -> getProducer(tgtNameFunc.apply(l), cycleProducerNameFunc.apply(l)); this.msgKeyFunc = lookupOptionalStrOpValueFunc(MSG_KEY_OP_PARAM); this.msgPropFunc = lookupOptionalStrOpValueFunc(MSG_PROP_OP_PARAM); @@ -65,7 +65,7 @@ public class MessageProducerOpDispenser extends PulsarClientOpDispenser { useTransactFunc.apply(cycle), seqTrackingFunc.apply(cycle), transactSupplierFunc.apply(cycle), - errSimuTypeSetFunc.apply(cycle), + msgSeqErrSimuTypeSetFunc.apply(cycle), producerFunc.apply(cycle), msgKeyFunc.apply(cycle), msgPropFunc.apply(cycle), diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java index f4e0a292a..3d40563e1 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java @@ -43,7 +43,8 @@ public class MessageReaderOpDispenser extends PulsarClientOpDispenser { PulsarSpace pulsarSpace) { super(adapter, op, tgtNameFunc, pulsarSpace); - this.cycleReaderNameFunc = lookupMandtoryStrOpValueFunc("reader_name"); + this.cycleReaderNameFunc = + lookupMandtoryStrOpValueFunc(PulsarAdapterUtil.READER_CONF_STD_KEY.readerName.label); this.msgStartPosStrFunc = lookupOptionalStrOpValueFunc( "start_msg_position", PulsarAdapterUtil.READER_MSG_POSITION_TYPE.earliest.label); this.readerFunc = (l) -> getReader( diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java index cfe9231ef..1de99a097 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java @@ -195,49 +195,49 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser start - ////////////////////////////////////// - // - // Topic name IS mandatory for a producer - // - It must be set at either global level or cycle level - // - If set at both levels, cycle level setting takes precedence - private String getEffectiveProducerTopicName(String cycleTopicName) { - if (!StringUtils.isBlank(cycleTopicName)) { - return cycleTopicName; + // A configuration parameter can be set either at the global level (config.properties file), + // or at the cycle level (.yaml file). + // If set at both levels, cycle level setting takes precedence + private String getEffectiveConValue(String confCategory, String confParamName, String cycleConfValue) { + if (!StringUtils.isBlank(cycleConfValue)) { + return cycleConfValue; } - String globalTopicName = pulsarSpace.getPulsarNBClientConf().getProducerTopicName(); - if (!StringUtils.isBlank(globalTopicName)) { - return globalTopicName; - } + if (PulsarAdapterUtil.isValidConfCategory(confCategory)) { + Map catConfMap = new HashMap<>(); - throw new PulsarAdapterInvalidParamException( - "Effective topic name for a producer can't NOT be empty, " + - "it must be set either as a corresponding adapter Op parameter value or " + - "set in the global Pulsar conf file."); - } + if (StringUtils.equalsIgnoreCase(confCategory, PulsarAdapterUtil.CONF_GATEGORY.Schema.label)) + catConfMap = pulsarSpace.getPulsarNBClientConf().getSchemaConfMapRaw(); + else if (StringUtils.equalsIgnoreCase(confCategory, PulsarAdapterUtil.CONF_GATEGORY.Client.label)) + catConfMap = pulsarSpace.getPulsarNBClientConf().getClientConfMapRaw(); + else if (StringUtils.equalsIgnoreCase(confCategory, PulsarAdapterUtil.CONF_GATEGORY.Producer.label)) + catConfMap = pulsarSpace.getPulsarNBClientConf().getProducerConfMapRaw(); + else if (StringUtils.equalsIgnoreCase(confCategory, PulsarAdapterUtil.CONF_GATEGORY.Consumer.label)) + catConfMap = pulsarSpace.getPulsarNBClientConf().getConsumerConfMapRaw(); + else if (StringUtils.equalsIgnoreCase(confCategory, PulsarAdapterUtil.CONF_GATEGORY.Reader.label)) + catConfMap = pulsarSpace.getPulsarNBClientConf().getReaderConfMapRaw(); - // Producer name is NOT mandatory - // - It can be set at either global level or cycle level - // - If set at both levels, cycle level setting takes precedence - private String getEffectiveProducerName(String cycleProducerName) { - if (!StringUtils.isBlank(cycleProducerName)) { - return cycleProducerName; - } - - String globalProducerName = pulsarSpace.getPulsarNBClientConf().getProducerName(); - if (!StringUtils.isBlank(globalProducerName)) { - return globalProducerName; + String globalConfValue = catConfMap.get(confParamName); + if (!StringUtils.isBlank(globalConfValue)) { + return globalConfValue; + } } return ""; } + public Producer getProducer(String cycleTopicName, String cycleProducerName) { - String topicName = getEffectiveProducerTopicName(cycleTopicName); - String producerName = getEffectiveProducerName(cycleProducerName); + String topicName = getEffectiveConValue( + PulsarAdapterUtil.CONF_GATEGORY.Producer.label, + PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.topicName.label, + cycleTopicName); + + String producerName = getEffectiveConValue( + PulsarAdapterUtil.CONF_GATEGORY.Producer.label, + PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.producerName.label, + cycleProducerName); String producerCacheKey = PulsarAdapterUtil.buildCacheKey(producerName, topicName); Producer producer = pulsarSpace.getProducer(producerCacheKey); @@ -248,7 +248,7 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser producerConf = pulsarSpace.getPulsarNBClientConf().getProducerConfMapTgt(); - // Remove global level settings: "topicName" and "producerName" + // Remove global level settings producerConf.remove(PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.topicName.label); producerConf.remove(PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.producerName.label); @@ -279,33 +279,11 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser start - ////////////////////////////////////// - // - - private String getEffectiveConsumerTopicNameListStr(String cycleTopicNameListStr) { - if (!StringUtils.isBlank(cycleTopicNameListStr)) { - return cycleTopicNameListStr; - } - - String globalTopicNames = pulsarSpace.getPulsarNBClientConf().getConsumerTopicNames(); - if (!StringUtils.isBlank(globalTopicNames)) { - return globalTopicNames; - } - - return ""; - } - private List getEffectiveConsumerTopicNameList(String cycleTopicNameListStr) { - String effectiveTopicNamesStr = getEffectiveConsumerTopicNameListStr(cycleTopicNameListStr); + String effectiveTopicNamesStr = getEffectiveConValue( + PulsarAdapterUtil.CONF_GATEGORY.Consumer.label, + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicNames.label, + cycleTopicNameListStr); String[] names = effectiveTopicNamesStr.split("[;,]"); ArrayList effectiveTopicNameList = new ArrayList<>(); @@ -318,21 +296,12 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser getConsumer(String cycleTopicNameListStr, String cycleTopicPatternStr, String cycleSubscriptionName, @@ -419,15 +343,28 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser topicNameList = getEffectiveConsumerTopicNameList(cycleTopicNameListStr); - String topicPatternStr = getEffectiveConsumerTopicPatternStr(cycleTopicPatternStr); + + String topicPatternStr = getEffectiveConValue( + PulsarAdapterUtil.CONF_GATEGORY.Consumer.label, + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label, + cycleTopicPatternStr); Pattern topicPattern = getEffectiveConsumerTopicPattern(cycleTopicPatternStr); - String subscriptionName = getEffectiveSubscriptionName(cycleSubscriptionName); + + String subscriptionName = getEffectiveConValue( + PulsarAdapterUtil.CONF_GATEGORY.Consumer.label, + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label, + cycleSubscriptionName); + SubscriptionType subscriptionType = getEffectiveSubscriptionType(cycleSubscriptionType); - String consumerName = getEffectiveConsumerName(cycleConsumerName); + + String consumerName = getEffectiveConValue( + PulsarAdapterUtil.CONF_GATEGORY.Consumer.label, + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.consumerName.label, + cycleConsumerName); if ( subscriptionType.equals(SubscriptionType.Exclusive) && (totalThreadNum > 1) ) { throw new PulsarAdapterInvalidParamException( - MessageConsumerOpDispenser.SUBSCRIPTION_TYPE_OP_PARAM, + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label, "creating multiple consumers of \"Exclusive\" subscription type under the same subscription name"); } @@ -456,21 +393,30 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser consumerConf = new HashMap<>(pulsarSpace.getPulsarNBClientConf().getConsumerConfMapTgt()); - - // Remove global level settings: - // - "topicNames", "topicsPattern", "subscriptionName", "subscriptionType", "consumerName" - consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicNames.label); - consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label); - consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label); - consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label); - consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.consumerName.label); - // Remove non-standard consumer configuration properties - consumerConf.remove(PulsarAdapterUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label); + Map consumerConf = + new HashMap<>(pulsarSpace.getPulsarNBClientConf().getConsumerConfMapTgt()); + Map consumerConfToLoad = new HashMap<>(); + consumerConfToLoad.putAll(consumerConf); try { ConsumerBuilder consumerBuilder; + // Remove settings that will be handled outside "loadConf()" + consumerConfToLoad.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicNames.label); + consumerConfToLoad.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label); + consumerConfToLoad.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label); + consumerConfToLoad.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label); + consumerConfToLoad.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.consumerName.label); + + // TODO: It looks like loadConf() method can't handle the following settings properly. + // Do these settings manually for now + // - deadLetterPolicy + // - negativeAckRedeliveryBackoff + // - ackTimeoutRedeliveryBackoff + consumerConfToLoad.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.deadLetterPolicy.label); + consumerConfToLoad.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.negativeAckRedeliveryBackoff.label); + consumerConfToLoad.remove(PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.ackTimeoutRedeliveryBackoff.label); + if (!multiTopicConsumer) { assert (topicNameList.size() == 1); consumerBuilder = pulsarClient.newConsumer(pulsarSpace.getPulsarSchema()); @@ -487,10 +433,24 @@ public abstract class PulsarBaseOpDispenser extends BaseOpDispenser Start - ////////////////////////////////////// - // - - // Topic name IS mandatory for a reader - // - It must be set at either global level or cycle level - // - If set at both levels, cycle level setting takes precedence - private String getEffectiveReaderTopicName(String cycleReaderTopicName) { - if (!StringUtils.isBlank(cycleReaderTopicName)) { - return cycleReaderTopicName; - } - - String globalReaderTopicName = pulsarSpace.getPulsarNBClientConf().getReaderTopicName(); - if (!StringUtils.isBlank(globalReaderTopicName)) { - return globalReaderTopicName; - } - - throw new PulsarAdapterInvalidParamException( - "Effective topic name for a reader can't NOT be empty, " + - "it must be set either as a corresponding adapter Op parameter value or " + - "set in the global Pulsar conf file."); - } - - // Reader name is NOT mandatory - // - It can be set at either global level or cycle level - // - If set at both levels, cycle level setting takes precedence - private String getEffectiveReaderName(String cycleReaderName) { - if (!StringUtils.isBlank(cycleReaderName)) { - return cycleReaderName; - } - - String globalReaderName = pulsarSpace.getPulsarNBClientConf().getReaderName(); - if (!StringUtils.isBlank(globalReaderName)) { - return globalReaderName; - } - - return ""; - } - - private String getEffectiveStartMsgPosStr(String cycleStartMsgPosStr) { - if (!StringUtils.isBlank(cycleStartMsgPosStr)) { - return cycleStartMsgPosStr; - } - - String globalStartMsgPosStr = pulsarSpace.getPulsarNBClientConf().getStartMsgPosStr(); - if (!StringUtils.isBlank(globalStartMsgPosStr)) { - return globalStartMsgPosStr; - } - - return PulsarAdapterUtil.READER_MSG_POSITION_TYPE.latest.label; - } - public Reader getReader(String cycleTopicName, String cycleReaderName, String cycleStartMsgPos) { - String topicName = getEffectiveReaderTopicName(cycleTopicName); - String readerName = getEffectiveReaderName(cycleReaderName); - String startMsgPosStr = getEffectiveStartMsgPosStr(cycleStartMsgPos); + String topicName = getEffectiveConValue( + PulsarAdapterUtil.CONF_GATEGORY.Reader.label, + PulsarAdapterUtil.READER_CONF_STD_KEY.topicName.label, + cycleTopicName); + + String readerName = getEffectiveConValue( + PulsarAdapterUtil.CONF_GATEGORY.Reader.label, + PulsarAdapterUtil.READER_CONF_STD_KEY.readerName.label, + cycleReaderName); + + String startMsgPosStr = getEffectiveConValue( + PulsarAdapterUtil.CONF_GATEGORY.Reader.label, + PulsarAdapterUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label, + cycleStartMsgPos); if (!PulsarAdapterUtil.isValideReaderStartPosition(startMsgPosStr)) { throw new RuntimeException("Reader:: Invalid value for reader start message position!"); } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java index b61dcbf33..1393935b3 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java @@ -49,7 +49,7 @@ public abstract class PulsarClientOpDispenser extends PulsarBaseOpDispenser { protected final LongFunction seqTrackingFunc; protected final LongFunction payloadRttFieldFunc; protected final LongFunction> transactSupplierFunc; - protected final LongFunction> errSimuTypeSetFunc; + protected final LongFunction> msgSeqErrSimuTypeSetFunc; public PulsarClientOpDispenser(DriverAdapter adapter, ParsedOp op, @@ -79,7 +79,7 @@ public abstract class PulsarClientOpDispenser extends PulsarBaseOpDispenser { this.transactSupplierFunc = (l) -> getTransactionSupplier(); - this.errSimuTypeSetFunc = getStaticErrSimuTypeSetOpValueFunc(); + this.msgSeqErrSimuTypeSetFunc = getStaticErrSimuTypeSetOpValueFunc(); } protected Supplier getTransactionSupplier() { @@ -101,16 +101,16 @@ public abstract class PulsarClientOpDispenser extends PulsarBaseOpDispenser { }; } - protected LongFunction> getStaticErrSimuTypeSetOpValueFunc() { - LongFunction> setStringLongFunction; + protected LongFunction> getStaticErrSimuTypeSetOpValueFunc() { + LongFunction> setStringLongFunction; setStringLongFunction = (l) -> parsedOp.getOptionalStaticValue("seqerr_simu", String.class) .filter(Predicate.not(String::isEmpty)) .map(value -> { - Set set = new HashSet<>(); + Set set = new HashSet<>(); if (StringUtils.contains(value,',')) { set = Arrays.stream(value.split(",")) - .map(PulsarAdapterUtil.SEQ_ERROR_SIMU_TYPE::parseSimuType) + .map(PulsarAdapterUtil.MSG_SEQ_ERROR_SIMU_TYPE::parseSimuType) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toCollection(LinkedHashSet::new)); diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java index 39c346473..0a226429c 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java @@ -49,7 +49,7 @@ public class MessageProducerOp extends PulsarClientOp { private final boolean useTransact; private final boolean seqTracking; private final Supplier transactSupplier; - private final Set errSimuTypeSet; + private final Set errSimuTypeSet; private final Producer producer; private final String msgKey; private final String msgPropRawJsonStr; @@ -66,7 +66,7 @@ public class MessageProducerOp extends PulsarClientOp { boolean useTransact, boolean seqTracking, Supplier transactSupplier, - Set errSimuTypeSet, + Set errSimuTypeSet, Producer producer, String msgKey, String msgProp, @@ -142,15 +142,6 @@ public class MessageProducerOp extends PulsarClientOp { int messageSize; SchemaType schemaType = pulsarSchema.getSchemaInfo().getType(); if (pulsarSchema instanceof KeyValueSchema) { - -// // {KEY IN JSON}||{VALUE IN JSON} -// int separator = msgValue.indexOf("}||{"); -// if (separator < 0) { -// throw new IllegalArgumentException("KeyValue payload MUST be in form {KEY IN JSON}||{VALUE IN JSON} (with 2 pipes that separate the KEY part from the VALUE part)"); -// } -// String keyInput = msgValue.substring(0, separator + 1); -// String valueInput = msgValue.substring(separator + 3); - KeyValueSchema keyValueSchema = (KeyValueSchema) pulsarSchema; org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration(); GenericRecord payload = PulsarAvroSchemaUtil.GetGenericRecord_PulsarAvro( diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/MessageSequenceNumberSendingHandler.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/MessageSequenceNumberSendingHandler.java index 9ff2aad5f..21208642a 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/MessageSequenceNumberSendingHandler.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/MessageSequenceNumberSendingHandler.java @@ -18,7 +18,6 @@ package io.nosqlbench.adapter.pulsar.util; */ -import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; import org.apache.commons.lang3.RandomUtils; import java.util.ArrayDeque; @@ -34,16 +33,16 @@ public class MessageSequenceNumberSendingHandler { long number = 1; Queue outOfOrderNumbers; - public long getNextSequenceNumber(Set simulatedErrorTypes) { + public long getNextSequenceNumber(Set simulatedErrorTypes) { return getNextSequenceNumber(simulatedErrorTypes, SIMULATED_ERROR_PROBABILITY_PERCENTAGE); } - long getNextSequenceNumber(Set simulatedErrorTypes, int errorProbabilityPercentage) { + long getNextSequenceNumber(Set simulatedErrorTypes, int errorProbabilityPercentage) { simulateError(simulatedErrorTypes, errorProbabilityPercentage); return nextNumber(); } - private void simulateError(Set simulatedErrorTypes, int errorProbabilityPercentage) { + private void simulateError(Set simulatedErrorTypes, int errorProbabilityPercentage) { if (!simulatedErrorTypes.isEmpty() && shouldSimulateError(errorProbabilityPercentage)) { int selectIndex = 0; int numberOfErrorTypes = simulatedErrorTypes.size(); @@ -51,7 +50,7 @@ public class MessageSequenceNumberSendingHandler { // pick one of the simulated error type randomly selectIndex = RandomUtils.nextInt(0, numberOfErrorTypes); } - PulsarAdapterUtil.SEQ_ERROR_SIMU_TYPE errorType = simulatedErrorTypes.stream() + PulsarAdapterUtil.MSG_SEQ_ERROR_SIMU_TYPE errorType = simulatedErrorTypes.stream() .skip(selectIndex) .findFirst() .get(); diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java index 2150c84bf..15072f82e 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java @@ -83,6 +83,29 @@ public class PulsarAdapterUtil { return Arrays.stream(PULSAR_API_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); } + + /////// + // Valid configuration categories + public enum CONF_GATEGORY { + Schema("schema"), + Client("client"), + Producer("producer"), + Consumer("consumer"), + Reader("reader"); + + public final String label; + + CONF_GATEGORY(String label) { + this.label = label; + } + } + public static boolean isValidConfCategory(String item) { + return Arrays.stream(CONF_GATEGORY.values()).anyMatch(t -> t.label.equals(item)); + } + public static String getValidConfCategoryList() { + return Arrays.stream(CONF_GATEGORY.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + /////// // Valid persistence type public enum PERSISTENT_TYPES { @@ -165,6 +188,27 @@ public class PulsarAdapterUtil { return Arrays.stream(PRODUCER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item)); } + // compressionType + public enum COMPRESSION_TYPE { + NONE("NONE"), + LZ4("LZ4"), + ZLIB("ZLIB"), + ZSTD("ZSTD"), + SNAPPY("SNAPPY"); + + public final String label; + + COMPRESSION_TYPE(String label) { + this.label = label; + } + } + public static boolean isValidCompressionType(String item) { + return Arrays.stream(COMPRESSION_TYPE.values()).anyMatch(t -> t.label.equals(item)); + } + public static String getValidCompressionTypeList() { + return Arrays.stream(COMPRESSION_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + /////// // Standard consumer configuration (activity-level settings) // - https://pulsar.apache.org/docs/en/client-libraries-java/#consumer @@ -189,7 +233,12 @@ public class PulsarAdapterUtil { regexSubscriptionMode("regexSubscriptionMode"), deadLetterPolicy("deadLetterPolicy"), autoUpdatePartitions("autoUpdatePartitions"), - replicateSubscriptionState("replicateSubscriptionState"); + replicateSubscriptionState("replicateSubscriptionState"), + negativeAckRedeliveryBackoff("negativeAckRedeliveryBackoff"), + ackTimeoutRedeliveryBackoff("ackTimeoutRedeliveryBackoff"), + autoAckOldestChunkedMessageOnQueueFull("autoAckOldestChunkedMessageOnQueueFull"), + maxPendingChunkedMessage("maxPendingChunkedMessage"), + expireTimeOfIncompleteChunkedMessageMillis("expireTimeOfIncompleteChunkedMessageMillis"); public final String label; @@ -206,7 +255,8 @@ public class PulsarAdapterUtil { // - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#consumer // - NB Pulsar driver consumer operation specific public enum CONSUMER_CONF_CUSTOM_KEY { - timeout("timeout"); + timeout("timeout"), + ranges("ranges"); public final String label; @@ -218,8 +268,7 @@ public class PulsarAdapterUtil { return Arrays.stream(CONSUMER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item)); } - /////// - // Pulsar subscription type + // subscriptionTyp public enum SUBSCRIPTION_TYPE { Exclusive("Exclusive"), Failover("Failover"), @@ -239,6 +288,43 @@ public class PulsarAdapterUtil { return Arrays.stream(SUBSCRIPTION_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); } + // subscriptionInitialPosition + public enum SUBSCRIPTION_INITIAL_POSITION { + Earliest("Earliest"), + Latest("Latest"); + + public final String label; + + SUBSCRIPTION_INITIAL_POSITION(String label) { + this.label = label; + } + } + public static boolean isValidSubscriptionInitialPosition(String item) { + return Arrays.stream(SUBSCRIPTION_INITIAL_POSITION.values()).anyMatch(t -> t.label.equals(item)); + } + public static String getValidSubscriptionInitialPositionList() { + return Arrays.stream(SUBSCRIPTION_INITIAL_POSITION.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + // regexSubscriptionMode + public enum REGEX_SUBSCRIPTION_MODE { + Persistent("PersistentOnly"), + NonPersistent("NonPersistentOnly"), + All("AllTopics"); + + public final String label; + + REGEX_SUBSCRIPTION_MODE(String label) { + this.label = label; + } + } + public static boolean isValidRegexSubscriptionMode(String item) { + return Arrays.stream(REGEX_SUBSCRIPTION_MODE.values()).anyMatch(t -> t.label.equals(item)); + } + public static String getValidRegexSubscriptionModeList() { + return Arrays.stream(REGEX_SUBSCRIPTION_MODE.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + /////// // Standard reader configuration (activity-level settings) // - https://pulsar.apache.org/docs/en/client-libraries-java/#reader @@ -284,8 +370,7 @@ public class PulsarAdapterUtil { // Valid read positions for a Pulsar reader public enum READER_MSG_POSITION_TYPE { earliest("earliest"), - latest("latest"), - custom("custom"); + latest("latest"); public final String label; @@ -298,22 +383,22 @@ public class PulsarAdapterUtil { } /////// - // Pulsar subscription type - public enum SEQ_ERROR_SIMU_TYPE { + // Message processing sequence error simulation types + public enum MSG_SEQ_ERROR_SIMU_TYPE { OutOfOrder("out_of_order"), MsgLoss("msg_loss"), MsgDup("msg_dup"); public final String label; - SEQ_ERROR_SIMU_TYPE(String label) { + MSG_SEQ_ERROR_SIMU_TYPE(String label) { this.label = label; } - private static final Map MAPPING = new HashMap<>(); + private static final Map MAPPING = new HashMap<>(); static { - for (SEQ_ERROR_SIMU_TYPE simuType : values()) { + for (MSG_SEQ_ERROR_SIMU_TYPE simuType : values()) { MAPPING.put(simuType.label, simuType); MAPPING.put(simuType.label.toLowerCase(), simuType); MAPPING.put(simuType.label.toUpperCase(), simuType); @@ -323,40 +408,15 @@ public class PulsarAdapterUtil { } } - public static Optional parseSimuType(String simuTypeString) { + public static Optional parseSimuType(String simuTypeString) { return Optional.ofNullable(MAPPING.get(simuTypeString.trim())); } } public static boolean isValidSeqErrSimuType(String item) { - return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).anyMatch(t -> t.label.equals(item)); + return Arrays.stream(MSG_SEQ_ERROR_SIMU_TYPE.values()).anyMatch(t -> t.label.equals(item)); } public static String getValidSeqErrSimuTypeList() { - return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); - } - - /////// - // Valid websocket-producer configuration (activity-level settings) - // TODO: to be added - public enum WEBSKT_PRODUCER_CONF_KEY { - ; - - public final String label; - - WEBSKT_PRODUCER_CONF_KEY(String label) { - this.label = label; - } - } - - /////// - // Valid managed-ledger configuration (activity-level settings) - // TODO: to be added - public enum MANAGED_LEDGER_CONF_KEY { - ; - - public final String label; - MANAGED_LEDGER_CONF_KEY(String label) { - this.label = label; - } + return Arrays.stream(MSG_SEQ_ERROR_SIMU_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); } /////// @@ -385,6 +445,10 @@ public class PulsarAdapterUtil { public static Schema getPrimitiveTypeSchema(String typeStr) { Schema schema; + if (StringUtils.isBlank(typeStr)) { + typeStr = "BYTES"; + } + switch (typeStr.toUpperCase()) { case "BOOLEAN": schema = Schema.BOOL; @@ -428,14 +492,12 @@ public class PulsarAdapterUtil { case "LOCAL_DATE_TIME": schema = Schema.LOCAL_DATE_TIME; break; - // Use BYTES as the default schema type if the type string is not specified - case "": case "BYTES": schema = Schema.BYTES; break; // Report an error if non-valid, non-empty schema type string is provided default: - throw new RuntimeException("Invalid Pulsar primitive schema type string : " + typeStr); + throw new PulsarAdapterInvalidParamException("Invalid Pulsar primitive schema type string : " + typeStr); } return schema; @@ -444,15 +506,12 @@ public class PulsarAdapterUtil { /////// // Complex strut type: Avro or Json public static boolean isAvroSchemaTypeStr(String typeStr) { - return typeStr.equalsIgnoreCase("AVRO"); - } - public static boolean isKeyValueTypeStr(String typeStr) { - return typeStr.equalsIgnoreCase("KEY_VALUE"); + return (StringUtils.isNotBlank(typeStr) && typeStr.equalsIgnoreCase("AVRO")); } // automatic decode the type from the Registry public static boolean isAutoConsumeSchemaTypeStr(String typeStr) { - return typeStr.equalsIgnoreCase("AUTO_CONSUME"); + return (StringUtils.isNotBlank(typeStr) && typeStr.equalsIgnoreCase("AUTO_CONSUME")); } public static Schema getAvroSchema(String typeStr, String definitionStr) { String schemaDefinitionStr = definitionStr; diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarClientConf.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarClientConf.java index 9fd56be8f..4af681509 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarClientConf.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarClientConf.java @@ -24,6 +24,7 @@ import org.apache.commons.configuration2.builder.fluent.Parameters; import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -39,11 +40,6 @@ public class PulsarClientConf { private String canonicalFilePath = ""; - public static final String SCHEMA_CONF_PREFIX = "schema"; - public static final String CLIENT_CONF_PREFIX = "client"; - public static final String PRODUCER_CONF_PREFIX = "producer"; - public static final String CONSUMER_CONF_PREFIX = "consumer"; - public static final String READER_CONF_PREFIX = "reader"; private final Map schemaConfMapRaw = new HashMap<>(); private final Map clientConfMapRaw = new HashMap<>(); @@ -75,8 +71,8 @@ public class PulsarClientConf { ////////////////// // Convert the raw configuration map () to the required map () - producerConfMapTgt.putAll(PulsarConfConverter.convertRawProducerConf(producerConfMapRaw)); - consumerConfMapTgt.putAll(PulsarConfConverter.convertRawConsumerConf(consumerConfMapRaw)); + producerConfMapTgt.putAll(PulsarConfConverter.convertStdRawProducerConf(producerConfMapRaw)); + consumerConfMapTgt.putAll(PulsarConfConverter.convertStdRawConsumerConf(consumerConfMapRaw)); // TODO: Reader API is not disabled at the moment. Revisit when needed } @@ -103,28 +99,28 @@ public class PulsarClientConf { if (!StringUtils.isBlank(confVal)) { // Get schema specific configuration settings, removing "schema." prefix - if (StringUtils.startsWith(confKey, SCHEMA_CONF_PREFIX)) { - schemaConfMapRaw.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), confVal); + if (StringUtils.startsWith(confKey, PulsarAdapterUtil.CONF_GATEGORY.Schema.label)) { + schemaConfMapRaw.put(confKey.substring(PulsarAdapterUtil.CONF_GATEGORY.Schema.label.length() + 1), confVal); } // Get client connection specific configuration settings, removing "client." prefix // <<< https://pulsar.apache.org/docs/reference-configuration/#client >>> - else if (StringUtils.startsWith(confKey, CLIENT_CONF_PREFIX)) { - clientConfMapRaw.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), confVal); + else if (StringUtils.startsWith(confKey, PulsarAdapterUtil.CONF_GATEGORY.Client.label)) { + clientConfMapRaw.put(confKey.substring(PulsarAdapterUtil.CONF_GATEGORY.Client.label.length() + 1), confVal); } // Get producer specific configuration settings, removing "producer." prefix // <<< https://pulsar.apache.org/docs/client-libraries-java/#configure-producer >>> - else if (StringUtils.startsWith(confKey, PRODUCER_CONF_PREFIX)) { - producerConfMapRaw.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), confVal); + else if (StringUtils.startsWith(confKey, PulsarAdapterUtil.CONF_GATEGORY.Producer.label)) { + producerConfMapRaw.put(confKey.substring(PulsarAdapterUtil.CONF_GATEGORY.Producer.label.length() + 1), confVal); } // Get consumer specific configuration settings, removing "consumer." prefix // <<< https://pulsar.apache.org/docs/client-libraries-java/#configure-consumer >>> - else if (StringUtils.startsWith(confKey, CONSUMER_CONF_PREFIX)) { - consumerConfMapRaw.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), confVal); + else if (StringUtils.startsWith(confKey, PulsarAdapterUtil.CONF_GATEGORY.Consumer.label)) { + consumerConfMapRaw.put(confKey.substring(PulsarAdapterUtil.CONF_GATEGORY.Consumer.label.length() + 1), confVal); } // Get reader specific configuration settings, removing "reader." prefix // <<< https://pulsar.apache.org/docs/2.10.x/client-libraries-java/#configure-reader >>> - else if (StringUtils.startsWith(confKey, READER_CONF_PREFIX)) { - readerConfMapRaw.put(confKey.substring(READER_CONF_PREFIX.length() + 1), confVal); + else if (StringUtils.startsWith(confKey, PulsarAdapterUtil.CONF_GATEGORY.Reader.label)) { + readerConfMapRaw.put(confKey.substring(PulsarAdapterUtil.CONF_GATEGORY.Reader.label.length() + 1), confVal); } } } @@ -161,163 +157,111 @@ public class PulsarClientConf { ////////////////// // Get Schema related config public boolean hasSchemaConfKey(String key) { - if (key.contains(SCHEMA_CONF_PREFIX)) - return schemaConfMapRaw.containsKey(key.substring(SCHEMA_CONF_PREFIX.length() + 1)); + if (key.contains(PulsarAdapterUtil.CONF_GATEGORY.Schema.label)) + return schemaConfMapRaw.containsKey(key.substring(PulsarAdapterUtil.CONF_GATEGORY.Schema.label.length() + 1)); else return schemaConfMapRaw.containsKey(key); } - public String getSchemaConfValue(String key) { - if (key.contains(SCHEMA_CONF_PREFIX)) - return schemaConfMapRaw.get(key.substring(SCHEMA_CONF_PREFIX.length()+1)); - else - return schemaConfMapRaw.get(key); + public String getSchemaConfValueRaw(String key) { + if (hasSchemaConfKey(key)) { + if (key.contains(PulsarAdapterUtil.CONF_GATEGORY.Schema.label)) + return schemaConfMapRaw.get(key.substring(PulsarAdapterUtil.CONF_GATEGORY.Schema.label.length() + 1)); + else + return schemaConfMapRaw.get(key); + } + else { + return ""; + } } ////////////////// // Get Pulsar client related config - public String getClientConfValue(String key) { - if (key.contains(CLIENT_CONF_PREFIX)) - return clientConfMapRaw.get(key.substring(CLIENT_CONF_PREFIX.length()+1)); + public boolean hasClientConfKey(String key) { + if (key.contains(PulsarAdapterUtil.CONF_GATEGORY.Client.label)) + return clientConfMapRaw.containsKey(key.substring(PulsarAdapterUtil.CONF_GATEGORY.Client.label.length() + 1)); else - return clientConfMapRaw.get(key); + return clientConfMapRaw.containsKey(key); + } + public String getClientConfValueRaw(String key) { + if (hasClientConfKey(key)) { + if (key.contains(PulsarAdapterUtil.CONF_GATEGORY.Client.label)) + return clientConfMapRaw.get(key.substring(PulsarAdapterUtil.CONF_GATEGORY.Client.label.length() + 1)); + else + return clientConfMapRaw.get(key); + } + else { + return ""; + } } ////////////////// // Get Pulsar producer related config - public Object getProducerConfValue(String key) { - if (key.contains(PRODUCER_CONF_PREFIX)) - return producerConfMapTgt.get(key.substring(PRODUCER_CONF_PREFIX.length()+1)); + public boolean hasProducerConfKey(String key) { + if (key.contains(PulsarAdapterUtil.CONF_GATEGORY.Producer.label)) + return producerConfMapRaw.containsKey(key.substring(PulsarAdapterUtil.CONF_GATEGORY.Producer.label.length() + 1)); else - return producerConfMapTgt.get(key); + return producerConfMapRaw.containsKey(key); } - // other producer helper functions ... - public String getProducerName() { - Object confValue = getProducerConfValue( - "producer." + PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.producerName.label); - if (confValue == null) + public String getProducerConfValueRaw(String key) { + if (hasProducerConfKey(key)) { + if (key.contains(PulsarAdapterUtil.CONF_GATEGORY.Producer.label)) + return producerConfMapRaw.get(key.substring(PulsarAdapterUtil.CONF_GATEGORY.Producer.label.length()+1)); + else + return producerConfMapRaw.get(key); + } + else { return ""; - else - return confValue.toString(); - } - public String getProducerTopicName() { - Object confValue = getProducerConfValue( - "producer." + PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.topicName); - if (confValue == null) - return ""; - else - return confValue.toString(); + } } ////////////////// // Get Pulsar consumer related config - public String getConsumerConfValue(String key) { - if (key.contains(CONSUMER_CONF_PREFIX)) - return consumerConfMapRaw.get(key.substring(CONSUMER_CONF_PREFIX.length() + 1)); + public boolean hasConsumerConfKey(String key) { + if (key.contains(PulsarAdapterUtil.CONF_GATEGORY.Consumer.label)) + return consumerConfMapRaw.containsKey(key.substring(PulsarAdapterUtil.CONF_GATEGORY.Consumer.label.length() + 1)); else - return consumerConfMapRaw.get(key); + return consumerConfMapRaw.containsKey(key); } - // Other consumer helper functions ... - public String getConsumerTopicNames() { - String confValue = getConsumerConfValue( - "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicNames.label); - if (confValue == null) + public String getConsumerConfValueRaw(String key) { + if (hasConsumerConfKey(key)) { + if (key.contains(PulsarAdapterUtil.CONF_GATEGORY.Consumer.label)) + return consumerConfMapRaw.get(key.substring(PulsarAdapterUtil.CONF_GATEGORY.Consumer.label.length() + 1)); + else + return consumerConfMapRaw.get(key); + } + else { return ""; - else - return confValue.toString(); - } - public String getConsumerTopicPattern() { - Object confValue = getConsumerConfValue( - "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - public String getConsumerSubscriptionName() { - Object confValue = getConsumerConfValue( - "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - public String getConsumerSubscriptionType() { - Object confValue = getConsumerConfValue( - "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - public String getConsumerName() { - Object confValue = getConsumerConfValue( - "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.consumerName.label); - if (confValue == null) - return ""; - else - return confValue.toString(); + } } // NOTE: Below are not a standard Pulsar consumer configuration parameter as // listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer" // They're custom-made configuration properties for NB pulsar driver consumer. public int getConsumerTimeoutSeconds() { - Object confValue = getConsumerConfValue( + String confValue = getConsumerConfValueRaw( "consumer." + PulsarAdapterUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label); - if (confValue == null) - return -1; // infinite - else - return Integer.parseInt(confValue.toString()); + return NumberUtils.toInt(confValue, -1); } ////////////////// // Get Pulsar reader related config public boolean hasReaderConfKey(String key) { - if (key.contains(READER_CONF_PREFIX)) - return readerConfMapRaw.containsKey(key.substring(READER_CONF_PREFIX.length() + 1)); + if (key.contains(PulsarAdapterUtil.CONF_GATEGORY.Reader.label)) + return readerConfMapRaw.containsKey(key.substring(PulsarAdapterUtil.CONF_GATEGORY.Reader.label.length() + 1)); else return readerConfMapRaw.containsKey(key); } - public Object getReaderConfValue(String key) { - if (key.contains(READER_CONF_PREFIX)) - return readerConfMapRaw.get(key.substring(READER_CONF_PREFIX.length() + 1)); - else - return readerConfMapRaw.get(key); - } - public void setReaderConfValue(String key, String value) { - if (key.contains(READER_CONF_PREFIX)) - readerConfMapRaw.put(key.substring(READER_CONF_PREFIX.length() + 1), value); - else - readerConfMapRaw.put(key, value); - } - // Other reader helper functions ... - public String getReaderTopicName() { - Object confValue = getReaderConfValue( - "reader." + PulsarAdapterUtil.READER_CONF_STD_KEY.topicName.label); - if (confValue == null) + public String getReaderConfValueRaw(String key) { + if (hasReaderConfKey(key)) { + if (key.contains(PulsarAdapterUtil.CONF_GATEGORY.Reader.label)) + return readerConfMapRaw.get(key.substring(PulsarAdapterUtil.CONF_GATEGORY.Reader.label.length() + 1)); + else + return readerConfMapRaw.get(key); + } + else { return ""; - else - return confValue.toString(); - } - public String getReaderName() { - Object confValue = getReaderConfValue( - "reader." + PulsarAdapterUtil.READER_CONF_STD_KEY.readerName.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - // NOTE: Below are not a standard Pulsar reader configuration parameter as - // listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#reader" - // They're custom-made configuration properties for NB pulsar driver reader. - public String getStartMsgPosStr() { - Object confValue = getReaderConfValue( - "reader." + PulsarAdapterUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label); - if (confValue == null) - return ""; - else - return confValue.toString(); + } } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarConfConverter.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarConfConverter.java index 8024cb39e..e187e7b82 100644 --- a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarConfConverter.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarConfConverter.java @@ -20,9 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterInvalidParamException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; -import org.apache.pulsar.client.api.CompressionType; -import org.apache.pulsar.client.api.DeadLetterPolicy; -import org.apache.pulsar.client.api.RedeliveryBackoff; +import org.apache.pulsar.client.api.*; import org.apache.pulsar.client.impl.MultiplierRedeliveryBackoff; import java.util.ArrayList; @@ -33,7 +31,7 @@ import java.util.Map; public class PulsarConfConverter { // <<< https://pulsar.apache.org/docs/client-libraries-java/#configure-producer >>> - private final static Map validPulsarProducerConfKeyTypeMap = Map.ofEntries( + private final static Map validStdProducerConfKeyTypeMap = Map.ofEntries( Map.entry("topicName", "String"), Map.entry("producerName","String"), Map.entry("sendTimeoutMs","long"), @@ -50,9 +48,9 @@ public class PulsarConfConverter { Map.entry("compressionType","CompressionType"), Map.entry("initialSubscriptionName","string") ); - public static Map convertRawProducerConf(Map pulsarProducerConfMapRaw) { + public static Map convertStdRawProducerConf(Map pulsarProducerConfMapRaw) { Map producerConfObjMap = new HashMap<>(); - setConfObjMapForPrimitives(producerConfObjMap, pulsarProducerConfMapRaw, validPulsarProducerConfKeyTypeMap); + setConfObjMapForPrimitives(producerConfObjMap, pulsarProducerConfMapRaw, validStdProducerConfKeyTypeMap); /** * Non-primitive type processing for Pulsar producer configuration items @@ -63,31 +61,35 @@ public class PulsarConfConverter { // * hashingScheme // * cryptoFailureAction - // "compressionType" has value type "CompressionType" + // "compressionType" // - expecting the following values: 'LZ4', 'ZLIB', 'ZSTD', 'SNAPPY' - String confKeyName = "compressionType"; + String confKeyName = PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.compressionType.label; String confVal = pulsarProducerConfMapRaw.get(confKeyName); - String expectedVal = "(LZ4|ZLIB|ZSTD|SNAPPY)"; + String expectedVal = PulsarAdapterUtil.getValidCompressionTypeList(); - if (StringUtils.isNotBlank(confVal)) { - if (StringUtils.equalsAnyIgnoreCase(confVal, "LZ4", "ZLIB", "ZSTD", "SNAPPY")) { + if ( StringUtils.isNotBlank(confVal) ) { + if (StringUtils.containsIgnoreCase(expectedVal, confVal)) { CompressionType compressionType = CompressionType.NONE; switch (StringUtils.upperCase(confVal)) { case "LZ4": compressionType = CompressionType.LZ4; + break; case "ZLIB": compressionType = CompressionType.ZLIB; + break; case "ZSTD": compressionType = CompressionType.ZSTD; + break; case "SNAPPY": compressionType = CompressionType.SNAPPY; + break; } producerConfObjMap.put(confKeyName, compressionType); } else { throw new PulsarAdapterInvalidParamException( - getInvalidConfValStr(confKeyName, confVal, "producer", expectedVal)); + getInvalidConfValStr(confKeyName, confVal, PulsarAdapterUtil.CONF_GATEGORY.Producer.label, expectedVal)); } } @@ -96,7 +98,7 @@ public class PulsarConfConverter { // https://pulsar.apache.org/docs/client-libraries-java/#configure-consumer - private final static Map validPulsarConsumerConfKeyTypeMap = Map.ofEntries( + private final static Map validStdConsumerConfKeyTypeMap = Map.ofEntries( Map.entry("topicNames", "Set"), Map.entry("topicsPattern","Pattern"), Map.entry("subscriptionName","String"), @@ -124,30 +126,27 @@ public class PulsarConfConverter { Map.entry("maxPendingChunkedMessage", "int"), Map.entry("expireTimeOfIncompleteChunkedMessageMillis", "long") ); - public static Map convertRawConsumerConf(Map pulsarConsumerConfMapRaw) { + public static Map convertStdRawConsumerConf(Map pulsarConsumerConfMapRaw) { Map consumerConfObjMap = new HashMap<>(); - setConfObjMapForPrimitives(consumerConfObjMap, pulsarConsumerConfMapRaw, validPulsarConsumerConfKeyTypeMap); + setConfObjMapForPrimitives(consumerConfObjMap, pulsarConsumerConfMapRaw, validStdConsumerConfKeyTypeMap); /** * Non-primitive type processing for Pulsar consumer configuration items */ // NOTE: The following non-primitive type configuration items are excluded since - // they'll be handled in PulsarBasedOpDispenser.getConsumer() method directly - // * topicNames - // * topicPattern - // * subscriptionType - + // they'll be handled in PulsarBasedOpDispenser.getConsumer() method directly + // * topicNames + // * topicPattern + // * subscriptionType // TODO: Skip the following Pulsar configuration items for now because they're not really // needed in the NB S4J testing right now. Add the support for them when needed. - // * subscriptionInitialPosition - // * regexSubscriptionMode // * cryptoFailureAction // "properties" has value type "SortedMap" // - expecting the value string has the format: a JSON string that includes a set of key/value pairs - String confKeyName = "properties"; + String confKeyName = PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.properties.label; String confVal = pulsarConsumerConfMapRaw.get(confKeyName); String expectedVal = "{\"property1\":\"value1\", \"property2\":\"value2\"}, ..."; @@ -164,17 +163,58 @@ public class PulsarConfConverter { } catch (Exception e) { throw new PulsarAdapterInvalidParamException( - getInvalidConfValStr(confKeyName, confVal, "consumer", expectedVal)); + getInvalidConfValStr(confKeyName, confVal, PulsarAdapterUtil.CONF_GATEGORY.Consumer.label, expectedVal)); + } + } + + // "subscriptionInitialPosition" + // - expecting the following values: 'Latest' (default), + confKeyName = PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionInitialPosition.label; + confVal = pulsarConsumerConfMapRaw.get(confKeyName); + expectedVal = PulsarAdapterUtil.getValidSubscriptionInitialPositionList(); + + if (StringUtils.isNotBlank(confVal)) { + try { + SubscriptionInitialPosition subInitPos = SubscriptionInitialPosition.Latest; + if (!StringUtils.isBlank(confVal)) { + subInitPos = SubscriptionInitialPosition.valueOf(confVal); + } + consumerConfObjMap.put(confKeyName, subInitPos); + + } catch (Exception e) { + throw new PulsarAdapterInvalidParamException( + getInvalidConfValStr(confKeyName, confVal, PulsarAdapterUtil.CONF_GATEGORY.Consumer.label, expectedVal)); + } + } + + // "regexSubscriptionMode" + // - expecting the following values: 'PersistentOnly' (default), 'NonPersistentOnly', and 'AllTopics' + confKeyName = PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.regexSubscriptionMode.label; + confVal = pulsarConsumerConfMapRaw.get(confKeyName); + expectedVal = PulsarAdapterUtil.getValidRegexSubscriptionModeList(); + + if (StringUtils.isNotBlank(confVal)) { + try { + RegexSubscriptionMode regexSubscriptionMode = RegexSubscriptionMode.PersistentOnly; + if (!StringUtils.isBlank(confVal)) { + regexSubscriptionMode = RegexSubscriptionMode.valueOf(confVal); + } + consumerConfObjMap.put(confKeyName, regexSubscriptionMode); + + } catch (Exception e) { + throw new PulsarAdapterInvalidParamException( + getInvalidConfValStr(confKeyName, confVal, PulsarAdapterUtil.CONF_GATEGORY.Consumer.label, expectedVal)); } } // "deadLetterPolicy" // - expecting the value is a JSON string has the format: // {"maxRedeliverCount":"","deadLetterTopic":"","initialSubscriptionName":""} - confKeyName = "deadLetterPolicy"; + confKeyName = PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.deadLetterPolicy.label; confVal = pulsarConsumerConfMapRaw.get(confKeyName); expectedVal = "{" + - "\"maxRedeliverCount\":\"\"," + + "\"maxRedeliverCount\":\"\" (mandatory)," + + "\"retryLetterTopic\":\"\"," + "\"deadLetterTopic\":\"\"," + "\"initialSubscriptionName\":\"\"}"; @@ -188,15 +228,15 @@ public class PulsarConfConverter { // The JSON key must be one of "maxRedeliverCount", "deadLetterTopic", "initialSubscriptionName" for (String key : dlqPolicyMap.keySet()) { - if (!StringUtils.equalsAnyIgnoreCase(key, - "maxRedeliverCount", "deadLetterTopic", "initialSubscriptionName")) { + if (!StringUtils.equalsAnyIgnoreCase(key, "maxRedeliverCount", + "retryLetterTopic", "deadLetterTopic", "initialSubscriptionName")) { valid = false; break; } } // DLQ.maxRedeliverCount is mandatory - if (valid && !dlqPolicyMap.containsKey("maxRedeliverCount")) { + if ( valid && !dlqPolicyMap.containsKey("maxRedeliverCount")) { valid = false; } @@ -206,28 +246,42 @@ public class PulsarConfConverter { } if (valid) { - DeadLetterPolicy deadLetterPolicy = DeadLetterPolicy.builder() - .maxRedeliverCount(NumberUtils.toInt(maxRedeliverCountStr)) - .deadLetterTopic(dlqPolicyMap.get("deadLetterTopic")) - .initialSubscriptionName(dlqPolicyMap.get("initialSubscriptionName")) - .build(); + DeadLetterPolicy.DeadLetterPolicyBuilder builder = DeadLetterPolicy.builder() + .maxRedeliverCount(NumberUtils.toInt(maxRedeliverCountStr)); + String retryTopicName = dlqPolicyMap.get("retryLetterTopic"); + String dlqTopicName = dlqPolicyMap.get("deadLetterTopic"); + String initialSubName = dlqPolicyMap.get("initialSubscriptionName"); + + if (StringUtils.isNotBlank(retryTopicName)) + builder.retryLetterTopic(retryTopicName); + + if (StringUtils.isNotBlank(dlqTopicName)) + builder.deadLetterTopic(dlqTopicName); + + if (StringUtils.isNotBlank(initialSubName)) + builder.initialSubscriptionName(initialSubName); + + DeadLetterPolicy deadLetterPolicy = builder.build(); consumerConfObjMap.put(confKeyName, deadLetterPolicy); } else { throw new PulsarAdapterInvalidParamException( - getInvalidConfValStr(confKeyName, confVal, "consumer", expectedVal)); + getInvalidConfValStr(confKeyName, confVal, PulsarAdapterUtil.CONF_GATEGORY.Consumer.label, expectedVal)); } } } catch (Exception e) { throw new PulsarAdapterInvalidParamException( - getInvalidConfValStr(confKeyName, confVal, "consumer", expectedVal)); + getInvalidConfValStr(confKeyName, confVal, PulsarAdapterUtil.CONF_GATEGORY.Consumer.label, expectedVal)); } } // "negativeAckRedeliveryBackoff" or "ackTimeoutRedeliveryBackoff" // - expecting the value is a JSON string has the format: // {"minDelayMs":"", "maxDelayMs":"", "multiplier":""} - String[] redeliveryBackoffConfigSet = {"negativeAckRedeliveryBackoff", "ackTimeoutRedeliveryBackoff"}; + String[] redeliveryBackoffConfigSet = { + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.negativeAckRedeliveryBackoff.label, + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.ackTimeoutRedeliveryBackoff.label + }; expectedVal = "{" + "\"minDelayMs\":\"\"," + "\"maxDelayMs\":\"\"," + @@ -274,24 +328,31 @@ public class PulsarConfConverter { } else { throw new PulsarAdapterInvalidParamException( - getInvalidConfValStr(confKey, confVal, "consumer", expectedVal)); + getInvalidConfValStr(confKey, confVal, PulsarAdapterUtil.CONF_GATEGORY.Consumer.label, expectedVal)); } } } catch (Exception e) { throw new PulsarAdapterInvalidParamException( - getInvalidConfValStr(confKey, confVal, "consumer", expectedVal)); + getInvalidConfValStr(confKey, confVal, PulsarAdapterUtil.CONF_GATEGORY.Consumer.label, expectedVal)); } } } + // Remove non-standard consumer configuration items + for (String confKey : consumerConfObjMap.keySet()) { + if (PulsarAdapterUtil.isCustomConsumerConfItem(confKey)) { + consumerConfObjMap.remove(confKey); + } + } + return consumerConfObjMap; } // Utility function // - get configuration key names by the value type - private static List getConfKeyNameByValueType(Map confKeyTypeMap, String tgtValType) { + private static List getStdConfKeyNameByValueType(Map confKeyTypeMap, String tgtValType) { ArrayList confKeyNames = new ArrayList<>(); for (Map.Entry entry: confKeyTypeMap.entrySet()) { @@ -310,10 +371,10 @@ public class PulsarConfConverter { Map srcConfMapRaw, Map validConfKeyTypeMap) { - List confKeyList = new ArrayList<>(); + List confKeyList; // All configuration items with "String" as the value type - confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "String"); + confKeyList = getStdConfKeyNameByValueType(validConfKeyTypeMap, "String"); for (String confKey : confKeyList) { if (srcConfMapRaw.containsKey(confKey)) { String confVal = srcConfMapRaw.get(confKey); @@ -324,7 +385,7 @@ public class PulsarConfConverter { } // All configuration items with "long" as the value type - confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "long"); + confKeyList = getStdConfKeyNameByValueType(validConfKeyTypeMap, "long"); for (String confKey : confKeyList) { if (srcConfMapRaw.containsKey(confKey)) { String confVal = srcConfMapRaw.get(confKey); @@ -335,7 +396,7 @@ public class PulsarConfConverter { } // All configuration items with "int" as the value type - confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "int"); + confKeyList = getStdConfKeyNameByValueType(validConfKeyTypeMap, "int"); for (String confKey : confKeyList) { if (srcConfMapRaw.containsKey(confKey)) { String confVal = srcConfMapRaw.get(confKey); @@ -346,7 +407,7 @@ public class PulsarConfConverter { } // All configuration items with "boolean" as the value type - confKeyList = getConfKeyNameByValueType(validConfKeyTypeMap, "boolean"); + confKeyList = getStdConfKeyNameByValueType(validConfKeyTypeMap, "boolean"); for (String confKey : confKeyList) { if (srcConfMapRaw.containsKey(confKey)) { String confVal = srcConfMapRaw.get(confKey); diff --git a/adapter-pulsar/src/main/resources/config.properties b/adapter-pulsar/src/main/resources/config.properties index 9cc804c1f..47146c100 100644 --- a/adapter-pulsar/src/main/resources/config.properties +++ b/adapter-pulsar/src/main/resources/config.properties @@ -22,7 +22,6 @@ client.connectionTimeoutMs=5000 client.authPluginClassName=org.apache.pulsar.client.impl.auth.AuthenticationToken # Cluster admin client.authParams= -client.tlsAllowInsecureConnection=true ### Producer related configurations (global) - producer.xxx @@ -35,18 +34,12 @@ producer.blockIfQueueFull=true ### Consumer related configurations (global) - consumer.xxx # http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer -consumer.topicNames= -consumer.topicsPattern= -consumer.subscriptionName= -consumer.subscriptionType= -consumer.consumerName= -consumer.receiverQueueSize= +consumer.subscriptionInitialPosition=Earliest +consumer.ackTimeoutMillis=10000 +consumer.regexSubscriptionMode=AllTopics +consumer.deadLetterPolicy={"maxRedeliverCount":"5","retryLetterTopic":"public/default/retry","deadLetterTopic":"public/default/dlq","initialSubscriptionName":"dlq-sub"} +consumer.ackTimeoutRedeliveryBackoff={"minDelayMs":"10","maxDelayMs":"20","multiplier":"1.2"} ### Reader related configurations (global) - reader.xxx # https://pulsar.apache.org/docs/en/client-libraries-java/#reader -# - valid Pos: earliest, latest, custom::file://// -reader.topicName= -reader.receiverQueueSize= -reader.readerName= -reader.startMessagePos=earliest diff --git a/adapter-pulsar/src/main/resources/msg_proc_kvraw.yaml b/adapter-pulsar/src/main/resources/msg_proc_kvraw.yaml index 4aea8f44a..b535eaa86 100644 --- a/adapter-pulsar/src/main/resources/msg_proc_kvraw.yaml +++ b/adapter-pulsar/src/main/resources/msg_proc_kvraw.yaml @@ -30,5 +30,5 @@ blocks: ops: op1: MessageConsume: "tnt0/ns0/tp0" - subscription_name: "mynbsub" -# subscription_type: "shared" + subscriptionName: "mynbsub" + subscriptionType: "Shared" diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java index 7bfc2d60a..d3a592e76 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/BaseOpDispenser.java @@ -39,7 +39,7 @@ public abstract class BaseOpDispenser implements OpDispenser private final String name; protected final DriverAdapter adapter; - protected boolean instrument; + private boolean instrument; private Histogram resultSizeHistogram; private Timer successTimer; private Timer errorTimer; diff --git a/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java b/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java index bff438d72..31008d972 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java +++ b/nb-api/src/main/java/io/nosqlbench/api/engine/metrics/ActivityMetrics.java @@ -83,15 +83,6 @@ public class ActivityMetrics { return metric; } - private static Metric register(String fullMetricName, MetricProvider metricProvider) { - Metric metric = get().getMetrics().get(fullMetricName); - if (metric == null) { - metric = metricProvider.getMetric(); - return get().register(fullMetricName, metric); - } - return metric; - } - private static Metric register(ScriptContext context, String name, MetricProvider metricProvider) { Metric metric = get().getMetrics().get(name); if (metric == null) { diff --git a/pom.xml b/pom.xml index c8dd8eeff..7eac71c2c 100644 --- a/pom.xml +++ b/pom.xml @@ -92,7 +92,6 @@ driver-jmx driver-jdbc driver-cockroachdb -