diff --git a/.run/NBCLI web foreground dryrun.run.xml b/.run/NBCLI web foreground dryrun.run.xml deleted file mode 100644 index 4180a9a89..000000000 --- a/.run/NBCLI web foreground dryrun.run.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - \ No newline at end of file diff --git a/driver-jms/pom.xml b/adapter-pulsar/pom.xml similarity index 58% rename from driver-jms/pom.xml rename to adapter-pulsar/pom.xml index c3cbcc803..2428c54fe 100644 --- a/driver-jms/pom.xml +++ b/adapter-pulsar/pom.xml @@ -17,61 +17,49 @@ 4.0.0 + adapter-pulsar + jar + mvn-defaults io.nosqlbench - 4.17.22-SNAPSHOT + 4.17.31-SNAPSHOT ../mvn-defaults - driver-jms - jar ${project.artifactId} - - A JMS driver for nosqlbench. This provides the ability to inject synthetic data - into a pulsar system via JMS 2.0 compatibile APIs. - - NOTE: this is JMS compatible driver from DataStax that allows using a Pulsar cluster - as the potential JMS Destination + A Pulsar driver for nosqlbench. This provides the ability to inject synthetic data + into a pulsar system. - - - - - - - - - - - - - - + + 2.10.1 + - io.nosqlbench engine-api - 4.17.22-SNAPSHOT + 4.17.31-SNAPSHOT - - org.apache.commons - commons-lang3 - 3.12.0 + io.nosqlbench + adapters-api + 4.17.31-SNAPSHOT - - org.projectlombok - lombok - 1.18.24 - provided + org.apache.pulsar + pulsar-client + ${pulsar.version} + + + + org.apache.pulsar + pulsar-client-admin + ${pulsar.version} @@ -88,13 +76,19 @@ 2.8.0 - + - com.datastax.oss - pulsar-jms - 2.4.11 + org.apache.avro + avro + 1.11.1 + + + org.apache.commons + commons-lang3 + 3.12.0 + diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java new file mode 100644 index 000000000..b121ca5e3 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar; + +import io.nosqlbench.adapter.pulsar.ops.PulsarOp; +import io.nosqlbench.engine.api.activityimpl.OpMapper; +import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.nb.annotations.Maturity; +import io.nosqlbench.nb.annotations.Service; +import io.nosqlbench.api.config.standard.NBConfigModel; +import io.nosqlbench.api.config.standard.NBConfiguration; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.Function; + +@Service(value = DriverAdapter.class, selector = "pulsar-nb5", maturity = Maturity.Experimental) +public class PulsarDriverAdapter extends BaseDriverAdapter { + + private final static Logger logger = LogManager.getLogger(PulsarDriverAdapter.class); + + @Override + public OpMapper getOpMapper() { + DriverSpaceCache spaceCache = getSpaceCache(); + NBConfiguration adapterConfig = getConfiguration(); + return new PulsarOpMapper(this, adapterConfig, spaceCache); + } + + @Override + public Function getSpaceInitializer(NBConfiguration cfg) { + return (s) -> new PulsarSpace(s, cfg); + } + + @Override + public NBConfigModel getConfigModel() { + return super.getConfigModel().add(PulsarSpace.getConfigModel()); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java new file mode 100644 index 000000000..babcb0349 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar; + +import io.nosqlbench.adapter.pulsar.dispensers.*; +import io.nosqlbench.adapter.pulsar.ops.PulsarOp; +import io.nosqlbench.api.config.standard.NBConfiguration; +import io.nosqlbench.engine.api.activityimpl.OpDispenser; +import io.nosqlbench.engine.api.activityimpl.OpMapper; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.engine.api.templating.ParsedOp; +import io.nosqlbench.engine.api.templating.TypeAndTarget; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +public class PulsarOpMapper implements OpMapper { + + private final static Logger logger = LogManager.getLogger(PulsarOpMapper.class); + + private final NBConfiguration cfg; + private final DriverSpaceCache cache; + private final DriverAdapter adapter; + + public PulsarOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache cache) { + this.cfg = cfg; + this.cache = cache; + this.adapter = adapter; + } + + @Override + public OpDispenser apply(ParsedOp op) { + String space = op.getStaticConfigOr("space", "default"); + + PulsarClient pulsarClient = cache.get(space).getPulsarClient(); + PulsarAdmin pulsarAdmin = cache.get(space).getPulsarAdmin(); + Schema pulsarSchema = cache.get(space).getPulsarSchema(); + + + + /* + * If the user provides a body element, then they want to provide the JSON or + * a data structure that can be converted into JSON, bypassing any further + * specialized type-checking or op-type specific features + */ + if (op.isDefined("body")) { + throw new RuntimeException("This mode is reserved for later. Do not use the 'body' op field."); + } + else { + TypeAndTarget opType = op.getTypeAndTarget(PulsarOpType.class, String.class); + + return switch (opType.enumId) { + case AdminTenant -> + new AdminTenantOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin); + case AdminNamespace -> + new AdminNamespaceOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin); + case AdminTopic -> + new AdminTopicOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin); + case MessageProduce -> + new MessageProducerOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema); + case MessageConsume -> + new MessageConsumerOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema); + case MessageRead -> + new MessageReaderOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema); + }; + } + } + +} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsTimeTrackOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java similarity index 62% rename from driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsTimeTrackOp.java rename to adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java index 8d588faff..c4bf6c3ee 100644 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsTimeTrackOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java @@ -14,20 +14,13 @@ * limitations under the License. */ -package io.nosqlbench.driver.jms.ops; +package io.nosqlbench.adapter.pulsar; -/** - * Base type of all Sync Pulsar Operations including Producers and Consumers. - */ -public abstract class JmsTimeTrackOp implements JmsOp { - - public void run(Runnable timeTracker) { - try { - this.run(); - } finally { - timeTracker.run(); - } - } - - public abstract void run(); +public enum PulsarOpType { + AdminTenant, + AdminNamespace, + AdminTopic, + MessageProduce, + MessageConsume, + MessageRead } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java new file mode 100644 index 000000000..708007527 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar; + +import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil; +import io.nosqlbench.adapter.pulsar.util.PulsarNBClientConf; +import io.nosqlbench.api.config.standard.ConfigModel; +import io.nosqlbench.api.config.standard.NBConfigModel; +import io.nosqlbench.api.config.standard.NBConfiguration; +import io.nosqlbench.api.config.standard.Param; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.schema.KeyValueEncodingType; + +import java.util.Map; + +public class PulsarSpace { + + private final static Logger logger = LogManager.getLogger(PulsarSpace.class); + + private final String name; + private final NBConfiguration cfg; + + private final String pulsarSvcUrl; + private final String webSvcUrl; + + private PulsarNBClientConf pulsarNBClientConf; + private PulsarClient pulsarClient; + private PulsarAdmin pulsarAdmin; + private Schema pulsarSchema; + + public PulsarSpace(String name, NBConfiguration cfg) { + this.name = name; + this.cfg = cfg; + + this.pulsarSvcUrl = cfg.get("service_url"); + this.webSvcUrl = cfg.get("web_url"); + + this.pulsarNBClientConf = new PulsarNBClientConf(cfg.get("config")); + + initPulsarAdminAndClientObj(); + createPulsarSchemaFromConf(); + } + + public static NBConfigModel getConfigModel() { + return ConfigModel.of(PulsarSpace.class) + .add(Param.defaultTo("service_url", "pulsar://localhost:6650") + .setDescription("Pulsar broker service URL.")) + .add(Param.defaultTo("web_url", "http://localhost:8080") + .setDescription("Pulsar web service URL.")) + .add(Param.defaultTo("config", "config.properties") + .setDescription("Pulsar client connection configuration property file.")) + .add(Param.defaultTo("cyclerate_per_thread", false) + .setDescription("Apply cycle rate per NB thread")) + .asReadOnly(); + } + + public String getPulsarSvcUrl() { return pulsarSvcUrl; } + public String getWebSvcUrl() { return webSvcUrl; } + public PulsarNBClientConf getPulsarNBClientConf() { return pulsarNBClientConf; } + public PulsarClient getPulsarClient() { return pulsarClient; } + public PulsarAdmin getPulsarAdmin() { return pulsarAdmin; } + public Schema getPulsarSchema() { return pulsarSchema; } + + /** + * Initialize + * - PulsarAdmin object for adding/deleting tenant, namespace, and topic + * - PulsarClient object for message publishing and consuming + */ + private void initPulsarAdminAndClientObj() { + PulsarAdminBuilder adminBuilder = + PulsarAdmin.builder() + .serviceHttpUrl(webSvcUrl); + + ClientBuilder clientBuilder = PulsarClient.builder(); + + try { + Map clientConfMap = pulsarNBClientConf.getClientConfMap(); + + // Override "client.serviceUrl" setting in config.properties + clientConfMap.remove("serviceUrl"); + clientBuilder.loadConf(clientConfMap).serviceUrl(pulsarSvcUrl); + + // Pulsar Authentication + String authPluginClassName = + (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authPulginClassName.label); + String authParams = + (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authParams.label); + + if ( !StringUtils.isAnyBlank(authPluginClassName, authParams) ) { + adminBuilder.authentication(authPluginClassName, authParams); + clientBuilder.authentication(authPluginClassName, authParams); + } + + boolean useTls = StringUtils.contains(pulsarSvcUrl, "pulsar+ssl"); + if ( useTls ) { + String tlsHostnameVerificationEnableStr = + (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsHostnameVerificationEnable.label); + boolean tlsHostnameVerificationEnable = BooleanUtils.toBoolean(tlsHostnameVerificationEnableStr); + + adminBuilder + .enableTlsHostnameVerification(tlsHostnameVerificationEnable); + clientBuilder + .enableTlsHostnameVerification(tlsHostnameVerificationEnable); + + String tlsTrustCertsFilePath = + (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsTrustCertsFilePath.label); + if (!StringUtils.isBlank(tlsTrustCertsFilePath)) { + adminBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath); + clientBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath); + } + + String tlsAllowInsecureConnectionStr = + (String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsAllowInsecureConnection.label); + boolean tlsAllowInsecureConnection = BooleanUtils.toBoolean(tlsAllowInsecureConnectionStr); + adminBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection); + clientBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection); + } + + pulsarAdmin = adminBuilder.build(); + pulsarClient = clientBuilder.build(); + + } catch (PulsarClientException e) { + logger.error("Fail to create PulsarAdmin and/or PulsarClient object from the global configuration!"); + throw new RuntimeException("Fail to create PulsarAdmin and/or PulsarClient object from global configuration!"); + } + } + + /** + * Get Pulsar schema from the definition string + */ + + private Schema buildSchemaFromDefinition(String schemaTypeConfEntry, + String schemaDefinitionConfEntry) { + Object value = pulsarNBClientConf.getSchemaConfValue(schemaTypeConfEntry); + Object schemaDefinition = pulsarNBClientConf.getSchemaConfValue(schemaDefinitionConfEntry); + String schemaType = (value != null) ? value.toString() : ""; + + Schema result; + if (PulsarAdapterUtil.isAvroSchemaTypeStr(schemaType)) { + String schemaDefStr = (schemaDefinition != null) ? schemaDefinition.toString() : ""; + result = PulsarAdapterUtil.getAvroSchema(schemaType, schemaDefStr); + } else if (PulsarAdapterUtil.isPrimitiveSchemaTypeStr(schemaType)) { + result = PulsarAdapterUtil.getPrimitiveTypeSchema(schemaType); + } else if (PulsarAdapterUtil.isAutoConsumeSchemaTypeStr(schemaType)) { + result = Schema.AUTO_CONSUME(); + } else { + throw new RuntimeException("Unsupported schema type string: " + schemaType + "; " + + "Only primitive type, Avro type and AUTO_CONSUME are supported at the moment!"); + } + return result; + } + private void createPulsarSchemaFromConf() { + pulsarSchema = buildSchemaFromDefinition("schema.type", "schema.definition"); + + // this is to allow KEY_VALUE schema + if (pulsarNBClientConf.hasSchemaConfKey("schema.key.type")) { + Schema pulsarKeySchema = buildSchemaFromDefinition("schema.key.type", "schema.key.definition"); + Object encodingType = pulsarNBClientConf.getSchemaConfValue("schema.keyvalue.encodingtype"); + KeyValueEncodingType keyValueEncodingType = KeyValueEncodingType.SEPARATED; + if (encodingType != null) { + keyValueEncodingType = KeyValueEncodingType.valueOf(encodingType.toString()); + } + pulsarSchema = Schema.KeyValue(pulsarKeySchema, pulsarSchema, keyValueEncodingType); + } + } +} + + diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java new file mode 100644 index 000000000..89514d4e6 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.AdminNamespaceOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.admin.PulsarAdmin; + +import java.util.function.LongFunction; + +public class AdminNamespaceOpDispenser extends PulsarAdminOpDispenser { + + public AdminNamespaceOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarAdmin pulsarAdmin) { + super(adapter, op, tgtNameFunc, pulsarAdmin); + } + + @Override + public AdminNamespaceOp apply(long cycle) { + return new AdminNamespaceOp( + pulsarAdmin, + asyncApiFunc.apply(cycle), + adminDelOpFunc.apply(cycle), + tgtNameFunc.apply(cycle)); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java new file mode 100644 index 000000000..5bca4ec78 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.AdminTenantOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.admin.PulsarAdmin; + +import java.util.*; +import java.util.function.LongFunction; + +public class AdminTenantOpDispenser extends PulsarAdminOpDispenser { + + private final LongFunction> adminRolesFunc; + private final LongFunction> allowedClustersFunc; + public AdminTenantOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarAdmin pulsarAdmin) { + super(adapter, op, tgtNameFunc, pulsarAdmin); + + adminRolesFunc = lookupStaticStrSetOpValueFunc("admin_roles"); + allowedClustersFunc = lookupStaticStrSetOpValueFunc("allowed_clusters"); + } + + @Override + public AdminTenantOp apply(long cycle) { + return new AdminTenantOp( + pulsarAdmin, + asyncApiFunc.apply(cycle), + adminDelOpFunc.apply(cycle), + tgtNameFunc.apply(cycle), + adminRolesFunc.apply(cycle), + allowedClustersFunc.apply(cycle)); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java new file mode 100644 index 000000000..70af09180 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.AdminTopicOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.admin.PulsarAdmin; + +import java.util.function.LongFunction; + +public class AdminTopicOpDispenser extends PulsarAdminOpDispenser { + + private final LongFunction enablePartFunc; + private final LongFunction partNumFunc; + + public AdminTopicOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarAdmin pulsarAdmin) { + super(adapter, op, tgtNameFunc, pulsarAdmin); + + // Non-partitioned topic is default + enablePartFunc = lookupStaticBoolConfigValueFunc("enable_partition", false); + partNumFunc = lookupStaticIntOpValueFunc("partition_num", 1); + } + + @Override + public AdminTopicOp apply(long cycle) { + + return new AdminTopicOp( + pulsarAdmin, + asyncApiFunc.apply(cycle), + adminDelOpFunc.apply(cycle), + tgtNameFunc.apply(cycle), + enablePartFunc.apply(cycle), + partNumFunc.apply(cycle) + ); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java new file mode 100644 index 000000000..8a0cb6ba6 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.MessageConsumerOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +import java.util.function.LongFunction; + +public class MessageConsumerOpDispenser extends PulsarClientOpDispenser { + + public MessageConsumerOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarClient pulsarClient, + Schema pulsarSchema) { + super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema); + } + + @Override + public MessageConsumerOp apply(long cycle) { + return new MessageConsumerOp(pulsarClient, pulsarSchema); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java new file mode 100644 index 000000000..ef9b2547d --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.MessageProducerOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +import java.util.function.LongFunction; + +public class MessageProducerOpDispenser extends PulsarClientOpDispenser { + + public MessageProducerOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarClient pulsarClient, + Schema pulsarSchema) { + super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema); + } + + @Override + public MessageProducerOp apply(long cycle) { + return new MessageProducerOp(pulsarClient, pulsarSchema); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java new file mode 100644 index 000000000..3e9f819fd --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.adapter.pulsar.ops.MessageReaderOp; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +import java.util.function.LongFunction; + +public class MessageReaderOpDispenser extends PulsarClientOpDispenser { + + public MessageReaderOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarClient pulsarClient, + Schema pulsarSchema) { + super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema); + } + + @Override + public MessageReaderOp apply(long cycle) { + return new MessageReaderOp(pulsarClient, pulsarSchema); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java new file mode 100644 index 000000000..d18eee305 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.admin.PulsarAdmin; + +import java.util.function.LongFunction; + +public abstract class PulsarAdminOpDispenser extends PulsarBaseOpDispenser { + + protected final PulsarAdmin pulsarAdmin; + protected final LongFunction adminDelOpFunc; + + public PulsarAdminOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarAdmin pulsarAdmin) { + super(adapter, op, tgtNameFunc); + this.pulsarAdmin = pulsarAdmin; + + // Creating admin objects (tenant, namespace, topic) is the default + this.adminDelOpFunc = lookupStaticBoolConfigValueFunc("admin_delop", false); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java new file mode 100644 index 000000000..1a0350ac1 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java @@ -0,0 +1,85 @@ +package io.nosqlbench.adapter.pulsar.dispensers; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import io.nosqlbench.adapter.pulsar.PulsarSpace; +import io.nosqlbench.adapter.pulsar.ops.PulsarOp; +import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.*; +import java.util.function.LongFunction; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public abstract class PulsarBaseOpDispenser extends BaseOpDispenser { + + private final static Logger logger = LogManager.getLogger("PulsarBaseOpDispenser"); + protected final ParsedOp parsedOp; + protected final LongFunction asyncApiFunc; + protected final LongFunction tgtNameFunc; + + public PulsarBaseOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction tgtNameFunc) { + + super(adapter, op); + + this.parsedOp = op; + this.tgtNameFunc = tgtNameFunc; + // Async API is the default + this.asyncApiFunc = lookupStaticBoolConfigValueFunc("async_api", true); + } + + protected LongFunction lookupStaticBoolConfigValueFunc(String paramName, boolean defaultValue) { + return (l) -> parsedOp.getOptionalStaticConfig(paramName, String.class) + .filter(Predicate.not(String::isEmpty)) + .map(value -> BooleanUtils.toBoolean(value)) + .orElse(defaultValue); + } + + protected LongFunction lookupStaticIntOpValueFunc(String paramName, int defaultValue) { + return (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) + .filter(Predicate.not(String::isEmpty)) + .map(value -> NumberUtils.toInt(value)) + .map(value -> { + if (value < 0) return 0; + else return value; + }).orElse(defaultValue); + } + + protected LongFunction> lookupStaticStrSetOpValueFunc(String paramName) { + return (l) -> parsedOp.getOptionalStaticValue(paramName, String.class) + .filter(Predicate.not(String::isEmpty)) + .map(value -> { + Set set = new HashSet<>(); + + if (StringUtils.contains(value,',')) { + set = Arrays.stream(value.split(",")) + .map(String::trim) + .filter(Predicate.not(String::isEmpty)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + return set; + }).orElse(Collections.emptySet()); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java new file mode 100644 index 000000000..43acd0bb8 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.dispensers; + +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +import java.util.function.LongFunction; + +public abstract class PulsarClientOpDispenser extends PulsarBaseOpDispenser { + + protected final PulsarClient pulsarClient; + protected final Schema pulsarSchema; + + public PulsarClientOpDispenser(DriverAdapter adapter, + ParsedOp op, + LongFunction tgtNameFunc, + PulsarClient pulsarClient, + Schema pulsarSchema) { + super(adapter, op, tgtNameFunc); + this.pulsarClient = pulsarClient; + this.pulsarSchema = pulsarSchema; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java new file mode 100644 index 000000000..1004a7a72 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.pulsar.exception; + +public class PulsarAdapterInvalidParamException extends RuntimeException { + + public PulsarAdapterInvalidParamException(String paramName, String errDesc) { + super("Invalid setting for parameter (" + paramName + "): " + errDesc); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnexpectedException.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnexpectedException.java new file mode 100644 index 000000000..4f0031fce --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnexpectedException.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.pulsar.exception; + +public class PulsarAdapterUnexpectedException extends RuntimeException { + + public PulsarAdapterUnexpectedException(String message) { + super(message); + printStackTrace(); + } + public PulsarAdapterUnexpectedException(Exception e) { + super(e); + printStackTrace(); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnsupportedOpException.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnsupportedOpException.java new file mode 100644 index 000000000..475d358ea --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnsupportedOpException.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.pulsar.exception; + +public class PulsarAdapterUnsupportedOpException extends RuntimeException { + + public PulsarAdapterUnsupportedOpException(String pulsarOpType) { + super("Unsupported Pulsar adapter operation type: \"" + pulsarOpType + "\""); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java new file mode 100644 index 000000000..71e6f79ee --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.admin.Namespaces; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; + +import java.util.concurrent.CompletableFuture; + +public class AdminNamespaceOp extends PulsarAdminOp { + + private final static Logger logger = LogManager.getLogger(AdminNamespaceOp.class); + + // in format: / + private final String nsName; + + public AdminNamespaceOp(PulsarAdmin pulsarAdmin, + boolean asyncApi, + boolean adminDelOp, + String nsName) { + super(pulsarAdmin, asyncApi, adminDelOp); + this.nsName = nsName; + } + + @Override + public Void apply(long value) { + + // Do nothing if the namespace name is empty + if ( !StringUtils.isBlank(nsName) ) { + + Namespaces namespaces = pulsarAdmin.namespaces(); + + // Admin API - create tenants and namespaces + if (!adminDelOp) { + try { + if (!asyncApi) { + namespaces.createNamespace(nsName); + if (logger.isDebugEnabled()) { + logger.debug("Successful sync creation of namespace \"{}\"", nsName); + } + } else { + CompletableFuture future = namespaces.createNamespaceAsync(nsName); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.trace("Successful async creation of namespace \"{}\"", nsName); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error("Failed async creation of namespace \"{}\"", nsName); + } + return null; + }); + } + } + catch (PulsarAdminException.ConflictException ce) { + if (logger.isDebugEnabled()) { + logger.error("Namespace \"{}\" already exists - skip creation!", nsName); + } + } + catch (PulsarAdminException e) { + throw new PulsarAdapterUnexpectedException( + "Unexpected error when creating pulsar namespace \"" + nsName + "\""); + } + } + // Admin API - delete tenants and namespaces + else { + try { + if (!asyncApi) { + namespaces.deleteNamespace(nsName); + if (logger.isDebugEnabled()) { + logger.debug("Successful sync deletion of namespace \"{}\"", nsName); + } + } else { + CompletableFuture future = namespaces.deleteNamespaceAsync(nsName, true); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug("Successful sync deletion of namespace \"{}\"", nsName); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error("Failed async deletion of namespace \"{}\"", nsName); + } + return null; + }); + } + } + catch (PulsarAdminException.NotFoundException nfe) { + if (logger.isDebugEnabled()) { + logger.error("Namespace \"{}\" doesn't exists - skip deletion!", nsName); + } + } + catch (PulsarAdminException e) { + throw new PulsarAdapterUnexpectedException( + "Unexpected error when deleting pulsar namespace \"" + nsName + "\""); + } + } + } + + return null; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java new file mode 100644 index 000000000..b40013ff7 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.adapter.pulsar.PulsarDriverAdapter; +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.admin.*; +import org.apache.pulsar.common.policies.data.TenantInfo; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +public class AdminTenantOp extends PulsarAdminOp { + + private final static Logger logger = LogManager.getLogger(AdminTenantOp.class); + + private final Set adminRoles; + private final Set allowedClusters; + private final String tntName; + + public AdminTenantOp(PulsarAdmin pulsarAdmin, + boolean asyncApi, + boolean adminDelOp, + String tntName, + Set adminRoles, + Set allowedClusters) { + super(pulsarAdmin, asyncApi, adminDelOp); + this.tntName = tntName; + this.adminRoles = adminRoles; + this.allowedClusters = allowedClusters; + } + + @Override + public Void apply(long value) { + + // Do nothing if the tenant name is empty + if ( !StringUtils.isBlank(tntName) ) { + Tenants tenants = pulsarAdmin.tenants(); + + // Admin API - create tenants and namespaces + if (!adminDelOp) { + try { + Set existingPulsarClusters = new HashSet<>(); + Clusters clusters = pulsarAdmin.clusters(); + CollectionUtils.addAll(existingPulsarClusters, clusters.getClusters().listIterator()); + + TenantInfo tenantInfo = TenantInfo.builder() + .adminRoles(adminRoles) + .allowedClusters(!allowedClusters.isEmpty() ? allowedClusters : existingPulsarClusters) + .build(); + + if (!asyncApi) { + tenants.createTenant(tntName, tenantInfo); + if (logger.isDebugEnabled()) { + logger.debug("Successful sync creation of tenant \"{}\"", tntName); + } + } + else { + CompletableFuture future = tenants.createTenantAsync(tntName, tenantInfo); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug("Successful async creation of tenant \"{}\"", tntName); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error("Failed async creation of tenant \"{}\"", tntName); + } + return null; + }); + } + } + catch (PulsarAdminException.ConflictException ce) { + if (logger.isDebugEnabled()) { + logger.error("Tenant \"{}\" already exists - skip creation!", tntName); + } + } + catch (PulsarAdminException e) { + throw new PulsarAdapterUnexpectedException( + "Unexpected error when creating pulsar tenant \"" + tntName + "\""); + } + } + // Admin API - delete tenants and namespaces + else { + try { + Namespaces namespaces = pulsarAdmin.namespaces(); + int nsNum = namespaces.getNamespaces(tntName).size(); + + // Only delete a tenant when there is no underlying namespaces + if ( nsNum == 0 ) { + if (!asyncApi) { + tenants.deleteTenant(tntName); + if (logger.isDebugEnabled()) { + logger.debug("Successful sync deletion of tenant \"{}\"", tntName); + } + } + else { + CompletableFuture future = tenants.deleteTenantAsync(tntName); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug("Successful async deletion of tenant \"{}\"", tntName); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error("Failed async deletion of tenant \"{}\"", tntName); + } + return null; + }); + } + } + } + catch (PulsarAdminException.NotFoundException nfe) { + if (logger.isDebugEnabled()) { + logger.error("Tenant \"{}\" doesn't exists - skip deletion!", tntName); + } + } + catch (PulsarAdminException e) { + throw new PulsarAdapterUnexpectedException( + "Unexpected error when deleting pulsar tenant \"" + tntName + "\""); + } + } + } + + return null; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java new file mode 100644 index 000000000..2c3735658 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.admin.Topics; + +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +public class AdminTopicOp extends PulsarAdminOp { + + private final static Logger logger = LogManager.getLogger(AdminTopicOp.class); + + private final String topicName; + private final boolean enablePart; + private final int partNum; + + public AdminTopicOp(PulsarAdmin pulsarAdmin, + boolean asyncApi, + boolean adminDelOp, + String topicName, + boolean enablePart, + int partNum) { + super(pulsarAdmin, asyncApi, adminDelOp); + this.topicName = topicName; + this.enablePart = enablePart; + this.partNum = partNum; + } + + @Override + public Void apply(long value) { + + // Do nothing if the topic name is empty + if ( !StringUtils.isBlank(topicName) ) { + Topics topics = pulsarAdmin.topics(); + + try { + // Create the topic + if (!adminDelOp) { + if (!enablePart) { + if (!asyncApi) { + topics.createNonPartitionedTopic(topicName); + if (logger.isDebugEnabled()) { + logger.debug("Successful sync creation of non-partitioned topic \"{}\"", topicName); + } + } else { + CompletableFuture future = topics.createNonPartitionedTopicAsync(topicName); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug("Successful async creation of non-partitioned topic \"{}\"", topicName); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error("Failed async creation non-partitioned topic \"{}\"", topicName); + return null; + } + return null; + }); + } + } else { + if (!asyncApi) { + topics.createPartitionedTopic(topicName, partNum); + if (logger.isDebugEnabled()) { + logger.debug( + "Successful sync creation of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + } else { + CompletableFuture future = topics.createPartitionedTopicAsync(topicName, partNum); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug( + "Successful async creation of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + }) + .exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error( + "Successful async creation of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + return null; + }); + } + } + } + // Delete the topic + else { + if (!enablePart) { + if (!asyncApi) { + topics.delete(topicName); + if (logger.isDebugEnabled()) { + logger.debug( + "Successful sync deletion of non-partitioned topic \"{}\"", + topicName); + } + } else { + CompletableFuture future = topics.deleteAsync(topicName); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug( + "Successful async deletion of non-partitioned topic \"{}\"", + topicName); + } + }) + .exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error( + "Failed async deletion of non-partitioned topic \"{}\"", + topicName); + } + return null; + }); + } + } else { + if (!asyncApi) { + topics.deletePartitionedTopic(topicName); + if (logger.isDebugEnabled()) { + logger.debug( + "Successful sync deletion of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + } else { + CompletableFuture future = topics.deletePartitionedTopicAsync(topicName); + future.whenComplete((unused, throwable) -> { + if (logger.isDebugEnabled()) { + logger.debug( + "Successful async deletion of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + }).exceptionally(ex -> { + if (logger.isDebugEnabled()) { + logger.error( + "Failed async deletion of partitioned topic \"{} (partition_num: {}\")", + topicName, partNum); + } + return null; + }); + } + } + } + } + catch (PulsarAdminException.NotFoundException nfe) { + if (logger.isDebugEnabled()) { + logger.error("Topic \"{}\" doesn't exists - skip deletion!", topicName); + } + } + catch (PulsarAdminException e) { + String errMsg = String.format("Unexpected error when %s pulsar topic: %s (partition enabled: %b; partition number: %d)", + (!adminDelOp ? "creating" : "deleting"), + topicName, + enablePart, + partNum); + throw new PulsarAdapterUnexpectedException(errMsg); + } + } + + return null; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java new file mode 100644 index 000000000..d9f28c7d3 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +public class MessageConsumerOp extends PulsarClientOp { + public MessageConsumerOp(PulsarClient pulsarClient, Schema pulsarSchema) { + super(pulsarClient, pulsarSchema); + } + + @Override + public Object apply(long value) { + return null; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java new file mode 100644 index 000000000..664509ec8 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +public class MessageProducerOp extends PulsarClientOp { + + public MessageProducerOp(PulsarClient pulsarClient, Schema pulsarSchema) { + super(pulsarClient, pulsarSchema); + } + + @Override + public Object apply(long value) { + return null; + } +} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java similarity index 60% rename from driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOp.java rename to adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java index b8f227ffd..c9d47ec30 100644 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOp.java +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java @@ -14,16 +14,19 @@ * limitations under the License. */ -package io.nosqlbench.driver.jms.ops; +package io.nosqlbench.adapter.pulsar.ops; -/** - * Base type of all Pulsar Operations including Producers and Consumers. - */ -public interface JmsOp { +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; - /** - * Execute the operation, invoke the timeTracker when the operation ended. - * The timeTracker can be invoked in a separate thread, it is only used for metrics. - */ - void run(Runnable timeTracker); +public class MessageReaderOp extends PulsarClientOp { + + public MessageReaderOp(PulsarClient pulsarClient, Schema pulsarSchema) { + super(pulsarClient, pulsarSchema); + } + + @Override + public Object apply(long value) { + return null; + } } diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java new file mode 100644 index 000000000..4729a8cc5 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp; +import org.apache.pulsar.client.admin.PulsarAdmin; + +public abstract class PulsarAdminOp extends PulsarOp { + protected PulsarAdmin pulsarAdmin; + protected boolean asyncApi; + protected boolean adminDelOp; + + public PulsarAdminOp(PulsarAdmin pulsarAdmin, boolean asyncApi, boolean adminDelOp) { + this.pulsarAdmin = pulsarAdmin; + this.asyncApi = asyncApi; + this.adminDelOp = adminDelOp; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java new file mode 100644 index 000000000..957616681 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; + +public abstract class PulsarClientOp extends PulsarOp { + protected PulsarClient pulsarClient; + protected Schema pulsarScheam; + + public PulsarClientOp(PulsarClient pulsarClient, Schema pulsarScheam) { + this.pulsarClient = pulsarClient; + this.pulsarScheam = pulsarScheam; + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java new file mode 100644 index 000000000..0c68c52d7 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.nosqlbench.adapter.pulsar.ops; + +import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp; + +public abstract class PulsarOp implements CycleOp { +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/AvroUtil.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/AvroUtil.java new file mode 100644 index 000000000..882f060f9 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/AvroUtil.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.util; + +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.JsonDecoder; +import org.apache.avro.io.BinaryDecoder; +import org.apache.pulsar.client.api.schema.Field; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.GenericRecordBuilder; +import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; + +public class AvroUtil { + //////////////////////// + // Get an OSS Apache Avro schema from a string definition + public static org.apache.avro.Schema GetSchema_ApacheAvro(String avroSchemDef) { + return new org.apache.avro.Schema.Parser().parse(avroSchemDef); + } + + // Get an OSS Apache Avro schema record from a JSON string that matches a specific OSS Apache Avro schema + public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, String jsonData) { + org.apache.avro.generic.GenericRecord record = null; + + try { + org.apache.avro.generic.GenericDatumReader reader; + reader = new org.apache.avro.generic.GenericDatumReader<>(schema); + + JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, jsonData); + + record = reader.read(null, decoder); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + + return record; + } + + // Get an OSS Apache Avro schema record from a byte array that matches a specific OSS Apache Avro schema + public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, byte[] bytesData) { + org.apache.avro.generic.GenericRecord record = null; + + try { + org.apache.avro.generic.GenericDatumReader reader; + reader = new org.apache.avro.generic.GenericDatumReader<>(schema); + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytesData, null); + + record = reader.read(null, decoder); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + + return record; + } + + + //////////////////////// + // Get a Pulsar Avro schema from a string definition + public static GenericAvroSchema GetSchema_PulsarAvro(String schemaName, String avroSchemDef) { + SchemaInfo schemaInfo = SchemaInfoImpl.builder() + .schema(avroSchemDef.getBytes(StandardCharsets.UTF_8)) + .type(SchemaType.AVRO) + .properties(new HashMap<>()) + .name(schemaName) + .build(); + return new GenericAvroSchema(schemaInfo); + } + + // Get a Pulsar Avro record from an OSS Avro schema record, matching a specific Pulsar Avro schema + public static GenericRecord GetGenericRecord_PulsarAvro( + GenericAvroSchema pulsarGenericAvroSchema, + org.apache.avro.generic.GenericRecord apacheAvroGenericRecord) + { + GenericRecordBuilder recordBuilder = pulsarGenericAvroSchema.newRecordBuilder(); + + List fieldList = pulsarGenericAvroSchema.getFields(); + for (Field field : fieldList) { + String fieldName = field.getName(); + recordBuilder.set(fieldName, apacheAvroGenericRecord.get(fieldName)); + } + + return recordBuilder.build(); + } + + // Get a Pulsar Avro record (GenericRecord) from a JSON string that matches a specific Pulsar Avro schema + public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, String avroSchemDefStr, String jsonData) { + org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr); + return GetGenericRecord_PulsarAvro(genericAvroSchema, avroSchema, jsonData); + } + + public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, org.apache.avro.Schema avroSchema, String jsonData) { + org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData); + return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord); + } + public static GenericRecord GetGenericRecord_PulsarAvro(String schemaName, String avroSchemDefStr, String jsonData) { + GenericAvroSchema genericAvroSchema = GetSchema_PulsarAvro(schemaName, avroSchemDefStr); + org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr); + org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData); + + return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord); + } +} diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java new file mode 100644 index 000000000..f2efbb8b6 --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java @@ -0,0 +1,538 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pulsar.client.api.Schema; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class PulsarAdapterUtil { + + private final static Logger logger = LogManager.getLogger(PulsarAdapterUtil.class); + + // Supported message operation types + // TODO: websocket-producer and managed-ledger + public enum OP_TYPES { + ADMIN_TENANT("admin-tenant"), + ADMIN_NAMESPACE("admin-namespace"), + ADMIN_TOPIC("admin-topic"), + E2E_MSG_PROC_SEND("e22-msg-proc-send"), + E2E_MSG_PROC_CONSUME("e22-msg-proc-consume"), +// BATCH_MSG_SEND_START("batch-msg-send-start"), +// BATCH_MSG_SEND("batch-msg-send"), +// BATCH_MSG_SEND_END("batch-msg-send-end"), + MSG_SEND("msg-send"), + MSG_CONSUME("msg-consume"), + MSG_READ("msg-read"), + MSG_MULTI_CONSUME("msg-mt-consume"); + + public final String label; + + OP_TYPES(String label) { + this.label = label; + } + } + + + public static boolean isValidClientType(String type) { + return Arrays.stream(OP_TYPES.values()).anyMatch(t -> t.label.equals(type)); + } + + public static final String MSG_SEQUENCE_NUMBER = "sequence_number"; + + /////// + // Valid document level parameters for Pulsar NB yaml file + public enum DOC_LEVEL_PARAMS { + TOPIC_URI("topic_uri"), + ASYNC_API("async_api"), + USE_TRANSACTION("use_transaction"), + ADMIN_DELOP("admin_delop"), + SEQ_TRACKING("seq_tracking"), + MSG_DEDUP_BROKER("msg_dedup_broker"), + E2E_STARTING_TIME_SOURCE("e2e_starting_time_source"); + + public final String label; + + DOC_LEVEL_PARAMS(String label) { + this.label = label; + } + } + public static boolean isValidDocLevelParam(String param) { + return Arrays.stream(DOC_LEVEL_PARAMS.values()).anyMatch(t -> t.label.equals(param)); + } + + /////// + // Valid Pulsar API type + public enum PULSAR_API_TYPE { + PRODUCER("producer"), + CONSUMER("consumer"), + READER("reader"); + + public final String label; + + PULSAR_API_TYPE(String label) { + this.label = label; + } + } + public static boolean isValidPulsarApiType(String param) { + return Arrays.stream(PULSAR_API_TYPE.values()).anyMatch(t -> t.label.equals(param)); + } + public static String getValidPulsarApiTypeList() { + return Arrays.stream(PULSAR_API_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + /////// + // Valid persistence type + public enum PERSISTENT_TYPES { + PERSISTENT("persistent"), + NON_PERSISTENT("non-persistent") + ; + + public final String label; + PERSISTENT_TYPES(String label) { + this.label = label; + } + } + public static boolean isValidPersistenceType(String type) { + return Arrays.stream(PERSISTENT_TYPES.values()).anyMatch(t -> t.label.equals(type)); + } + + /////// + // Valid Pulsar client configuration (activity-level settings) + // - https://pulsar.apache.org/docs/en/client-libraries-java/#client + public enum CLNT_CONF_KEY { + serviceUrl("serviceUrl"), + authPulginClassName("authPluginClassName"), + authParams("authParams"), + pperationTimeoutMs("operationTimeoutMs"), + statsIntervalSeconds("statsIntervalSeconds"), + numIoThreads("numIoThreads"), + numListenerThreads("numListenerThreads"), + useTcpNoDelay("useTcpNoDelay"), + enableTls("enableTls"), + tlsTrustCertsFilePath("tlsTrustCertsFilePath"), + tlsAllowInsecureConnection("tlsAllowInsecureConnection"), + tlsHostnameVerificationEnable("tlsHostnameVerificationEnable"), + concurrentLookupRequest("concurrentLookupRequest"), + maxLookupRequest("maxLookupRequest"), + maxNumberOfRejectedRequestPerConnection("maxNumberOfRejectedRequestPerConnection"), + keepAliveIntervalSeconds("keepAliveIntervalSeconds"), + connectionTimeoutMs("connectionTimeoutMs"), + requestTimeoutMs("requestTimeoutMs"), + defaultBackoffIntervalNanos("defaultBackoffIntervalNanos"), + maxBackoffIntervalNanos("maxBackoffIntervalNanos"), + socks5ProxyAddress("socks5ProxyAddress"), + socks5ProxyUsername("socks5ProxyUsername"), + socks5ProxyPassword("socks5ProxyPassword") + ; + + public final String label; + CLNT_CONF_KEY(String label) { + this.label = label; + } + } + public static boolean isValidClientConfItem(String item) { + return Arrays.stream(CLNT_CONF_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Standard producer configuration (activity-level settings) + // - https://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer + public enum PRODUCER_CONF_STD_KEY { + topicName("topicName"), + producerName("producerName"), + sendTimeoutMs("sendTimeoutMs"), + blockIfQueueFull("blockIfQueueFull"), + maxPendingMessages("maxPendingMessages"), + maxPendingMessagesAcrossPartitions("maxPendingMessagesAcrossPartitions"), + messageRoutingMode("messageRoutingMode"), + hashingScheme("hashingScheme"), + cryptoFailureAction("cryptoFailureAction"), + batchingMaxPublishDelayMicros("batchingMaxPublishDelayMicros"), + batchingMaxMessages("batchingMaxMessages"), + batchingEnabled("batchingEnabled"), + compressionType("compressionType"); + + public final String label; + + PRODUCER_CONF_STD_KEY(String label) { + this.label = label; + } + } + public static boolean isStandardProducerConfItem(String item) { + return Arrays.stream(PRODUCER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Standard consumer configuration (activity-level settings) + // - https://pulsar.apache.org/docs/en/client-libraries-java/#consumer + public enum CONSUMER_CONF_STD_KEY { + topicNames("topicNames"), + topicsPattern("topicsPattern"), + subscriptionName("subscriptionName"), + subscriptionType("subscriptionType"), + receiverQueueSize("receiverQueueSize"), + acknowledgementsGroupTimeMicros("acknowledgementsGroupTimeMicros"), + negativeAckRedeliveryDelayMicros("negativeAckRedeliveryDelayMicros"), + maxTotalReceiverQueueSizeAcrossPartitions("maxTotalReceiverQueueSizeAcrossPartitions"), + consumerName("consumerName"), + ackTimeoutMillis("ackTimeoutMillis"), + tickDurationMillis("tickDurationMillis"), + priorityLevel("priorityLevel"), + cryptoFailureAction("cryptoFailureAction"), + properties("properties"), + readCompacted("readCompacted"), + subscriptionInitialPosition("subscriptionInitialPosition"), + patternAutoDiscoveryPeriod("patternAutoDiscoveryPeriod"), + regexSubscriptionMode("regexSubscriptionMode"), + deadLetterPolicy("deadLetterPolicy"), + autoUpdatePartitions("autoUpdatePartitions"), + replicateSubscriptionState("replicateSubscriptionState"); + + public final String label; + + CONSUMER_CONF_STD_KEY(String label) { + this.label = label; + } + } + public static boolean isStandardConsumerConfItem(String item) { + return Arrays.stream(CONSUMER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Custom consumer configuration (activity-level settings) + // - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#consumer + // - NB Pulsar driver consumer operation specific + public enum CONSUMER_CONF_CUSTOM_KEY { + timeout("timeout"); + + public final String label; + + CONSUMER_CONF_CUSTOM_KEY(String label) { + this.label = label; + } + } + public static boolean isCustomConsumerConfItem(String item) { + return Arrays.stream(CONSUMER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Pulsar subscription type + public enum SUBSCRIPTION_TYPE { + Exclusive("Exclusive"), + Failover("Failover"), + Shared("Shared"), + Key_Shared("Key_Shared"); + + public final String label; + + SUBSCRIPTION_TYPE(String label) { + this.label = label; + } + } + public static boolean isValidSubscriptionType(String item) { + return Arrays.stream(SUBSCRIPTION_TYPE.values()).anyMatch(t -> t.label.equals(item)); + } + public static String getValidSubscriptionTypeList() { + return Arrays.stream(SUBSCRIPTION_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + /////// + // Standard reader configuration (activity-level settings) + // - https://pulsar.apache.org/docs/en/client-libraries-java/#reader + public enum READER_CONF_STD_KEY { + topicName("topicName"), + receiverQueueSize("receiverQueueSize"), + readerListener("readerListener"), + readerName("readerName"), + subscriptionRolePrefix("subscriptionRolePrefix"), + cryptoKeyReader("cryptoKeyReader"), + cryptoFailureAction("cryptoFailureAction"), + readCompacted("readCompacted"), + resetIncludeHead("resetIncludeHead"); + + public final String label; + + READER_CONF_STD_KEY(String label) { + this.label = label; + } + } + public static boolean isStandardReaderConfItem(String item) { + return Arrays.stream(READER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Custom reader configuration (activity-level settings) + // - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#reader + // - NB Pulsar driver reader operation specific + public enum READER_CONF_CUSTOM_KEY { + startMessagePos("startMessagePos"); + + public final String label; + + READER_CONF_CUSTOM_KEY(String label) { + this.label = label; + } + } + public static boolean isCustomReaderConfItem(String item) { + return Arrays.stream(READER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Valid read positions for a Pulsar reader + public enum READER_MSG_POSITION_TYPE { + earliest("earliest"), + latest("latest"), + custom("custom"); + + public final String label; + + READER_MSG_POSITION_TYPE(String label) { + this.label = label; + } + } + public static boolean isValideReaderStartPosition(String item) { + return Arrays.stream(READER_MSG_POSITION_TYPE.values()).anyMatch(t -> t.label.equals(item)); + } + + /////// + // Pulsar subscription type + public enum SEQ_ERROR_SIMU_TYPE { + OutOfOrder("out_of_order"), + MsgLoss("msg_loss"), + MsgDup("msg_dup"); + + public final String label; + + SEQ_ERROR_SIMU_TYPE(String label) { + this.label = label; + } + + private static final Map MAPPING = new HashMap<>(); + + static { + for (SEQ_ERROR_SIMU_TYPE simuType : values()) { + MAPPING.put(simuType.label, simuType); + MAPPING.put(simuType.label.toLowerCase(), simuType); + MAPPING.put(simuType.label.toUpperCase(), simuType); + MAPPING.put(simuType.name(), simuType); + MAPPING.put(simuType.name().toLowerCase(), simuType); + MAPPING.put(simuType.name().toUpperCase(), simuType); + } + } + + public static Optional parseSimuType(String simuTypeString) { + return Optional.ofNullable(MAPPING.get(simuTypeString.trim())); + } + } + public static boolean isValidSeqErrSimuType(String item) { + return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).anyMatch(t -> t.label.equals(item)); + } + public static String getValidSeqErrSimuTypeList() { + return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); + } + + /////// + // Valid websocket-producer configuration (activity-level settings) + // TODO: to be added + public enum WEBSKT_PRODUCER_CONF_KEY { + ; + + public final String label; + + WEBSKT_PRODUCER_CONF_KEY(String label) { + this.label = label; + } + } + + /////// + // Valid managed-ledger configuration (activity-level settings) + // TODO: to be added + public enum MANAGED_LEDGER_CONF_KEY { + ; + + public final String label; + MANAGED_LEDGER_CONF_KEY(String label) { + this.label = label; + } + } + + /////// + // Primitive Schema type + public static boolean isPrimitiveSchemaTypeStr(String typeStr) { + boolean isPrimitive = false; + + // Use "BYTES" as the default type if the type string is not explicitly specified + if (StringUtils.isBlank(typeStr)) { + typeStr = "BYTES"; + } + + if (typeStr.equalsIgnoreCase("BOOLEAN") || typeStr.equalsIgnoreCase("INT8") || + typeStr.equalsIgnoreCase("INT16") || typeStr.equalsIgnoreCase("INT32") || + typeStr.equalsIgnoreCase("INT64") || typeStr.equalsIgnoreCase("FLOAT") || + typeStr.equalsIgnoreCase("DOUBLE") || typeStr.equalsIgnoreCase("BYTES") || + typeStr.equalsIgnoreCase("DATE") || typeStr.equalsIgnoreCase("TIME") || + typeStr.equalsIgnoreCase("TIMESTAMP") || typeStr.equalsIgnoreCase("INSTANT") || + typeStr.equalsIgnoreCase("LOCAL_DATE") || typeStr.equalsIgnoreCase("LOCAL_TIME") || + typeStr.equalsIgnoreCase("LOCAL_DATE_TIME")) { + isPrimitive = true; + } + + return isPrimitive; + } + public static Schema getPrimitiveTypeSchema(String typeStr) { + Schema schema; + + switch (typeStr.toUpperCase()) { + case "BOOLEAN": + schema = Schema.BOOL; + break; + case "INT8": + schema = Schema.INT8; + break; + case "INT16": + schema = Schema.INT16; + break; + case "INT32": + schema = Schema.INT32; + break; + case "INT64": + schema = Schema.INT64; + break; + case "FLOAT": + schema = Schema.FLOAT; + break; + case "DOUBLE": + schema = Schema.DOUBLE; + break; + case "DATE": + schema = Schema.DATE; + break; + case "TIME": + schema = Schema.TIME; + break; + case "TIMESTAMP": + schema = Schema.TIMESTAMP; + break; + case "INSTANT": + schema = Schema.INSTANT; + break; + case "LOCAL_DATE": + schema = Schema.LOCAL_DATE; + break; + case "LOCAL_TIME": + schema = Schema.LOCAL_TIME; + break; + case "LOCAL_DATE_TIME": + schema = Schema.LOCAL_DATE_TIME; + break; + // Use BYTES as the default schema type if the type string is not specified + case "": + case "BYTES": + schema = Schema.BYTES; + break; + // Report an error if non-valid, non-empty schema type string is provided + default: + throw new RuntimeException("Invalid Pulsar primitive schema type string : " + typeStr); + } + + return schema; + } + + /////// + // Complex strut type: Avro or Json + public static boolean isAvroSchemaTypeStr(String typeStr) { + return typeStr.equalsIgnoreCase("AVRO"); + } + public static boolean isKeyValueTypeStr(String typeStr) { + return typeStr.equalsIgnoreCase("KEY_VALUE"); + } + + // automatic decode the type from the Registry + public static boolean isAutoConsumeSchemaTypeStr(String typeStr) { + return typeStr.equalsIgnoreCase("AUTO_CONSUME"); + } + public static Schema getAvroSchema(String typeStr, String definitionStr) { + String schemaDefinitionStr = definitionStr; + String filePrefix = "file://"; + Schema schema; + + // Check if payloadStr points to a file (e.g. "file:///path/to/a/file") + if (isAvroSchemaTypeStr(typeStr)) { + if (StringUtils.isBlank(schemaDefinitionStr)) { + throw new RuntimeException("Schema definition must be provided for \"Avro\" schema type!"); + } else if (schemaDefinitionStr.startsWith(filePrefix)) { + try { + Path filePath = Paths.get(URI.create(schemaDefinitionStr)); + schemaDefinitionStr = Files.readString(filePath, StandardCharsets.US_ASCII); + } catch (IOException ioe) { + throw new RuntimeException("Error reading the specified \"Avro\" schema definition file: " + definitionStr + ": " + ioe.getMessage()); + } + } + + schema = AvroUtil.GetSchema_PulsarAvro("NBAvro", schemaDefinitionStr); + } else { + throw new RuntimeException("Trying to create a \"Avro\" schema for a non-Avro schema type string: " + typeStr); + } + + return schema; + } + + /////// + // Generate effective key string + public static String buildCacheKey(String... keyParts) { + // Ignore blank keyPart + String joinedKeyStr = + Stream.of(keyParts) + .filter(s -> !StringUtils.isBlank(s)) + .collect(Collectors.joining(",")); + + return Base64.getEncoder().encodeToString(joinedKeyStr.getBytes()); + } + + /////// + // Convert JSON string to a key/value map + public static Map convertJsonToMap(String jsonStr) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(jsonStr, Map.class); + } + + /////// + // Get full namespace name (/) from a Pulsar topic URI + public static String getFullNamespaceName(String topicUri) { + // Get tenant/namespace string + // - topicUri : persistent://// + // - tmpStr : // + // - fullNsName : / + + String tmpStr = StringUtils.substringAfter(topicUri,"://"); + return StringUtils.substringBeforeLast(tmpStr, "/"); + } +} + diff --git a/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarNBClientConf.java b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarNBClientConf.java new file mode 100644 index 000000000..a434bc10d --- /dev/null +++ b/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarNBClientConf.java @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.pulsar.util; + +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.FileBasedConfiguration; +import org.apache.commons.configuration2.PropertiesConfiguration; +import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder; +import org.apache.commons.configuration2.builder.fluent.Parameters; +import org.apache.commons.configuration2.ex.ConfigurationException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +public class PulsarNBClientConf { + + private final static Logger logger = LogManager.getLogger(PulsarNBClientConf.class); + + private String canonicalFilePath = ""; + + public static final String SCHEMA_CONF_PREFIX = "schema"; + public static final String CLIENT_CONF_PREFIX = "client"; + public static final String PRODUCER_CONF_PREFIX = "producer"; + public static final String CONSUMER_CONF_PREFIX = "consumer"; + public static final String READER_CONF_PREFIX = "reader"; + private final HashMap schemaConfMap = new HashMap<>(); + private final HashMap clientConfMap = new HashMap<>(); + private final HashMap producerConfMap = new HashMap<>(); + private final HashMap consumerConfMap = new HashMap<>(); + private final HashMap readerConfMap = new HashMap<>(); + // TODO: add support for other operation types: websocket-producer, managed-ledger + + public PulsarNBClientConf(String fileName) { + File file = new File(fileName); + + try { + canonicalFilePath = file.getCanonicalPath(); + + Parameters params = new Parameters(); + + FileBasedConfigurationBuilder builder = + new FileBasedConfigurationBuilder(PropertiesConfiguration.class) + .configure(params.properties() + .setFileName(fileName)); + + Configuration config = builder.getConfiguration(); + + // Get schema specific configuration settings + for (Iterator it = config.getKeys(SCHEMA_CONF_PREFIX); it.hasNext(); ) { + String confKey = it.next(); + String confVal = config.getProperty(confKey).toString(); + if (!StringUtils.isBlank(confVal)) + schemaConfMap.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), config.getProperty(confKey)); + } + + // Get client connection specific configuration settings + for (Iterator it = config.getKeys(CLIENT_CONF_PREFIX); it.hasNext(); ) { + String confKey = it.next(); + String confVal = config.getProperty(confKey).toString(); + if (!StringUtils.isBlank(confVal)) + clientConfMap.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), config.getProperty(confKey)); + } + + // Get producer specific configuration settings + for (Iterator it = config.getKeys(PRODUCER_CONF_PREFIX); it.hasNext(); ) { + String confKey = it.next(); + String confVal = config.getProperty(confKey).toString(); + if (!StringUtils.isBlank(confVal)) + producerConfMap.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); + } + + // Get consumer specific configuration settings + for (Iterator it = config.getKeys(CONSUMER_CONF_PREFIX); it.hasNext(); ) { + String confKey = it.next(); + String confVal = config.getProperty(confKey).toString(); + if (!StringUtils.isBlank(confVal)) + consumerConfMap.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); + } + + // Get reader specific configuration settings + for (Iterator it = config.getKeys(READER_CONF_PREFIX); it.hasNext(); ) { + String confKey = it.next(); + String confVal = config.getProperty(confKey).toString(); + if (!StringUtils.isBlank(confVal)) + readerConfMap.put(confKey.substring(READER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); + } + } catch (IOException ioe) { + logger.error("Can't read the specified config properties file!"); + ioe.printStackTrace(); + } catch (ConfigurationException cex) { + logger.error("Error loading configuration items from the specified config properties file: " + canonicalFilePath); + cex.printStackTrace(); + } + } + + + ////////////////// + // Get Schema related config + public Map getSchemaConfMap() { + return this.schemaConfMap; + } + public boolean hasSchemaConfKey(String key) { + if (key.contains(SCHEMA_CONF_PREFIX)) + return schemaConfMap.containsKey(key.substring(SCHEMA_CONF_PREFIX.length() + 1)); + else + return schemaConfMap.containsKey(key); + } + public Object getSchemaConfValue(String key) { + if (key.contains(SCHEMA_CONF_PREFIX)) + return schemaConfMap.get(key.substring(SCHEMA_CONF_PREFIX.length()+1)); + else + return schemaConfMap.get(key); + } + public void setSchemaConfValue(String key, Object value) { + if (key.contains(SCHEMA_CONF_PREFIX)) + schemaConfMap.put(key.substring(SCHEMA_CONF_PREFIX.length() + 1), value); + else + schemaConfMap.put(key, value); + } + + + ////////////////// + // Get Pulsar client related config + public Map getClientConfMap() { + return this.clientConfMap; + } + public boolean hasClientConfKey(String key) { + if (key.contains(CLIENT_CONF_PREFIX)) + return clientConfMap.containsKey(key.substring(CLIENT_CONF_PREFIX.length() + 1)); + else + return clientConfMap.containsKey(key); + } + public Object getClientConfValue(String key) { + if (key.contains(CLIENT_CONF_PREFIX)) + return clientConfMap.get(key.substring(CLIENT_CONF_PREFIX.length()+1)); + else + return clientConfMap.get(key); + } + public void setClientConfValue(String key, Object value) { + if (key.contains(CLIENT_CONF_PREFIX)) + clientConfMap.put(key.substring(CLIENT_CONF_PREFIX.length() + 1), value); + else + clientConfMap.put(key, value); + } + + + ////////////////// + // Get Pulsar producer related config + public Map getProducerConfMap() { + return this.producerConfMap; + } + public boolean hasProducerConfKey(String key) { + if (key.contains(PRODUCER_CONF_PREFIX)) + return producerConfMap.containsKey(key.substring(PRODUCER_CONF_PREFIX.length() + 1)); + else + return producerConfMap.containsKey(key); + } + public Object getProducerConfValue(String key) { + if (key.contains(PRODUCER_CONF_PREFIX)) + return producerConfMap.get(key.substring(PRODUCER_CONF_PREFIX.length()+1)); + else + return producerConfMap.get(key); + } + public void setProducerConfValue(String key, Object value) { + if (key.contains(PRODUCER_CONF_PREFIX)) + producerConfMap.put(key.substring(PRODUCER_CONF_PREFIX.length()+1), value); + else + producerConfMap.put(key, value); + } + // other producer helper functions ... + public String getProducerName() { + Object confValue = getProducerConfValue( + "producer." + PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.producerName.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getProducerTopicName() { + Object confValue = getProducerConfValue( + "producer." + PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.topicName); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + + + ////////////////// + // Get Pulsar consumer related config + public Map getConsumerConfMap() { + return this.consumerConfMap; + } + public boolean hasConsumerConfKey(String key) { + if (key.contains(CONSUMER_CONF_PREFIX)) + return consumerConfMap.containsKey(key.substring(CONSUMER_CONF_PREFIX.length() + 1)); + else + return consumerConfMap.containsKey(key); + } + public Object getConsumerConfValue(String key) { + if (key.contains(CONSUMER_CONF_PREFIX)) + return consumerConfMap.get(key.substring(CONSUMER_CONF_PREFIX.length() + 1)); + else + return consumerConfMap.get(key); + } + public void setConsumerConfValue(String key, Object value) { + if (key.contains(CONSUMER_CONF_PREFIX)) + consumerConfMap.put(key.substring(CONSUMER_CONF_PREFIX.length() + 1), value); + else + consumerConfMap.put(key, value); + } + // Other consumer helper functions ... + public String getConsumerTopicNames() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicNames.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getConsumerTopicPattern() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getConsumerSubscriptionName() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getConsumerSubscriptionType() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getConsumerName() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.consumerName.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + // NOTE: Below are not a standard Pulsar consumer configuration parameter as + // listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer" + // They're custom-made configuration properties for NB pulsar driver consumer. + public int getConsumerTimeoutSeconds() { + Object confValue = getConsumerConfValue( + "consumer." + PulsarAdapterUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label); + if (confValue == null) + return -1; // infinite + else + return Integer.parseInt(confValue.toString()); + } + + ////////////////// + // Get Pulsar reader related config + public Map getReaderConfMap() { + return this.readerConfMap; + } + public boolean hasReaderConfKey(String key) { + if (key.contains(READER_CONF_PREFIX)) + return readerConfMap.containsKey(key.substring(READER_CONF_PREFIX.length() + 1)); + else + return readerConfMap.containsKey(key); + } + public Object getReaderConfValue(String key) { + if (key.contains(READER_CONF_PREFIX)) + return readerConfMap.get(key.substring(READER_CONF_PREFIX.length() + 1)); + else + return readerConfMap.get(key); + } + public void setReaderConfValue(String key, Object value) { + if (key.contains(READER_CONF_PREFIX)) + readerConfMap.put(key.substring(READER_CONF_PREFIX.length() + 1), value); + else + readerConfMap.put(key, value); + } + // Other reader helper functions ... + public String getReaderTopicName() { + Object confValue = getReaderConfValue( + "reader." + PulsarAdapterUtil.READER_CONF_STD_KEY.topicName.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + public String getReaderName() { + Object confValue = getReaderConfValue( + "reader." + PulsarAdapterUtil.READER_CONF_STD_KEY.readerName.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } + // NOTE: Below are not a standard Pulsar reader configuration parameter as + // listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#reader" + // They're custom-made configuration properties for NB pulsar driver reader. + public String getStartMsgPosStr() { + Object confValue = getReaderConfValue( + "reader." + PulsarAdapterUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label); + if (confValue == null) + return ""; + else + return confValue.toString(); + } +} diff --git a/adapter-pulsar/src/main/resources/admin-namespace.yaml b/adapter-pulsar/src/main/resources/admin-namespace.yaml new file mode 100644 index 000000000..b80e9cff2 --- /dev/null +++ b/adapter-pulsar/src/main/resources/admin-namespace.yaml @@ -0,0 +1,16 @@ +bindings: + # 20 topics: 10 tenants, 2 namespaces/tenant + tenant: Mod(20); Div(2); ToString(); Prefix("tnt") + namespace: Mod(2); ToString(); Prefix("ns") + +params: + async_api: "false" + admin_delop: "true" + +blocks: + admin-namespace-block: + tags: + phase: admin-namespace + ops: + op1: + AdminNamespace: "{tenant}/{namespace}" diff --git a/adapter-pulsar/src/main/resources/admin-tenant.yaml b/adapter-pulsar/src/main/resources/admin-tenant.yaml new file mode 100644 index 000000000..2278e8c41 --- /dev/null +++ b/adapter-pulsar/src/main/resources/admin-tenant.yaml @@ -0,0 +1,17 @@ +bindings: + # 10 tenants + tenant: Mod(10); ToString(); Prefix("tnt") + +params: + async_api: "false" + admin_delop: "true" + +blocks: + admin-tenant-block: + tags: + phase: admin-tenant + ops: + op1: + AdminTenant: "{tenant}" + admin_roles: "" + allowed_clusters: "" diff --git a/adapter-pulsar/src/main/resources/admin-topic.yaml b/adapter-pulsar/src/main/resources/admin-topic.yaml new file mode 100644 index 000000000..a5f81d0dd --- /dev/null +++ b/adapter-pulsar/src/main/resources/admin-topic.yaml @@ -0,0 +1,19 @@ +bindings: + # 100 topics: 10 tenants, 2 namespaces/tenant, 5 topics/namespace + tenant: Mod(100); Div(10); ToString(); Prefix("tnt") + namespace: Mod(10); Div(5); ToString(); Prefix("ns") + topic: Mod(5); ToString(); Prefix("tp") + +params: + async_api: "false" + admin_delop: "true" + +blocks: + admin-topic-block: + tags: + phase: admin-topic + ops: + op1: + AdminTopic: "{tenant}/{namespace}/{topic}" + enable_partition: "true" + partition_num: "5" diff --git a/adapter-pulsar/src/main/resources/bindingtest.yaml b/adapter-pulsar/src/main/resources/bindingtest.yaml new file mode 100644 index 000000000..e687f5fa4 --- /dev/null +++ b/adapter-pulsar/src/main/resources/bindingtest.yaml @@ -0,0 +1,4 @@ +bindings: + tenant: Mod(100); Div(10); ToString(); Prefix("tnt") + namespace: Mod(10); Div(5); ToString(); Prefix("ns") + topic: Mod(5); ToString(); Prefix("tp") diff --git a/adapter-pulsar/src/main/resources/config.properties b/adapter-pulsar/src/main/resources/config.properties new file mode 100644 index 000000000..37be64384 --- /dev/null +++ b/adapter-pulsar/src/main/resources/config.properties @@ -0,0 +1,52 @@ +### Schema related configurations - schema.xxx +# valid types: +# - primitive type (https://pulsar.apache.org/docs/en/schema-understand/#primitive-type) +# - keyvalue (https://pulsar.apache.org/docs/en/schema-understand/#keyvalue) +# - strut (complex type) (https://pulsar.apache.org/docs/en/schema-understand/#struct) +# avro, json, protobuf +# +# TODO: as a starting point, only supports the following types +# 1) primitive types, including bytearray (byte[]) which is default, for messages without schema +# 2) Avro for messages with schema +#schema.type=avro +#schema.definition=file:///Users/yabinmeng/DataStax/MyNoSQLBench/nosqlbench/driver-pulsar/src/main/resources/activities/iot-example.avsc +schema.type= +schema.definition= + + +### Pulsar client related configurations - client.xxx +# http://pulsar.apache.org/docs/en/client-libraries-java/#client +client.connectionTimeoutMs=5000 +client.authPluginClassName=org.apache.pulsar.client.impl.auth.AuthenticationToken +# Cluster admin +client.authParams= +client.tlsAllowInsecureConnection=true + + +### Producer related configurations (global) - producer.xxx +# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer +producer.producerName= +producer.topicName= +producer.sendTimeoutMs= +producer.blockIfQueueFull=true +producer.maxPendingMessages=5000 +producer.batchingMaxMessages=5000 + + +### Consumer related configurations (global) - consumer.xxx +# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer +consumer.topicNames= +consumer.topicsPattern= +consumer.subscriptionName= +consumer.subscriptionType= +consumer.consumerName= +consumer.receiverQueueSize= + + +### Reader related configurations (global) - reader.xxx +# https://pulsar.apache.org/docs/en/client-libraries-java/#reader +# - valid Pos: earliest, latest, custom::file://// +reader.topicName= +reader.receiverQueueSize= +reader.readerName= +reader.startMessagePos=earliest diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsAction.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsAction.java deleted file mode 100644 index 3a3ef9b2a..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsAction.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms; - -import com.codahale.metrics.Timer; -import io.nosqlbench.driver.jms.ops.JmsOp; -import io.nosqlbench.engine.api.activityapi.core.SyncAction; -import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorDetail; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.function.LongFunction; - -public class JmsAction implements SyncAction { - - private final static Logger logger = LogManager.getLogger(JmsAction.class); - - private final JmsActivity activity; - private final int slot; - - int maxTries; - - public JmsAction(JmsActivity activity, int slot) { - this.activity = activity; - this.slot = slot; - this.maxTries = activity.getActivityDef().getParams().getOptionalInteger("maxtries").orElse(10); - } - - @Override - public void init() { } - - @Override - public int runCycle(long cycle) { - // let's fail the action if some async operation failed - activity.failOnAsyncOperationFailure(); - - long start = System.nanoTime(); - - JmsOp jmsOp; - try (Timer.Context ctx = activity.getBindTimer().time()) { - LongFunction readyJmsOp = activity.getSequencer().apply(cycle); - jmsOp = readyJmsOp.apply(cycle); - } catch (Exception bindException) { - // if diagnostic mode ... - activity.getErrorhandler().handleError(bindException, cycle, 0); - throw new RuntimeException( - "while binding request in cycle " + cycle + ": " + bindException.getMessage(), bindException - ); - } - - for (int i = 0; i < maxTries; i++) { - Timer.Context ctx = activity.getExecuteTimer().time(); - try { - // it is up to the jmsOp to call Context#close when the activity is executed - // this allows us to track time for async operations - jmsOp.run(ctx::close); - break; - } catch (RuntimeException err) { - ErrorDetail errorDetail = activity - .getErrorhandler() - .handleError(err, cycle, System.nanoTime() - start); - if (!errorDetail.isRetryable()) { - break; - } - } - } - - return 0; - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java deleted file mode 100644 index 2da3a440c..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.Timer; -import com.datastax.oss.pulsar.jms.PulsarConnectionFactory; -import io.nosqlbench.driver.jms.conn.JmsConnInfo; -import io.nosqlbench.driver.jms.conn.JmsPulsarConnInfo; -import io.nosqlbench.driver.jms.ops.JmsOp; -import io.nosqlbench.driver.jms.util.JmsUtil; -import io.nosqlbench.driver.jms.util.PulsarConfig; -import io.nosqlbench.engine.api.activityapi.errorhandling.modular.NBErrorHandler; -import io.nosqlbench.engine.api.activityapi.planning.OpSequence; -import io.nosqlbench.api.engine.activityimpl.ActivityDef; -import io.nosqlbench.engine.api.activityimpl.OpDispenser; -import io.nosqlbench.engine.api.activityimpl.SimpleActivity; -import io.nosqlbench.api.engine.metrics.ActivityMetrics; -import org.apache.commons.lang3.StringUtils; - -import javax.jms.Destination; -import javax.jms.JMSContext; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - - -public class JmsActivity extends SimpleActivity { - - private final ConcurrentHashMap jmsDestinations = new ConcurrentHashMap<>(); - - private String jmsProviderType; - private JmsConnInfo jmsConnInfo; - - private JMSContext jmsContext; - - private OpSequence> sequence; - private volatile Throwable asyncOperationFailure; - private NBErrorHandler errorhandler; - - private Timer bindTimer; - private Timer executeTimer; - private Counter bytesCounter; - private Histogram messagesizeHistogram; - - public JmsActivity(ActivityDef activityDef) { - super(activityDef); - } - - @Override - public void initActivity() { - super.initActivity(); - - // default JMS type: Pulsar - // - currently this is the only supported JMS provider - jmsProviderType = - activityDef.getParams() - .getOptionalString(JmsUtil.JMS_PROVIDER_TYPE_KEY_STR) - .orElse(JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label); - - // "Pulsar" as the JMS provider - if (StringUtils.equalsIgnoreCase(jmsProviderType, JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label )) { - - String webSvcUrl = - activityDef.getParams() - .getOptionalString(JmsUtil.JMS_PULSAR_PROVIDER_WEB_URL_KEY_STR) - .orElse("http://localhost:8080"); - String pulsarSvcUrl = - activityDef.getParams() - .getOptionalString(JmsUtil.JMS_PULSAR_PROVIDER_SVC_URL_KEY_STR) - .orElse("pulsar://localhost:6650"); - - if (StringUtils.isAnyBlank(webSvcUrl, pulsarSvcUrl)) { - throw new RuntimeException("For \"" + JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label + "\" type, " + - "\"" + JmsUtil.JMS_PULSAR_PROVIDER_WEB_URL_KEY_STR + "\" and " + - "\"" + JmsUtil.JMS_PULSAR_PROVIDER_SVC_URL_KEY_STR + "\" parameters are manadatory!"); - } - - // Check if extra Pulsar config. file is in place - // - default file: "pulsar_config.properties" under the current directory - String pulsarCfgFile = - activityDef.getParams() - .getOptionalString(JmsUtil.JMS_PULSAR_PROVIDER_CFG_FILE_KEY_STR) - .orElse(JmsUtil.JMS_PULSAR_PROVIDER_DFT_CFG_FILE_NAME); - - PulsarConfig pulsarConfig = new PulsarConfig(pulsarCfgFile); - - jmsConnInfo = new JmsPulsarConnInfo(jmsProviderType, webSvcUrl, pulsarSvcUrl, pulsarConfig); - } - else { - throw new RuntimeException("Unsupported JMS driver type : " + jmsProviderType); - } - - PulsarConnectionFactory factory; - factory = new PulsarConnectionFactory(jmsConnInfo.getJmsConnConfig()); - this.jmsContext = factory.createContext(); - - bindTimer = ActivityMetrics.timer(activityDef, "bind", this.getHdrDigits()); - executeTimer = ActivityMetrics.timer(activityDef, "execute", this.getHdrDigits()); - bytesCounter = ActivityMetrics.counter(activityDef, "bytes"); - messagesizeHistogram = ActivityMetrics.histogram(activityDef, "messagesize", this.getHdrDigits()); - - if (StringUtils.equalsIgnoreCase(jmsProviderType, JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label )) { - this.sequence = createOpSequence((ot) -> new ReadyPulsarJmsOp(ot, this), false, Optional.empty()); - } - - setDefaultsFromOpSequence(sequence); - onActivityDefUpdate(activityDef); - - this.errorhandler = new NBErrorHandler( - () -> activityDef.getParams().getOptionalString("errors").orElse("stop"), - this::getExceptionMetrics - ); - } - - private static String buildCacheKey(String... keyParts) { - return String.join("::", keyParts); - } - - /** - * If the JMS destination that corresponds to a topic exists, reuse it; Otherwise, create it - */ - public Destination getOrCreateJmsDestination(String jmsDestinationType, String destName) { - String destinationCacheKey = buildCacheKey(jmsDestinationType, destName); - Destination destination = jmsDestinations.get(destinationCacheKey); - - if ( destination == null ) { - // TODO: should we match Persistent/Non-peristent JMS Delivery mode with - // Pulsar Persistent/Non-prsistent topic? - if (StringUtils.equalsIgnoreCase(jmsDestinationType, JmsUtil.JMS_DESTINATION_TYPES.QUEUE.label)) { - destination = jmsContext.createQueue(destName); - } else if (StringUtils.equalsIgnoreCase(jmsDestinationType, JmsUtil.JMS_DESTINATION_TYPES.TOPIC.label)) { - destination = jmsContext.createTopic(destName); - } - - jmsDestinations.put(destinationCacheKey, destination); - } - - return destination; - } - - @Override - public synchronized void onActivityDefUpdate(ActivityDef activityDef) { super.onActivityDefUpdate(activityDef); } - public OpSequence> getSequencer() { return sequence; } - - public String getJmsProviderType() { return jmsProviderType; } - public JmsConnInfo getJmsConnInfo() { return jmsConnInfo; } - public JMSContext getJmsContext() { return jmsContext; } - - public Timer getBindTimer() { return bindTimer; } - public Timer getExecuteTimer() { return this.executeTimer; } - public Counter getBytesCounter() { return bytesCounter; } - public Histogram getMessagesizeHistogram() { return messagesizeHistogram; } - - public NBErrorHandler getErrorhandler() { return errorhandler; } - - public void failOnAsyncOperationFailure() { - if (asyncOperationFailure != null) { - throw new RuntimeException(asyncOperationFailure); - } - } - public void asyncOperationFailed(Throwable ex) { - this.asyncOperationFailure = ex; - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivityType.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivityType.java deleted file mode 100644 index b964516ea..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivityType.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms; - -import io.nosqlbench.engine.api.activityapi.core.Action; -import io.nosqlbench.engine.api.activityapi.core.ActionDispenser; -import io.nosqlbench.engine.api.activityapi.core.ActivityType; -import io.nosqlbench.api.engine.activityimpl.ActivityDef; -import io.nosqlbench.nb.annotations.Service; - -@Service(value = ActivityType.class, selector = "jms") -public class JmsActivityType implements ActivityType { - @Override - public ActionDispenser getActionDispenser(JmsActivity activity) { - return new PulsarJmsActionDispenser(activity); - } - - @Override - public JmsActivity getActivity(ActivityDef activityDef) { - return new JmsActivity(activityDef); - } - - private static class PulsarJmsActionDispenser implements ActionDispenser { - private final JmsActivity activity; - public PulsarJmsActionDispenser(JmsActivity activity) { - this.activity = activity; - } - - @Override - public Action getAction(int slot) { - return new JmsAction(activity, slot); - } - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyJmsOp.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyJmsOp.java deleted file mode 100644 index bd532033f..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyJmsOp.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms; - -import io.nosqlbench.driver.jms.ops.JmsOp; -import io.nosqlbench.driver.jms.util.JmsUtil; -import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser; -import io.nosqlbench.engine.api.templating.CommandTemplate; -import org.apache.commons.lang3.BooleanUtils; - -import java.util.function.LongFunction; - -abstract public class ReadyJmsOp extends BaseOpDispenser { - - protected final OpTemplate optpl; - protected final CommandTemplate cmdTpl; - protected final JmsActivity jmsActivity; - - protected final String stmtOpType; - protected LongFunction asyncApiFunc; - protected LongFunction jmsDestinationTypeFunc; - - protected final LongFunction opFunc; - - public ReadyJmsOp(OpTemplate opTemplate, JmsActivity jmsActivity) { - super(opTemplate); - this.optpl = opTemplate; - this.cmdTpl = new CommandTemplate(optpl); - this.jmsActivity = jmsActivity; - - if (!cmdTpl.containsKey("optype") || !cmdTpl.isStatic("optype")) { - throw new RuntimeException("Statement parameter \"optype\" must be static and have a valid value!"); - } - this.stmtOpType = cmdTpl.getStatic("optype"); - - // Global/Doc-level parameter: async_api - if (cmdTpl.containsKey(JmsUtil.ASYNC_API_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.ASYNC_API_KEY_STR)) { - boolean value = BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.ASYNC_API_KEY_STR)); - this.asyncApiFunc = (l) -> value; - } else { - throw new RuntimeException("\"" + JmsUtil.ASYNC_API_KEY_STR + "\" parameter cannot be dynamic!"); - } - } - - // Global/Doc-level parameter: jms_desitation_type - // - queue: point-to-point - // - topic: pub/sub - if (cmdTpl.containsKey(JmsUtil.JMS_DESTINATION_TYPE_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_DESTINATION_TYPE_KEY_STR)) { - jmsDestinationTypeFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_DESTINATION_TYPE_KEY_STR); - } else { - throw new RuntimeException("\"" + JmsUtil.JMS_DESTINATION_TYPE_KEY_STR + "\" parameter cannot be dynamic!"); - } - } - - this.opFunc = resolveJms(); - } - - public JmsOp apply(long value) { return opFunc.apply(value); } - - abstract LongFunction resolveJms(); -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyPulsarJmsOp.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyPulsarJmsOp.java deleted file mode 100644 index 72c397a97..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyPulsarJmsOp.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms; - -import io.nosqlbench.driver.jms.ops.JmsMsgReadMapper; -import io.nosqlbench.driver.jms.ops.JmsMsgSendMapper; -import io.nosqlbench.driver.jms.ops.JmsOp; -import io.nosqlbench.driver.jms.util.JmsHeaderLongFunc; -import io.nosqlbench.driver.jms.util.JmsUtil; -import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; - -import javax.jms.DeliveryMode; -import javax.jms.Destination; -import javax.jms.JMSRuntimeException; -import javax.jms.Message; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.function.LongFunction; -import java.util.stream.Collectors; - -public class ReadyPulsarJmsOp extends ReadyJmsOp { - - public ReadyPulsarJmsOp(OpTemplate opTemplate, JmsActivity jmsActivity) { - super(opTemplate, jmsActivity); - } - - public LongFunction resolveJms() { - // Global/Doc-level parameter: topic_uri - LongFunction topicUriFunc = (l) -> null; - if (cmdTpl.containsKey(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR)) { - topicUriFunc = (l) -> cmdTpl.getStatic(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR); - } else { - topicUriFunc = (l) -> cmdTpl.getDynamic(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR, l); - } - } - - // Global: JMS destination - LongFunction jmsDestinationFunc; - try { - LongFunction finalTopicUriFunc = topicUriFunc; - jmsDestinationFunc = (l) -> jmsActivity.getOrCreateJmsDestination( - jmsDestinationTypeFunc.apply(l), - finalTopicUriFunc.apply(l)); - } - catch (JMSRuntimeException ex) { - throw new RuntimeException("Unable to create JMS destination!"); - } - - if (StringUtils.equalsIgnoreCase(stmtOpType, JmsUtil.OP_TYPES.MSG_SEND.label)) { - return resolveMsgSend(asyncApiFunc, jmsDestinationFunc); - } else if (StringUtils.equalsIgnoreCase(stmtOpType, JmsUtil.OP_TYPES.MSG_READ.label)) { - return resolveMsgRead(asyncApiFunc, jmsDestinationFunc); - } else { - throw new RuntimeException("Unsupported JMS operation type"); - } - } - - private LongFunction resolveMsgSend( - LongFunction async_api_func, - LongFunction jmsDestinationFunc - ) { - JmsHeaderLongFunc jmsHeaderLongFunc = new JmsHeaderLongFunc(); - - // JMS header: delivery mode - LongFunction msgDeliveryModeFunc = (l) -> DeliveryMode.PERSISTENT; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label)) { - msgDeliveryModeFunc = (l) -> NumberUtils.toInt(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label)); - } - else { - msgDeliveryModeFunc = (l) -> NumberUtils.toInt(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label, l)); - } - } - jmsHeaderLongFunc.setDeliveryModeFunc(msgDeliveryModeFunc); - - // JMS header: message priority - LongFunction msgPriorityFunc = (l) -> Message.DEFAULT_PRIORITY; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label)) { - msgPriorityFunc = (l) -> NumberUtils.toInt(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label)); - } - else { - msgPriorityFunc = (l) -> NumberUtils.toInt(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label, l)); - } - } - jmsHeaderLongFunc.setMsgPriorityFunc(msgPriorityFunc); - - // JMS header: message TTL - LongFunction msgTtlFunc = (l) -> Message.DEFAULT_TIME_TO_LIVE; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label)) { - msgTtlFunc = (l) -> NumberUtils.toLong(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label)); - } - else { - msgTtlFunc = (l) -> NumberUtils.toLong(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label, l)); - } - } - jmsHeaderLongFunc.setMsgTtlFunc(msgTtlFunc); - - // JMS header: message delivery delay - LongFunction msgDeliveryDelayFunc = (l) -> Message.DEFAULT_DELIVERY_DELAY; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label)) { - msgDeliveryDelayFunc = (l) -> NumberUtils.toLong(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label)); - } - else { - msgDeliveryDelayFunc = (l) -> NumberUtils.toLong(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label, l)); - } - } - jmsHeaderLongFunc.setMsgDeliveryDelayFunc(msgDeliveryDelayFunc); - - // JMS header: disable message timestamp - LongFunction disableMsgTimestampFunc = (l) -> false; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label)) { - disableMsgTimestampFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label)); - } - else { - disableMsgTimestampFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label, l)); - } - } - jmsHeaderLongFunc.setDisableMsgTimestampFunc(disableMsgTimestampFunc); - - // JMS header: disable message ID - LongFunction disableMsgIdFunc = (l) -> false; - if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label)) { - if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label)) { - disableMsgIdFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label)); - } - else { - disableMsgIdFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label, l)); - } - } - jmsHeaderLongFunc.setDisableMsgIdFunc(disableMsgIdFunc); - - // JMS message properties - String jmsMsgPropertyListStr = ""; - if (cmdTpl.containsKey(JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR)) { - jmsMsgPropertyListStr = cmdTpl.getStatic(JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR); - } else { - throw new RuntimeException("\"" + JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR + "\" parameter cannot be dynamic!"); - } - } - - Map jmsMsgProperties = new HashMap<>(); - if ( !StringUtils.isEmpty(jmsMsgPropertyListStr) ) { - jmsMsgProperties = Arrays.stream(jmsMsgPropertyListStr.split(";")) - .map(s -> s.split("=", 2)) - .collect(Collectors.toMap(a -> a[0], a -> a.length > 1 ? a[1] : "")); - } - - LongFunction msgBodyFunc; - if (cmdTpl.containsKey(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR)) { - msgBodyFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR)) { - msgBodyFunc = (l) -> cmdTpl.getDynamic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR, l); - } else { - msgBodyFunc = (l) -> null; - } - } else { - throw new RuntimeException("JMS message send:: \"msg_body\" field must be specified!"); - } - - return new JmsMsgSendMapper( - jmsActivity, - async_api_func, - jmsDestinationFunc, - jmsHeaderLongFunc, - jmsMsgProperties, - msgBodyFunc); - } - - private LongFunction resolveMsgRead( - LongFunction async_api_func, - LongFunction jmsDestinationFunc - ) { - // For Pulsar JMS, make "durable" as the default - LongFunction jmsConsumerDurableFunc = (l) -> true; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR)) { - jmsConsumerDurableFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR)); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR)) { - jmsConsumerDurableFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR, l)); - } - } - - LongFunction jmsConsumerSharedFunc = (l) -> true; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR)) { - jmsConsumerSharedFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR)); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR)) { - jmsConsumerSharedFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR, l)); - } - } - - LongFunction jmsMsgSubscriptionFunc = (l) -> ""; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR)) { - jmsMsgSubscriptionFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR)) { - jmsMsgSubscriptionFunc = (l) -> cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR, l); - } - } - - LongFunction jmsMsgReadSelectorFunc = (l) -> ""; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR)) { - jmsMsgReadSelectorFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR)) { - jmsMsgReadSelectorFunc = (l) -> cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR, l); - } - } - - LongFunction jmsMsgNoLocalFunc = (l) -> true; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR)) { - jmsMsgNoLocalFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR)); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR)) { - jmsMsgNoLocalFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR, l)); - } - } - - LongFunction jmsReadTimeoutFunc = (l) -> 0L; - if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR)) { - if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR)) { - jmsReadTimeoutFunc = (l) -> NumberUtils.toLong(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR)); - } else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR)) { - jmsReadTimeoutFunc = (l) -> NumberUtils.toLong(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR, l)); - } - } - - return new JmsMsgReadMapper( - jmsActivity, - async_api_func, - jmsDestinationFunc, - jmsConsumerDurableFunc, - jmsConsumerSharedFunc, - jmsMsgSubscriptionFunc, - jmsMsgReadSelectorFunc, - jmsMsgNoLocalFunc, - jmsReadTimeoutFunc); - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsConnInfo.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsConnInfo.java deleted file mode 100644 index 738f83678..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsConnInfo.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.conn; - -import java.util.HashMap; -import java.util.Map; - -public class JmsConnInfo { - - protected final String jmsProviderType; - protected final Map jmsConnConfig; - - protected JmsConnInfo(String jmsProviderType) { - this.jmsProviderType = jmsProviderType; - this.jmsConnConfig = new HashMap<>(); - } - - public Map getJmsConnConfig() { return this.jmsConnConfig; } - public void resetJmsConnConfig() { this.jmsConnConfig.clear(); } - public void addJmsConnConfigItems(Map cfgItems) { this.jmsConnConfig.putAll(cfgItems); } - public void addJmsConnConfigItem(String key, Object value) { this.jmsConnConfig.put(key, value); } - public void removeJmsConnConfigItem(String key) { this.jmsConnConfig.remove(key); } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsPulsarConnInfo.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsPulsarConnInfo.java deleted file mode 100644 index f1e09fe78..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsPulsarConnInfo.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.conn; - -import io.nosqlbench.driver.jms.util.PulsarConfig; - -import java.util.Map; - -public class JmsPulsarConnInfo extends JmsConnInfo { - - private final String webSvcUrl; - private final String pulsarSvcUrl; - private final PulsarConfig extraPulsarConfig; - - public JmsPulsarConnInfo(String jmsProviderType, String webSvcUrl, String pulsarSvcUrl, PulsarConfig pulsarConfig) { - super(jmsProviderType); - - this.webSvcUrl = webSvcUrl; - this.pulsarSvcUrl = pulsarSvcUrl; - this.extraPulsarConfig = pulsarConfig; - - this.addJmsConnConfigItem("webServiceUrl", this.webSvcUrl); - this.addJmsConnConfigItem("brokerServiceUrl", this.pulsarSvcUrl); - - Map clientCfgMap = this.extraPulsarConfig.getClientConfMap(); - if (!clientCfgMap.isEmpty()) { - this.addJmsConnConfigItems(clientCfgMap); - } - - Map producerCfgMap = this.extraPulsarConfig.getProducerConfMap(); - if (!producerCfgMap.isEmpty()) { - this.addJmsConnConfigItem("producerConfig", producerCfgMap); - } - - Map consumerCfgMap = this.extraPulsarConfig.getConsumerConfMap(); - if (!consumerCfgMap.isEmpty()) { - this.addJmsConnConfigItem("consumerConfig", consumerCfgMap); - } - } - - public String getWebSvcUrl() { return this.webSvcUrl; } - public String getPulsarSvcUrl() { return this.pulsarSvcUrl; } - public PulsarConfig getExtraPulsarConfig() { return this.extraPulsarConfig; } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadMapper.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadMapper.java deleted file mode 100644 index 0108c0608..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadMapper.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.ops; - -import io.nosqlbench.driver.jms.JmsActivity; - -import javax.jms.Destination; -import java.util.function.LongFunction; - -/** - * This maps a set of specifier functions to a pulsar operation. The pulsar operation contains - * enough state to define a pulsar operation such that it can be executed, measured, and possibly - * retried if needed. - * - * This function doesn't act *as* the operation. It merely maps the construction logic into - * a simple functional type, given the component functions. - * - * For additional parameterization, the command template is also provided. - */ -public class JmsMsgReadMapper extends JmsOpMapper { - - private final LongFunction jmsConsumerDurableFunc; - private final LongFunction jmsConsumerSharedFunc; - private final LongFunction jmsMsgSubscriptionFunc; - private final LongFunction jmsMsgReadSelectorFunc; - private final LongFunction jmsMsgNoLocalFunc; - private final LongFunction jmsReadTimeoutFunc; - - public JmsMsgReadMapper(JmsActivity jmsActivity, - LongFunction asyncApiFunc, - LongFunction jmsDestinationFunc, - LongFunction jmsConsumerDurableFunc, - LongFunction jmsConsumerSharedFunc, - LongFunction jmsMsgSubscriptionFunc, - LongFunction jmsMsgReadSelectorFunc, - LongFunction jmsMsgNoLocalFunc, - LongFunction jmsReadTimeoutFunc) { - super(jmsActivity, asyncApiFunc, jmsDestinationFunc); - - this.jmsConsumerDurableFunc = jmsConsumerDurableFunc; - this.jmsConsumerSharedFunc = jmsConsumerSharedFunc; - this.jmsMsgSubscriptionFunc = jmsMsgSubscriptionFunc; - this.jmsMsgReadSelectorFunc = jmsMsgReadSelectorFunc; - this.jmsMsgNoLocalFunc = jmsMsgNoLocalFunc; - this.jmsReadTimeoutFunc = jmsReadTimeoutFunc; - } - - @Override - public JmsOp apply(long value) { - boolean asyncApi = asyncApiFunc.apply(value); - Destination jmsDestination = jmsDestinationFunc.apply(value); - boolean jmsConsumerDurable = jmsConsumerDurableFunc.apply(value); - boolean jmsConsumerShared = jmsConsumerSharedFunc.apply(value); - String jmsMsgSubscription = jmsMsgSubscriptionFunc.apply(value); - String jmsMsgReadSelector = jmsMsgReadSelectorFunc.apply(value); - boolean jmsMsgNoLocal = jmsMsgNoLocalFunc.apply(value); - long jmsReadTimeout = jmsReadTimeoutFunc.apply(value); - - // Default to NO read timeout - if (jmsReadTimeout < 0) jmsReadTimeout = 0; - - return new JmsMsgReadOp( - jmsActivity, - asyncApi, - jmsDestination, - jmsConsumerDurable, - jmsConsumerShared, - jmsMsgSubscription, - jmsMsgReadSelector, - jmsMsgNoLocal, - jmsReadTimeout - ); - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadOp.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadOp.java deleted file mode 100644 index de92f73c5..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadOp.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.ops; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Histogram; -import io.nosqlbench.driver.jms.JmsActivity; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import javax.jms.*; - -public class JmsMsgReadOp extends JmsTimeTrackOp { - - private final static Logger logger = LogManager.getLogger(JmsMsgReadOp.class); - - private final JmsActivity jmsActivity; - private final boolean asyncJmsOp; - private final Destination jmsDestination; - - private final JMSContext jmsContext; - private final JMSConsumer jmsConsumer; - private final boolean jmsConsumerDurable; - private final boolean jmsConsumerShared; - private final String jmsMsgSubscrption; - private final String jmsMsgReadSelector; - private final boolean jmsMsgNoLocal; - private final long jmsReadTimeout; - - private final Counter bytesCounter; - private final Histogram messagesizeHistogram; - - public JmsMsgReadOp(JmsActivity jmsActivity, - boolean asyncJmsOp, - Destination jmsDestination, - boolean jmsConsumerDurable, - boolean jmsConsumerShared, - String jmsMsgSubscrption, - String jmsMsgReadSelector, - boolean jmsMsgNoLocal, - long jmsReadTimeout) { - this.jmsActivity = jmsActivity; - this.asyncJmsOp = asyncJmsOp; - this.jmsDestination = jmsDestination; - this.jmsConsumerDurable = jmsConsumerDurable; - this.jmsConsumerShared = jmsConsumerShared; - this.jmsMsgReadSelector = jmsMsgReadSelector; - this.jmsMsgSubscrption = jmsMsgSubscrption; - this.jmsMsgNoLocal = jmsMsgNoLocal; - this.jmsReadTimeout = jmsReadTimeout; - - this.jmsContext = jmsActivity.getJmsContext(); - this.jmsConsumer = createJmsConsumer(); - - this.bytesCounter = jmsActivity.getBytesCounter(); - this.messagesizeHistogram = jmsActivity.getMessagesizeHistogram(); - } - - private JMSConsumer createJmsConsumer() { - JMSConsumer jmsConsumer; - - try { - if (jmsConsumerDurable) { - if (jmsConsumerShared) - jmsConsumer = jmsContext.createSharedDurableConsumer((Topic) jmsDestination, jmsMsgSubscrption, jmsMsgReadSelector); - else - jmsConsumer = jmsContext.createDurableConsumer((Topic) jmsDestination, jmsMsgSubscrption, jmsMsgReadSelector, jmsMsgNoLocal); - } else { - if (jmsConsumerShared) - jmsConsumer = jmsContext.createSharedConsumer((Topic) jmsDestination, jmsMsgSubscrption, jmsMsgReadSelector); - else - jmsConsumer = jmsContext.createConsumer(jmsDestination, jmsMsgReadSelector, jmsMsgNoLocal); - } - } - catch (InvalidDestinationRuntimeException invalidDestinationRuntimeException) { - throw new RuntimeException("Failed to create JMS consumer: invalid destination!"); - } - catch (InvalidSelectorRuntimeException invalidSelectorRuntimeException) { - throw new RuntimeException("Failed to create JMS consumer: invalid message selector!"); - } - catch (JMSRuntimeException jmsRuntimeException) { - jmsRuntimeException.printStackTrace(); - throw new RuntimeException("Failed to create JMS consumer: runtime internal error!"); - } - - // TODO: async consumer -// if (this.asyncJmsOp) { -// jmsConsumer.setMessageListener(); -// } - - return jmsConsumer; - } - - @Override - public void run() { - // FIXME: jmsReadTimeout being 0 behaves like receiveNoWait() instead of waiting indefinitley - Message receivedMsg = jmsConsumer.receive(jmsReadTimeout); - try { - if (receivedMsg != null) { - receivedMsg.acknowledge(); - byte[] receivedMsgBody = receivedMsg.getBody(byte[].class); - - if (logger.isDebugEnabled()) { - logger.debug("received msg-payload={}", new String(receivedMsgBody)); - } - - int messagesize = receivedMsgBody.length; - bytesCounter.inc(messagesize); - messagesizeHistogram.update(messagesize); - } - } catch (JMSException e) { - e.printStackTrace(); - throw new RuntimeException("Failed to acknowledge the received JMS message."); - } - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendMapper.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendMapper.java deleted file mode 100644 index fb649f013..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendMapper.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.ops; - -import io.nosqlbench.driver.jms.JmsActivity; -import io.nosqlbench.driver.jms.util.JmsHeader; -import io.nosqlbench.driver.jms.util.JmsHeaderLongFunc; - -import javax.jms.Destination; -import java.util.Map; -import java.util.function.LongFunction; - -/** - * This maps a set of specifier functions to a pulsar operation. The pulsar operation contains - * enough state to define a pulsar operation such that it can be executed, measured, and possibly - * retried if needed. - * - * This function doesn't act *as* the operation. It merely maps the construction logic into - * a simple functional type, given the component functions. - * - * For additional parameterization, the command template is also provided. - */ -public class JmsMsgSendMapper extends JmsOpMapper { - private final JmsHeaderLongFunc jmsHeaderLongFunc; - private final Map jmsMsgProperties; - private final LongFunction msgBodyFunc; - - public JmsMsgSendMapper(JmsActivity jmsActivity, - LongFunction asyncApiFunc, - LongFunction jmsDestinationFunc, - JmsHeaderLongFunc jmsHeaderLongFunc, - Map jmsMsgProperties, - LongFunction msgBodyFunc) { - super(jmsActivity, asyncApiFunc, jmsDestinationFunc); - - this.jmsHeaderLongFunc = jmsHeaderLongFunc; - this.jmsMsgProperties = jmsMsgProperties; - this.msgBodyFunc = msgBodyFunc; - } - - @Override - public JmsOp apply(long value) { - boolean asyncApi = asyncApiFunc.apply(value); - Destination jmsDestination = jmsDestinationFunc.apply(value); - JmsHeader jmsHeader = (JmsHeader)jmsHeaderLongFunc.apply(value); - String msgBody = msgBodyFunc.apply(value); - - return new JmsMsgSendOp( - jmsActivity, - asyncApi, - jmsDestination, - jmsHeader, - jmsMsgProperties, - msgBody - ); - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendOp.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendOp.java deleted file mode 100644 index 2f432502c..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendOp.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.ops; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Histogram; -import io.nosqlbench.driver.jms.JmsActivity; -import io.nosqlbench.driver.jms.util.JmsHeader; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import javax.jms.*; -import java.nio.charset.StandardCharsets; -import java.util.Iterator; -import java.util.Map; - -public class JmsMsgSendOp extends JmsTimeTrackOp { - - private final static Logger logger = LogManager.getLogger(JmsMsgSendOp.class); - - private final JmsActivity jmsActivity; - private final boolean asyncJmsOp; - private final Destination jmsDestination; - private final JmsHeader jmsHeader; - private final Map jmsMsgProperties; - - private final JMSContext jmsContext; - private final JMSProducer jmsProducer; - private final String msgBody; - - private final Counter bytesCounter; - private final Histogram messagesizeHistogram; - - public JmsMsgSendOp(JmsActivity jmsActivity, - boolean asyncJmsOp, - Destination jmsDestination, - JmsHeader jmsHeader, - Map jmsMsgProperties, - String msgBody) { - this.jmsActivity = jmsActivity; - this.asyncJmsOp = asyncJmsOp; - this.jmsDestination = jmsDestination; - - this.jmsHeader = jmsHeader; - this.jmsMsgProperties = jmsMsgProperties; - this.msgBody = msgBody; - - if (!jmsHeader.isValidHeader()) { - throw new RuntimeException(jmsHeader.getInvalidJmsHeaderMsgText()); - } - - if ((msgBody == null) || msgBody.isEmpty()) { - throw new RuntimeException("JMS message body can't be empty!"); - } - - this.jmsContext = jmsActivity.getJmsContext(); - this.jmsProducer = createJmsProducer(); - - this.bytesCounter = jmsActivity.getBytesCounter(); - this.messagesizeHistogram = jmsActivity.getMessagesizeHistogram(); - } - - private JMSProducer createJmsProducer() { - JMSProducer jmsProducer = this.jmsContext.createProducer(); - - jmsProducer.setDeliveryMode(this.jmsHeader.getDeliveryMode()); - jmsProducer.setPriority(this.jmsHeader.getMsgPriority()); - jmsProducer.setDeliveryDelay(this.jmsHeader.getMsgDeliveryDelay()); - jmsProducer.setDisableMessageTimestamp(this.jmsHeader.isDisableMsgTimestamp()); - jmsProducer.setDisableMessageID(this.jmsHeader.isDisableMsgId()); - - if (this.asyncJmsOp) { - jmsProducer.setAsync(new CompletionListener() { - @Override - public void onCompletion(Message msg) { - try { - byte[] msgBody = msg.getBody(byte[].class); - if (logger.isTraceEnabled()) { - logger.trace("Async message send success - message body: " + new String(msgBody)); - } - } - catch (JMSException jmsException) { - jmsException.printStackTrace(); - logger.warn("Unexpected error when parsing message body: " + jmsException.getMessage()); - } - } - - @Override - public void onException(Message msg, Exception e) { - try { - byte[] msgBody = msg.getBody(byte[].class); - if (logger.isTraceEnabled()) { - logger.trace("Async message send failure - message body: " + new String(msgBody)); - } - } - catch (JMSException jmsException) { - jmsException.printStackTrace(); - logger.warn("Unexpected error when parsing message body: " + jmsException.getMessage()); - } - } - }); - } - - for (Map.Entry entry : jmsMsgProperties.entrySet()) { - jmsProducer.setProperty(entry.getKey(), entry.getValue()); - } - - return jmsProducer; - } - - @Override - public void run() { - try { - byte[] msgBytes = msgBody.getBytes(StandardCharsets.UTF_8); - int messageSize = msgBytes.length; - jmsProducer.send(jmsDestination, msgBytes); - - messagesizeHistogram.update(messageSize); - bytesCounter.inc(messageSize); - } - catch (Exception ex) { - logger.error("Failed to send JMS message - " + msgBody); - } - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOpMapper.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOpMapper.java deleted file mode 100644 index b52385cdd..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOpMapper.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.ops; - -import io.nosqlbench.driver.jms.JmsActivity; -import io.nosqlbench.driver.jms.util.JmsHeaderLongFunc; - -import javax.jms.Destination; -import java.util.Map; -import java.util.function.LongFunction; - -public abstract class JmsOpMapper implements LongFunction { - protected final JmsActivity jmsActivity; - protected final LongFunction asyncApiFunc; - protected final LongFunction jmsDestinationFunc; - - public JmsOpMapper(JmsActivity jmsActivity, - LongFunction asyncApiFunc, - LongFunction jmsDestinationFunc) - { - this.jmsActivity = jmsActivity; - this.asyncApiFunc = asyncApiFunc; - this.jmsDestinationFunc = jmsDestinationFunc; - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeader.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeader.java deleted file mode 100644 index e7d889d8a..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeader.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.util; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import org.apache.commons.lang.StringUtils; - -import javax.jms.DeliveryMode; - -@Setter -@Getter -@AllArgsConstructor -@ToString -public class JmsHeader { - private int deliveryMode; - private int msgPriority; - private long msgTtl; - private long msgDeliveryDelay; - private boolean disableMsgTimestamp; - private boolean disableMsgId; - - public boolean isValidDeliveryMode() { - return (deliveryMode == DeliveryMode.NON_PERSISTENT) || (deliveryMode == DeliveryMode.PERSISTENT); - } - - public boolean isValidPriority() { - return (msgPriority >= 0) && (msgPriority <= 9); - } - - public boolean isValidTtl() { - return msgTtl >= 0; - } - - public boolean isValidDeliveryDelay() { - return msgTtl >= 0; - } - - public boolean isValidHeader() { - return isValidDeliveryMode() - && isValidPriority() - && isValidTtl() - && isValidDeliveryDelay(); - } - - public String getInvalidJmsHeaderMsgText() { - StringBuilder sb = new StringBuilder(); - - if (!isValidDeliveryMode()) - sb.append("delivery mode - " + deliveryMode + "; "); - if (!isValidPriority()) - sb.append("message priority - " + msgPriority + "; "); - if (!isValidTtl()) - sb.append("message TTL - " + msgTtl + "; "); - if (!isValidDeliveryDelay()) - sb.append("message delivery delay - " + msgDeliveryDelay + "; "); - - String invalidMsgText = sb.toString(); - if (StringUtils.length(invalidMsgText) > 0) - invalidMsgText = StringUtils.substringBeforeLast(invalidMsgText, ";"); - else - invalidMsgText = "none"; - - return "Invalid JMS header values: " + invalidMsgText; - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeaderLongFunc.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeaderLongFunc.java deleted file mode 100644 index 75616091a..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeaderLongFunc.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.util; - -import lombok.*; - -import javax.jms.DeliveryMode; -import javax.jms.Message; -import java.util.function.LongFunction; - -@Setter -@Getter -@NoArgsConstructor -public class JmsHeaderLongFunc implements LongFunction { - private LongFunction deliveryModeFunc; - private LongFunction msgPriorityFunc; - private LongFunction msgTtlFunc; - private LongFunction msgDeliveryDelayFunc; - private LongFunction disableMsgTimestampFunc; - private LongFunction disableMsgIdFunc; - - @Override - public Object apply(long value) { - return new JmsHeader( - (deliveryModeFunc != null) ? deliveryModeFunc.apply(value) : DeliveryMode.PERSISTENT, - (msgPriorityFunc != null) ? msgPriorityFunc.apply(value) : Message.DEFAULT_PRIORITY, - (msgTtlFunc != null) ? msgTtlFunc.apply(value) : Message.DEFAULT_TIME_TO_LIVE, - (msgTtlFunc != null) ? msgTtlFunc.apply(value) : Message.DEFAULT_DELIVERY_DELAY, - (disableMsgTimestampFunc != null) ? disableMsgTimestampFunc.apply(value) : false, - (disableMsgIdFunc != null) ? disableMsgIdFunc.apply(value) : false - ); - } -} diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsUtil.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsUtil.java deleted file mode 100644 index 278cb189b..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsUtil.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.util; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.Arrays; - -public class JmsUtil { - - private final static Logger logger = LogManager.getLogger(JmsUtil.class); - - // Supported JMS provider type - public enum JMS_PROVIDER_TYPES { - PULSAR("pulsar"); - - public final String label; - JMS_PROVIDER_TYPES(String label) { - this.label = label; - } - } - public static boolean isValidJmsProviderType(String type) { - return Arrays.stream(JMS_PROVIDER_TYPES.values()).anyMatch(t -> t.label.equals(type)); - } - - ///// - // NB command line parameters - // - JMS provider type - public final static String JMS_PROVIDER_TYPE_KEY_STR = "provider_type"; - - /// Only applicable when the provider is "Pulsar" - // - Pulsar configuration properties file - public final static String JMS_PULSAR_PROVIDER_CFG_FILE_KEY_STR = "pulsar_cfg_file"; - public final static String JMS_PULSAR_PROVIDER_DFT_CFG_FILE_NAME = "pulsar_config.properties"; - // - Pulsar web url - public final static String JMS_PULSAR_PROVIDER_WEB_URL_KEY_STR = "web_url"; - // - Pulsar service url - public final static String JMS_PULSAR_PROVIDER_SVC_URL_KEY_STR = "service_url"; - - - public final static String ASYNC_API_KEY_STR = "async_api"; - public final static String JMS_DESTINATION_TYPE_KEY_STR = "jms_desitation_type"; - - ///// JMS Producer - // Supported JMS provider type - public enum JMS_MSG_HEADER_KEYS { - DELIVERY_MODE("jms_producer_header_msg_delivery_mode"), - PRIORITY("jms_producer_header_msg_priority"), - TTL("jms_producer_header_msg_ttl"), - DELIVERY_DELAY("jms_producer_header_msg_delivery_delay"), - DISABLE_TIMESTAMP("jms_producer_header_disable_msg_timestamp"), - DISABLE_ID("jms_producer_header_disable_msg_id"); - - public final String label; - JMS_MSG_HEADER_KEYS(String label) { - this.label = label; - } - } - public static boolean isValidJmsHeaderKey(String type) { - return Arrays.stream(JMS_MSG_HEADER_KEYS.values()).anyMatch(t -> t.label.equals(type)); - } - public final static String JMS_PRODUCER_MSG_PROPERTY_KEY_STR = "jms_producer_msg_properties"; - public final static String JMS_PRODUCER_MSG_BODY_KEY_STR = "msg_body"; - - ///// JMS Consumer - public final static String JMS_CONSUMER_DURABLE_KEY_STR = "jms_consumer_msg_durable"; - public final static String JMS_CONSUMER_SHARED_KEY_STR = "jms_consumer_msg_shared"; - public final static String JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR = "jms_consumer_subscription"; - public final static String JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR = "jms_consumer_msg_read_selector"; - public final static String JMS_CONSUMER_MSG_NOLOCAL_KEY_STR = "jms_consumer_msg_nolocal"; - public final static String JMS_CONSUMER_READ_TIMEOUT_KEY_STR = "jms_consumer_msg_read_timeout"; - - - // Only applicable to Pulsar JMS provider - public final static String PULSAR_JMS_TOPIC_URI_KEY_STR = "pulsar_topic_uri"; - - // Supported message operation types - public enum OP_TYPES { - MSG_SEND("msg_send"), - MSG_READ("msg_read"); - - public final String label; - OP_TYPES(String label) { - this.label = label; - } - } - public static boolean isValidClientType(String type) { - return Arrays.stream(OP_TYPES.values()).anyMatch(t -> t.label.equals(type)); - } - - // JMS Destination Types - public enum JMS_DESTINATION_TYPES { - QUEUE("queue"), - TOPIC("topic"); - - public final String label; - JMS_DESTINATION_TYPES(String label) { - this.label = label; - } - } - public static boolean isValidJmsDestinationType(String type) { - return Arrays.stream(JMS_DESTINATION_TYPES.values()).anyMatch(t -> t.label.equals(type)); - } -} - diff --git a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/PulsarConfig.java b/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/PulsarConfig.java deleted file mode 100644 index 5a015cddf..000000000 --- a/driver-jms/src/main/java/io/nosqlbench/driver/jms/util/PulsarConfig.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.jms.util; - -import org.apache.commons.configuration2.Configuration; -import org.apache.commons.configuration2.FileBasedConfiguration; -import org.apache.commons.configuration2.PropertiesConfiguration; -import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder; -import org.apache.commons.configuration2.builder.fluent.Parameters; -import org.apache.commons.configuration2.ex.ConfigurationException; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -public class PulsarConfig { - private final static Logger logger = LogManager.getLogger(PulsarConfig.class); - - public static final String SCHEMA_CONF_PREFIX = "schema"; - public static final String CLIENT_CONF_PREFIX = "client"; - public static final String PRODUCER_CONF_PREFIX = "producer"; - public static final String CONSUMER_CONF_PREFIX = "consumer"; - - private final Map schemaConfMap = new HashMap<>(); - private final Map clientConfMap = new HashMap<>(); - private final Map producerConfMap = new HashMap<>(); - private final Map consumerConfMap = new HashMap<>(); - - public PulsarConfig(String fileName) { - File file = new File(fileName); - - try { - String canonicalFilePath = file.getCanonicalPath(); - - Parameters params = new Parameters(); - - FileBasedConfigurationBuilder builder = - new FileBasedConfigurationBuilder(PropertiesConfiguration.class) - .configure(params.properties() - .setFileName(fileName)); - - Configuration config = builder.getConfiguration(); - - // Get schema specific configuration settings - for (Iterator it = config.getKeys(SCHEMA_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - schemaConfMap.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get client connection specific configuration settings - for (Iterator it = config.getKeys(CLIENT_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - clientConfMap.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get producer specific configuration settings - for (Iterator it = config.getKeys(PRODUCER_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - producerConfMap.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get consumer specific configuration settings - for (Iterator it = config.getKeys(CONSUMER_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - consumerConfMap.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - } catch (IOException ioe) { - logger.error("Can't read the specified config properties file: " + fileName); - ioe.printStackTrace(); - } catch (ConfigurationException cex) { - logger.error("Error loading configuration items from the specified config properties file: " + fileName + ":" + cex.getMessage()); - cex.printStackTrace(); - } - } - - public Map getSchemaConfMap() { - return this.schemaConfMap; - } - public Map getClientConfMap() { - return this.clientConfMap; - } - public Map getProducerConfMap() { - return this.producerConfMap; - } - public Map getConsumerConfMap() { - return this.consumerConfMap; - } -} diff --git a/driver-jms/src/main/resources/jms.md b/driver-jms/src/main/resources/jms.md deleted file mode 100644 index 07dd0c5c7..000000000 --- a/driver-jms/src/main/resources/jms.md +++ /dev/null @@ -1 +0,0 @@ -# Overview diff --git a/driver-jms/src/main/resources/pulsar_config.properties b/driver-jms/src/main/resources/pulsar_config.properties deleted file mode 100644 index f711535ac..000000000 --- a/driver-jms/src/main/resources/pulsar_config.properties +++ /dev/null @@ -1,33 +0,0 @@ -### Schema related configurations - schema.xxx -# valid types: -# - primitive type (https://pulsar.apache.org/docs/en/schema-understand/#primitive-type) -# - keyvalue (https://pulsar.apache.org/docs/en/schema-understand/#keyvalue) -# - strut (complex type) (https://pulsar.apache.org/docs/en/schema-understand/#struct) -# avro, json, protobuf -# -# NOTE: for JMS client, Pulsar "schema" is NOT supported yet -schema.type= -schema.definition= - - -### Pulsar client related configurations - client.xxx -# http://pulsar.apache.org/docs/en/client-libraries-java/#client -client.connectionTimeoutMs=5000 -#client.authPluginClassName=org.apache.pulsar.client.impl.auth.AuthenticationToken -#client.authParams= -#client.tlsAllowInsecureConnection=true -client.numIoThreads=10 -client.numListenerThreads=10 - - -### Producer related configurations (global) - producer.xxx -# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer -producer.sendTimeoutMs= -producer.blockIfQueueFull=true -producer.maxPendingMessages=10000 -producer.batchingMaxMessages=10000 - - -### Consumer related configurations (global) - consumer.xxx -# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer -consumer.receiverQueueSize=2000 diff --git a/driver-jms/src/main/resources/pulsar_jms.yaml b/driver-jms/src/main/resources/pulsar_jms.yaml deleted file mode 100644 index 755f05ed6..000000000 --- a/driver-jms/src/main/resources/pulsar_jms.yaml +++ /dev/null @@ -1,89 +0,0 @@ -bindings: - payload: NumberNameToString() #AlphaNumericString(20) - tenant: Mod(10000); Div(10L); ToString(); Prefix("tnt") - namespace: Mod(10); Div(5L); ToString(); Prefix("ns") - core_topic_name: Mod(5); ToString(); Prefix("t") - -# document level parameters that apply to all Pulsar client types: -params: - ### static only - async_api: "true" - - ### Static only - # Valid values: queue (point-to-point) or topic (pub-sub) - jms_desitation_type: "topic" - - ### Static Only - # NOTE: ONLY relevant when the JMS provider is Pulsar - #pulsar_topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}" - #pulsar_topic_uri: "persistent://public/default/pt100" - #pulsar_topic_uri: "persistent://public/default/t0" - pulsar_topic_uri: "persistent://public/default/pt100_10" - #pulsar_topic_uri: "persistent://public/default/pt200_10" - #pulsar_topic_uri: "persistent://public/default/pt300_10" - #pulsar_topic_uri: "persistent://public/default/pt400_10" - -blocks: - - name: "producer-block" - tags: - phase: "jms_producer" - statements: - - name: "s1" - optype: "msg_send" - - ### JMS PRODUCER message header - ### https://docs.oracle.com/javaee/7/api/constant-values.html#javax.jms.DeliveryMode.NON_PERSISTENT - # - static or dynamic - # - Producer only - # Valid values: non-persistent(1), or persistent(2) - default - jms_producer_header_msg_delivery_mode: "2" - # Valid values: 0~9 (4 as default) - jms_producer_header_msg_priority: "4" - # Valid values: non-negative long; default 0 (never expires) - jms_producer_header_msg_ttl: "0" - # Valid values: non-negative long; default 0 (no delay) - jms_producer_header_msg_delivery_delay: "0" - # Valid values: true/false; default false (message timestamp is enabled) - jms_producer_header_disable_msg_timestamp: "false" - # Valid values: true/false; default false (message ID is enabled) - jms_producer_header_disable_msg_id: "false" - - ### JMS PRODUCER message properties - # - static only - # - Producer only - # - In format: "key1=value1;key2=value2;..." - jms_producer_msg_properties: "key1=value1;key2=value2" - - ### JMS PRODUCER message body - msg_body: "{payload}" - - - name: "consumer-block" - tags: - phase: "jms_consumer" - statements: - - name: "s1" - optype: "msg_read" - - ### JMS CONSUMER durable and shared - jms_consumer_msg_durable: "true" - jms_consumer_msg_shared: "true" - - ### JMS CONSUMER subscription name - # - only relevant for durable consumer - jms_consumer_subscription: "mysub" - - ### JMS CONSUMER subscription name - # - only relevant for unshared consumer - jms_consumer_nolocal: "false" - - ### JMS CONSUMER message read timeout - # - unit: milliseconds - # - 0 means call blocks indefinitely - # - FIXME: 0 supposes to wait indefinitly; but - # it actually behaves like no wait at all - jms_consumer_msg_read_timeout: "10000" - - ### JMS CONSUMER message selector - # - empty string means no message selector - # - https://docs.oracle.com/cd/E19798-01/821-1841/bncer/index.html - jms_consumer_msg_read_selector: "" diff --git a/nb/pom.xml b/nb/pom.xml index 3cbffe028..43e93653e 100644 --- a/nb/pom.xml +++ b/nb/pom.xml @@ -67,6 +67,12 @@ 4.17.22-SNAPSHOT + + io.nosqlbench + adapter-pulsar + 4.17.22-SNAPSHOT + + diff --git a/nb5/pom.xml b/nb5/pom.xml index f2fda67a0..f801d40cd 100644 --- a/nb5/pom.xml +++ b/nb5/pom.xml @@ -88,6 +88,12 @@ 4.17.31-SNAPSHOT + + io.nosqlbench + adapter-pulsar + 4.17.31-SNAPSHOT + +