IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

- Delete unsed code "driver-jms"

Browse Source 
- NB5 Pulsar driver Admin functionality in place - create/delete tenant, namespace, and topic
This commit is contained in:
yabinmeng
2022-11-11 18:36:06 -06:00
parent dae02bb60b
commit c309d200ac
56 changed files with 2611 additions and 1781 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.run/NBCLI web foreground dryrun.run.xml
View File

@@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="NBCLI web foreground dryrun" type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="io.nosqlbench.engine.cli.NBCLI" />
<module name="nb" />
<option name="PROGRAM_PARAMETERS" value="run type=webdriver yaml=local/webtest cycles=1000 cyclerate=100 threads=1 dryrun=true -vv" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="io.nosqlbench.engine.cli.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>

70
driver-jms/pom.xml → adapter-pulsar/pom.xml
View File

@@ -17,61 +17,49 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>adapter-pulsar</artifactId>
<packaging>jar</packaging>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.22-SNAPSHOT</version>
<version>4.17.31-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
<artifactId>driver-jms</artifactId>
<packaging>jar</packaging>
<name>${project.artifactId}</name>
<description>
A JMS driver for nosqlbench. This provides the ability to inject synthetic data
into a pulsar system via JMS 2.0 compatibile APIs.
NOTE: this is JMS compatible driver from DataStax that allows using a Pulsar cluster
as the potential JMS Destination
A Pulsar driver for nosqlbench. This provides the ability to inject synthetic data
into a pulsar system.
</description>
<!-- <repositories>-->
<!-- &lt;!&ndash; Tempoarily needed for Pulsar JMS Java library &ndash;&gt;-->
<!-- <repository>-->
<!-- <id>datastax-releases-local</id>-->
<!-- <name>DataStax Local Releases</name>-->
<!-- <url>https://repo.sjc.dsinternal.org/artifactory/datastax-snapshots-local/</url>-->
<!-- <releases>-->
<!-- <enabled>false</enabled>-->
<!-- </releases>-->
<!-- <snapshots>-->
<!-- <enabled>true</enabled>-->
<!-- </snapshots>-->
<!-- </repository>-->
<!-- </repositories>-->
<properties>
<pulsar.version>2.10.1</pulsar.version>
</properties>
<dependencies>
<!-- core dependencies -->
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.22-SNAPSHOT</version>
<version>4.17.31-SNAPSHOT</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>4.17.31-SNAPSHOT</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.projectlombok/lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.24</version>
<scope>provided</scope>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client</artifactId>
<version>${pulsar.version}</version>
</dependency>
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client-admin</artifactId>
<version>${pulsar.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils -->
@@ -88,13 +76,19 @@
<version>2.8.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.datastax.oss/pulsar-jms -->
<!-- https://mvnrepository.com/artifact/org.apache.avro/avro -->
<dependency>
<groupId>com.datastax.oss</groupId>
<artifactId>pulsar-jms</artifactId>
<version>2.4.11</version>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.11.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
</dependencies>
</project>

54
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar;
import io.nosqlbench.adapter.pulsar.ops.PulsarOp;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.nb.annotations.Maturity;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.api.config.standard.NBConfigModel;
import io.nosqlbench.api.config.standard.NBConfiguration;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.function.Function;
@Service(value = DriverAdapter.class, selector = "pulsar-nb5", maturity = Maturity.Experimental)
public class PulsarDriverAdapter extends BaseDriverAdapter<PulsarOp, PulsarSpace> {
private final static Logger logger = LogManager.getLogger(PulsarDriverAdapter.class);
@Override
public OpMapper<PulsarOp> getOpMapper() {
DriverSpaceCache<? extends PulsarSpace> spaceCache = getSpaceCache();
NBConfiguration adapterConfig = getConfiguration();
return new PulsarOpMapper(this, adapterConfig, spaceCache);
}
@Override
public Function<String, ? extends PulsarSpace> getSpaceInitializer(NBConfiguration cfg) {
return (s) -> new PulsarSpace(s, cfg);
}
@Override
public NBConfigModel getConfigModel() {
return super.getConfigModel().add(PulsarSpace.getConfigModel());
}
}

86
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java Normal file
View File

@@ -0,0 +1,86 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar;
import io.nosqlbench.adapter.pulsar.dispensers.*;
import io.nosqlbench.adapter.pulsar.ops.PulsarOp;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.engine.api.activityimpl.OpDispenser;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.engine.api.templating.ParsedOp;
import io.nosqlbench.engine.api.templating.TypeAndTarget;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.Schema;
public class PulsarOpMapper implements OpMapper<PulsarOp> {
private final static Logger logger = LogManager.getLogger(PulsarOpMapper.class);
private final NBConfiguration cfg;
private final DriverSpaceCache<? extends PulsarSpace> cache;
private final DriverAdapter adapter;
public PulsarOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache<? extends PulsarSpace> cache) {
this.cfg = cfg;
this.cache = cache;
this.adapter = adapter;
}
@Override
public OpDispenser<? extends PulsarOp> apply(ParsedOp op) {
String space = op.getStaticConfigOr("space", "default");
PulsarClient pulsarClient = cache.get(space).getPulsarClient();
PulsarAdmin pulsarAdmin = cache.get(space).getPulsarAdmin();
Schema<?> pulsarSchema = cache.get(space).getPulsarSchema();
/*
* If the user provides a body element, then they want to provide the JSON or
* a data structure that can be converted into JSON, bypassing any further
* specialized type-checking or op-type specific features
*/
if (op.isDefined("body")) {
throw new RuntimeException("This mode is reserved for later. Do not use the 'body' op field.");
}
else {
TypeAndTarget<PulsarOpType, String> opType = op.getTypeAndTarget(PulsarOpType.class, String.class);
return switch (opType.enumId) {
case AdminTenant ->
new AdminTenantOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin);
case AdminNamespace ->
new AdminNamespaceOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin);
case AdminTopic ->
new AdminTopicOpDispenser(adapter, op, opType.targetFunction, pulsarAdmin);
case MessageProduce ->
new MessageProducerOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema);
case MessageConsume ->
new MessageConsumerOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema);
case MessageRead ->
new MessageReaderOpDispenser(adapter, op, opType.targetFunction, pulsarClient, pulsarSchema);
};
}
}
}

23
driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsTimeTrackOp.java → adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpType.java
View File

@@ -14,20 +14,13 @@
* limitations under the License.
*/
package io.nosqlbench.driver.jms.ops;
package io.nosqlbench.adapter.pulsar;
/**
* Base type of all Sync Pulsar Operations including Producers and Consumers.
*/
public abstract class JmsTimeTrackOp implements JmsOp {
public void run(Runnable timeTracker) {
try {
this.run();
} finally {
timeTracker.run();
}
}
public abstract void run();
public enum PulsarOpType {
AdminTenant,
AdminNamespace,
AdminTopic,
MessageProduce,
MessageConsume,
MessageRead
}

191
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarSpace.java Normal file
View File

@@ -0,0 +1,191 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar;
import io.nosqlbench.adapter.pulsar.util.PulsarAdapterUtil;
import io.nosqlbench.adapter.pulsar.util.PulsarNBClientConf;
import io.nosqlbench.api.config.standard.ConfigModel;
import io.nosqlbench.api.config.standard.NBConfigModel;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.api.config.standard.Param;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminBuilder;
import org.apache.pulsar.client.api.ClientBuilder;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.common.schema.KeyValueEncodingType;
import java.util.Map;
public class PulsarSpace {
private final static Logger logger = LogManager.getLogger(PulsarSpace.class);
private final String name;
private final NBConfiguration cfg;
private final String pulsarSvcUrl;
private final String webSvcUrl;
private PulsarNBClientConf pulsarNBClientConf;
private PulsarClient pulsarClient;
private PulsarAdmin pulsarAdmin;
private Schema<?> pulsarSchema;
public PulsarSpace(String name, NBConfiguration cfg) {
this.name = name;
this.cfg = cfg;
this.pulsarSvcUrl = cfg.get("service_url");
this.webSvcUrl = cfg.get("web_url");
this.pulsarNBClientConf = new PulsarNBClientConf(cfg.get("config"));
initPulsarAdminAndClientObj();
createPulsarSchemaFromConf();
}
public static NBConfigModel getConfigModel() {
return ConfigModel.of(PulsarSpace.class)
.add(Param.defaultTo("service_url", "pulsar://localhost:6650")
.setDescription("Pulsar broker service URL."))
.add(Param.defaultTo("web_url", "http://localhost:8080")
.setDescription("Pulsar web service URL."))
.add(Param.defaultTo("config", "config.properties")
.setDescription("Pulsar client connection configuration property file."))
.add(Param.defaultTo("cyclerate_per_thread", false)
.setDescription("Apply cycle rate per NB thread"))
.asReadOnly();
}
public String getPulsarSvcUrl() { return pulsarSvcUrl; }
public String getWebSvcUrl() { return webSvcUrl; }
public PulsarNBClientConf getPulsarNBClientConf() { return pulsarNBClientConf; }
public PulsarClient getPulsarClient() { return pulsarClient; }
public PulsarAdmin getPulsarAdmin() { return pulsarAdmin; }
public Schema<?> getPulsarSchema() { return pulsarSchema; }
/**
* Initialize
* - PulsarAdmin object for adding/deleting tenant, namespace, and topic
* - PulsarClient object for message publishing and consuming
*/
private void initPulsarAdminAndClientObj() {
PulsarAdminBuilder adminBuilder =
PulsarAdmin.builder()
.serviceHttpUrl(webSvcUrl);
ClientBuilder clientBuilder = PulsarClient.builder();
try {
Map<String, Object> clientConfMap = pulsarNBClientConf.getClientConfMap();
// Override "client.serviceUrl" setting in config.properties
clientConfMap.remove("serviceUrl");
clientBuilder.loadConf(clientConfMap).serviceUrl(pulsarSvcUrl);
// Pulsar Authentication
String authPluginClassName =
(String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authPulginClassName.label);
String authParams =
(String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.authParams.label);
if ( !StringUtils.isAnyBlank(authPluginClassName, authParams) ) {
adminBuilder.authentication(authPluginClassName, authParams);
clientBuilder.authentication(authPluginClassName, authParams);
}
boolean useTls = StringUtils.contains(pulsarSvcUrl, "pulsar+ssl");
if ( useTls ) {
String tlsHostnameVerificationEnableStr =
(String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsHostnameVerificationEnable.label);
boolean tlsHostnameVerificationEnable = BooleanUtils.toBoolean(tlsHostnameVerificationEnableStr);
adminBuilder
.enableTlsHostnameVerification(tlsHostnameVerificationEnable);
clientBuilder
.enableTlsHostnameVerification(tlsHostnameVerificationEnable);
String tlsTrustCertsFilePath =
(String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsTrustCertsFilePath.label);
if (!StringUtils.isBlank(tlsTrustCertsFilePath)) {
adminBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath);
clientBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath);
}
String tlsAllowInsecureConnectionStr =
(String) pulsarNBClientConf.getClientConfValue(PulsarAdapterUtil.CLNT_CONF_KEY.tlsAllowInsecureConnection.label);
boolean tlsAllowInsecureConnection = BooleanUtils.toBoolean(tlsAllowInsecureConnectionStr);
adminBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection);
clientBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection);
}
pulsarAdmin = adminBuilder.build();
pulsarClient = clientBuilder.build();
} catch (PulsarClientException e) {
logger.error("Fail to create PulsarAdmin and/or PulsarClient object from the global configuration!");
throw new RuntimeException("Fail to create PulsarAdmin and/or PulsarClient object from global configuration!");
}
}
/**
* Get Pulsar schema from the definition string
*/
private Schema<?> buildSchemaFromDefinition(String schemaTypeConfEntry,
String schemaDefinitionConfEntry) {
Object value = pulsarNBClientConf.getSchemaConfValue(schemaTypeConfEntry);
Object schemaDefinition = pulsarNBClientConf.getSchemaConfValue(schemaDefinitionConfEntry);
String schemaType = (value != null) ? value.toString() : "";
Schema<?> result;
if (PulsarAdapterUtil.isAvroSchemaTypeStr(schemaType)) {
String schemaDefStr = (schemaDefinition != null) ? schemaDefinition.toString() : "";
result = PulsarAdapterUtil.getAvroSchema(schemaType, schemaDefStr);
} else if (PulsarAdapterUtil.isPrimitiveSchemaTypeStr(schemaType)) {
result = PulsarAdapterUtil.getPrimitiveTypeSchema(schemaType);
} else if (PulsarAdapterUtil.isAutoConsumeSchemaTypeStr(schemaType)) {
result = Schema.AUTO_CONSUME();
} else {
throw new RuntimeException("Unsupported schema type string: " + schemaType + "; " +
"Only primitive type, Avro type and AUTO_CONSUME are supported at the moment!");
}
return result;
}
private void createPulsarSchemaFromConf() {
pulsarSchema = buildSchemaFromDefinition("schema.type", "schema.definition");
// this is to allow KEY_VALUE schema
if (pulsarNBClientConf.hasSchemaConfKey("schema.key.type")) {
Schema<?> pulsarKeySchema = buildSchemaFromDefinition("schema.key.type", "schema.key.definition");
Object encodingType = pulsarNBClientConf.getSchemaConfValue("schema.keyvalue.encodingtype");
KeyValueEncodingType keyValueEncodingType = KeyValueEncodingType.SEPARATED;
if (encodingType != null) {
keyValueEncodingType = KeyValueEncodingType.valueOf(encodingType.toString());
}
pulsarSchema = Schema.KeyValue(pulsarKeySchema, pulsarSchema, keyValueEncodingType);
}
}
}

43
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminNamespaceOpDispenser.java Normal file
View File

@@ -0,0 +1,43 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.dispensers;
import io.nosqlbench.adapter.pulsar.ops.AdminNamespaceOp;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.templating.ParsedOp;
import org.apache.pulsar.client.admin.PulsarAdmin;
import java.util.function.LongFunction;
public class AdminNamespaceOpDispenser extends PulsarAdminOpDispenser {
public AdminNamespaceOpDispenser(DriverAdapter adapter,
ParsedOp op,
LongFunction<String> tgtNameFunc,
PulsarAdmin pulsarAdmin) {
super(adapter, op, tgtNameFunc, pulsarAdmin);
}
@Override
public AdminNamespaceOp apply(long cycle) {
return new AdminNamespaceOp(
pulsarAdmin,
asyncApiFunc.apply(cycle),
adminDelOpFunc.apply(cycle),
tgtNameFunc.apply(cycle));
}
}

51
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTenantOpDispenser.java Normal file
View File

@@ -0,0 +1,51 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.dispensers;
import io.nosqlbench.adapter.pulsar.ops.AdminTenantOp;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.templating.ParsedOp;
import org.apache.pulsar.client.admin.PulsarAdmin;
import java.util.*;
import java.util.function.LongFunction;
public class AdminTenantOpDispenser extends PulsarAdminOpDispenser {
private final LongFunction<Set<String>> adminRolesFunc;
private final LongFunction<Set<String>> allowedClustersFunc;
public AdminTenantOpDispenser(DriverAdapter adapter,
ParsedOp op,
LongFunction<String> tgtNameFunc,
PulsarAdmin pulsarAdmin) {
super(adapter, op, tgtNameFunc, pulsarAdmin);
adminRolesFunc = lookupStaticStrSetOpValueFunc("admin_roles");
allowedClustersFunc = lookupStaticStrSetOpValueFunc("allowed_clusters");
}
@Override
public AdminTenantOp apply(long cycle) {
return new AdminTenantOp(
pulsarAdmin,
asyncApiFunc.apply(cycle),
adminDelOpFunc.apply(cycle),
tgtNameFunc.apply(cycle),
adminRolesFunc.apply(cycle),
allowedClustersFunc.apply(cycle));
}
}

54
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/AdminTopicOpDispenser.java Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.dispensers;
import io.nosqlbench.adapter.pulsar.ops.AdminTopicOp;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.templating.ParsedOp;
import org.apache.pulsar.client.admin.PulsarAdmin;
import java.util.function.LongFunction;
public class AdminTopicOpDispenser extends PulsarAdminOpDispenser {
private final LongFunction<Boolean> enablePartFunc;
private final LongFunction<Integer> partNumFunc;
public AdminTopicOpDispenser(DriverAdapter adapter,
ParsedOp op,
LongFunction<String> tgtNameFunc,
PulsarAdmin pulsarAdmin) {
super(adapter, op, tgtNameFunc, pulsarAdmin);
// Non-partitioned topic is default
enablePartFunc = lookupStaticBoolConfigValueFunc("enable_partition", false);
partNumFunc = lookupStaticIntOpValueFunc("partition_num", 1);
}
@Override
public AdminTopicOp apply(long cycle) {
return new AdminTopicOp(
pulsarAdmin,
asyncApiFunc.apply(cycle),
adminDelOpFunc.apply(cycle),
tgtNameFunc.apply(cycle),
enablePartFunc.apply(cycle),
partNumFunc.apply(cycle)
);
}
}

41
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageConsumerOpDispenser.java Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.dispensers;
import io.nosqlbench.adapter.pulsar.ops.MessageConsumerOp;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.templating.ParsedOp;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.Schema;
import java.util.function.LongFunction;
public class MessageConsumerOpDispenser extends PulsarClientOpDispenser {
public MessageConsumerOpDispenser(DriverAdapter adapter,
ParsedOp op,
LongFunction<String> tgtNameFunc,
PulsarClient pulsarClient,
Schema<?> pulsarSchema) {
super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema);
}
@Override
public MessageConsumerOp apply(long cycle) {
return new MessageConsumerOp(pulsarClient, pulsarSchema);
}
}

41
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageProducerOpDispenser.java Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.dispensers;
import io.nosqlbench.adapter.pulsar.ops.MessageProducerOp;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.templating.ParsedOp;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.Schema;
import java.util.function.LongFunction;
public class MessageProducerOpDispenser extends PulsarClientOpDispenser {
public MessageProducerOpDispenser(DriverAdapter adapter,
ParsedOp op,
LongFunction<String> tgtNameFunc,
PulsarClient pulsarClient,
Schema<?> pulsarSchema) {
super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema);
}
@Override
public MessageProducerOp apply(long cycle) {
return new MessageProducerOp(pulsarClient, pulsarSchema);
}
}

41
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/MessageReaderOpDispenser.java Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.dispensers;
import io.nosqlbench.adapter.pulsar.ops.MessageReaderOp;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.templating.ParsedOp;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.Schema;
import java.util.function.LongFunction;
public class MessageReaderOpDispenser extends PulsarClientOpDispenser {
public MessageReaderOpDispenser(DriverAdapter adapter,
ParsedOp op,
LongFunction<String> tgtNameFunc,
PulsarClient pulsarClient,
Schema<?> pulsarSchema) {
super(adapter, op, tgtNameFunc, pulsarClient, pulsarSchema);
}
@Override
public MessageReaderOp apply(long cycle) {
return new MessageReaderOp(pulsarClient, pulsarSchema);
}
}

40
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarAdminOpDispenser.java Normal file
View File

@@ -0,0 +1,40 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.dispensers;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.templating.ParsedOp;
import org.apache.pulsar.client.admin.PulsarAdmin;
import java.util.function.LongFunction;
public abstract class PulsarAdminOpDispenser extends PulsarBaseOpDispenser {
protected final PulsarAdmin pulsarAdmin;
protected final LongFunction<Boolean> adminDelOpFunc;
public PulsarAdminOpDispenser(DriverAdapter adapter,
ParsedOp op,
LongFunction<String> tgtNameFunc,
PulsarAdmin pulsarAdmin) {
super(adapter, op, tgtNameFunc);
this.pulsarAdmin = pulsarAdmin;
// Creating admin objects (tenant, namespace, topic) is the default
this.adminDelOpFunc = lookupStaticBoolConfigValueFunc("admin_delop", false);
}
}

85
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarBaseOpDispenser.java Normal file
View File

@@ -0,0 +1,85 @@
package io.nosqlbench.adapter.pulsar.dispensers;
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import io.nosqlbench.adapter.pulsar.PulsarSpace;
import io.nosqlbench.adapter.pulsar.ops.PulsarOp;
import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.templating.ParsedOp;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.*;
import java.util.function.LongFunction;
import java.util.function.Predicate;
import java.util.stream.Collectors;
public abstract class PulsarBaseOpDispenser extends BaseOpDispenser<PulsarOp, PulsarSpace> {
private final static Logger logger = LogManager.getLogger("PulsarBaseOpDispenser");
protected final ParsedOp parsedOp;
protected final LongFunction<Boolean> asyncApiFunc;
protected final LongFunction<String> tgtNameFunc;
public PulsarBaseOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction<String> tgtNameFunc) {
super(adapter, op);
this.parsedOp = op;
this.tgtNameFunc = tgtNameFunc;
// Async API is the default
this.asyncApiFunc = lookupStaticBoolConfigValueFunc("async_api", true);
}
protected LongFunction<Boolean> lookupStaticBoolConfigValueFunc(String paramName, boolean defaultValue) {
return (l) -> parsedOp.getOptionalStaticConfig(paramName, String.class)
.filter(Predicate.not(String::isEmpty))
.map(value -> BooleanUtils.toBoolean(value))
.orElse(defaultValue);
}
protected LongFunction<Integer> lookupStaticIntOpValueFunc(String paramName, int defaultValue) {
return (l) -> parsedOp.getOptionalStaticValue(paramName, String.class)
.filter(Predicate.not(String::isEmpty))
.map(value -> NumberUtils.toInt(value))
.map(value -> {
if (value < 0) return 0;
else return value;
}).orElse(defaultValue);
}
protected LongFunction<Set<String>> lookupStaticStrSetOpValueFunc(String paramName) {
return (l) -> parsedOp.getOptionalStaticValue(paramName, String.class)
.filter(Predicate.not(String::isEmpty))
.map(value -> {
Set<String > set = new HashSet<>();
if (StringUtils.contains(value,',')) {
set = Arrays.stream(value.split(","))
.map(String::trim)
.filter(Predicate.not(String::isEmpty))
.collect(Collectors.toCollection(LinkedHashSet::new));
}
return set;
}).orElse(Collections.emptySet());
}
}

40
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/dispensers/PulsarClientOpDispenser.java Normal file
View File

@@ -0,0 +1,40 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.dispensers;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.templating.ParsedOp;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.Schema;
import java.util.function.LongFunction;
public abstract class PulsarClientOpDispenser extends PulsarBaseOpDispenser {
protected final PulsarClient pulsarClient;
protected final Schema<?> pulsarSchema;
public PulsarClientOpDispenser(DriverAdapter adapter,
ParsedOp op,
LongFunction<String> tgtNameFunc,
PulsarClient pulsarClient,
Schema<?> pulsarSchema) {
super(adapter, op, tgtNameFunc);
this.pulsarClient = pulsarClient;
this.pulsarSchema = pulsarSchema;
}
}

25
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterInvalidParamException.java Normal file
View File

@@ -0,0 +1,25 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.nosqlbench.adapter.pulsar.exception;
public class PulsarAdapterInvalidParamException extends RuntimeException {
public PulsarAdapterInvalidParamException(String paramName, String errDesc) {
super("Invalid setting for parameter (" + paramName + "): " + errDesc);
}
}

30
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnexpectedException.java Normal file
View File

@@ -0,0 +1,30 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.nosqlbench.adapter.pulsar.exception;
public class PulsarAdapterUnexpectedException extends RuntimeException {
public PulsarAdapterUnexpectedException(String message) {
super(message);
printStackTrace();
}
public PulsarAdapterUnexpectedException(Exception e) {
super(e);
printStackTrace();
}
}

25
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/exception/PulsarAdapterUnsupportedOpException.java Normal file
View File

@@ -0,0 +1,25 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.nosqlbench.adapter.pulsar.exception;
public class PulsarAdapterUnsupportedOpException extends RuntimeException {
public PulsarAdapterUnsupportedOpException(String pulsarOpType) {
super("Unsupported Pulsar adapter operation type: \"" + pulsarOpType + "\"");
}
}

120
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminNamespaceOp.java Normal file
View File

@@ -0,0 +1,120 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.ops;
import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.admin.Namespaces;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import java.util.concurrent.CompletableFuture;
public class AdminNamespaceOp extends PulsarAdminOp {
private final static Logger logger = LogManager.getLogger(AdminNamespaceOp.class);
// in format: <tenant>/<namespace>
private final String nsName;
public AdminNamespaceOp(PulsarAdmin pulsarAdmin,
boolean asyncApi,
boolean adminDelOp,
String nsName) {
super(pulsarAdmin, asyncApi, adminDelOp);
this.nsName = nsName;
}
@Override
public Void apply(long value) {
// Do nothing if the namespace name is empty
if ( !StringUtils.isBlank(nsName) ) {
Namespaces namespaces = pulsarAdmin.namespaces();
// Admin API - create tenants and namespaces
if (!adminDelOp) {
try {
if (!asyncApi) {
namespaces.createNamespace(nsName);
if (logger.isDebugEnabled()) {
logger.debug("Successful sync creation of namespace \"{}\"", nsName);
}
} else {
CompletableFuture<Void> future = namespaces.createNamespaceAsync(nsName);
future.whenComplete((unused, throwable) -> {
if (logger.isDebugEnabled()) {
logger.trace("Successful async creation of namespace \"{}\"", nsName);
}
}).exceptionally(ex -> {
if (logger.isDebugEnabled()) {
logger.error("Failed async creation of namespace \"{}\"", nsName);
}
return null;
});
}
}
catch (PulsarAdminException.ConflictException ce) {
if (logger.isDebugEnabled()) {
logger.error("Namespace \"{}\" already exists - skip creation!", nsName);
}
}
catch (PulsarAdminException e) {
throw new PulsarAdapterUnexpectedException(
"Unexpected error when creating pulsar namespace \"" + nsName + "\"");
}
}
// Admin API - delete tenants and namespaces
else {
try {
if (!asyncApi) {
namespaces.deleteNamespace(nsName);
if (logger.isDebugEnabled()) {
logger.debug("Successful sync deletion of namespace \"{}\"", nsName);
}
} else {
CompletableFuture<Void> future = namespaces.deleteNamespaceAsync(nsName, true);
future.whenComplete((unused, throwable) -> {
if (logger.isDebugEnabled()) {
logger.debug("Successful sync deletion of namespace \"{}\"", nsName);
}
}).exceptionally(ex -> {
if (logger.isDebugEnabled()) {
logger.error("Failed async deletion of namespace \"{}\"", nsName);
}
return null;
});
}
}
catch (PulsarAdminException.NotFoundException nfe) {
if (logger.isDebugEnabled()) {
logger.error("Namespace \"{}\" doesn't exists - skip deletion!", nsName);
}
}
catch (PulsarAdminException e) {
throw new PulsarAdapterUnexpectedException(
"Unexpected error when deleting pulsar namespace \"" + nsName + "\"");
}
}
}
return null;
}
}

144
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTenantOp.java Normal file
View File

@@ -0,0 +1,144 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.ops;
import io.nosqlbench.adapter.pulsar.PulsarDriverAdapter;
import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.admin.*;
import org.apache.pulsar.common.policies.data.TenantInfo;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
public class AdminTenantOp extends PulsarAdminOp {
private final static Logger logger = LogManager.getLogger(AdminTenantOp.class);
private final Set<String> adminRoles;
private final Set<String> allowedClusters;
private final String tntName;
public AdminTenantOp(PulsarAdmin pulsarAdmin,
boolean asyncApi,
boolean adminDelOp,
String tntName,
Set<String> adminRoles,
Set<String> allowedClusters) {
super(pulsarAdmin, asyncApi, adminDelOp);
this.tntName = tntName;
this.adminRoles = adminRoles;
this.allowedClusters = allowedClusters;
}
@Override
public Void apply(long value) {
// Do nothing if the tenant name is empty
if ( !StringUtils.isBlank(tntName) ) {
Tenants tenants = pulsarAdmin.tenants();
// Admin API - create tenants and namespaces
if (!adminDelOp) {
try {
Set<String> existingPulsarClusters = new HashSet<>();
Clusters clusters = pulsarAdmin.clusters();
CollectionUtils.addAll(existingPulsarClusters, clusters.getClusters().listIterator());
TenantInfo tenantInfo = TenantInfo.builder()
.adminRoles(adminRoles)
.allowedClusters(!allowedClusters.isEmpty() ? allowedClusters : existingPulsarClusters)
.build();
if (!asyncApi) {
tenants.createTenant(tntName, tenantInfo);
if (logger.isDebugEnabled()) {
logger.debug("Successful sync creation of tenant \"{}\"", tntName);
}
}
else {
CompletableFuture<Void> future = tenants.createTenantAsync(tntName, tenantInfo);
future.whenComplete((unused, throwable) -> {
if (logger.isDebugEnabled()) {
logger.debug("Successful async creation of tenant \"{}\"", tntName);
}
}).exceptionally(ex -> {
if (logger.isDebugEnabled()) {
logger.error("Failed async creation of tenant \"{}\"", tntName);
}
return null;
});
}
}
catch (PulsarAdminException.ConflictException ce) {
if (logger.isDebugEnabled()) {
logger.error("Tenant \"{}\" already exists - skip creation!", tntName);
}
}
catch (PulsarAdminException e) {
throw new PulsarAdapterUnexpectedException(
"Unexpected error when creating pulsar tenant \"" + tntName + "\"");
}
}
// Admin API - delete tenants and namespaces
else {
try {
Namespaces namespaces = pulsarAdmin.namespaces();
int nsNum = namespaces.getNamespaces(tntName).size();
// Only delete a tenant when there is no underlying namespaces
if ( nsNum == 0 ) {
if (!asyncApi) {
tenants.deleteTenant(tntName);
if (logger.isDebugEnabled()) {
logger.debug("Successful sync deletion of tenant \"{}\"", tntName);
}
}
else {
CompletableFuture<Void> future = tenants.deleteTenantAsync(tntName);
future.whenComplete((unused, throwable) -> {
if (logger.isDebugEnabled()) {
logger.debug("Successful async deletion of tenant \"{}\"", tntName);
}
}).exceptionally(ex -> {
if (logger.isDebugEnabled()) {
logger.error("Failed async deletion of tenant \"{}\"", tntName);
}
return null;
});
}
}
}
catch (PulsarAdminException.NotFoundException nfe) {
if (logger.isDebugEnabled()) {
logger.error("Tenant \"{}\" doesn't exists - skip deletion!", tntName);
}
}
catch (PulsarAdminException e) {
throw new PulsarAdapterUnexpectedException(
"Unexpected error when deleting pulsar tenant \"" + tntName + "\"");
}
}
}
return null;
}
}

181
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/AdminTopicOp.java Normal file
View File

@@ -0,0 +1,181 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.ops;
import io.nosqlbench.adapter.pulsar.exception.PulsarAdapterUnexpectedException;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.admin.Topics;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
public class AdminTopicOp extends PulsarAdminOp {
private final static Logger logger = LogManager.getLogger(AdminTopicOp.class);
private final String topicName;
private final boolean enablePart;
private final int partNum;
public AdminTopicOp(PulsarAdmin pulsarAdmin,
boolean asyncApi,
boolean adminDelOp,
String topicName,
boolean enablePart,
int partNum) {
super(pulsarAdmin, asyncApi, adminDelOp);
this.topicName = topicName;
this.enablePart = enablePart;
this.partNum = partNum;
}
@Override
public Void apply(long value) {
// Do nothing if the topic name is empty
if ( !StringUtils.isBlank(topicName) ) {
Topics topics = pulsarAdmin.topics();
try {
// Create the topic
if (!adminDelOp) {
if (!enablePart) {
if (!asyncApi) {
topics.createNonPartitionedTopic(topicName);
if (logger.isDebugEnabled()) {
logger.debug("Successful sync creation of non-partitioned topic \"{}\"", topicName);
}
} else {
CompletableFuture<Void> future = topics.createNonPartitionedTopicAsync(topicName);
future.whenComplete((unused, throwable) -> {
if (logger.isDebugEnabled()) {
logger.debug("Successful async creation of non-partitioned topic \"{}\"", topicName);
}
}).exceptionally(ex -> {
if (logger.isDebugEnabled()) {
logger.error("Failed async creation non-partitioned topic \"{}\"", topicName);
return null;
}
return null;
});
}
} else {
if (!asyncApi) {
topics.createPartitionedTopic(topicName, partNum);
if (logger.isDebugEnabled()) {
logger.debug(
"Successful sync creation of partitioned topic \"{} (partition_num: {}\")",
topicName, partNum);
}
} else {
CompletableFuture<Void> future = topics.createPartitionedTopicAsync(topicName, partNum);
future.whenComplete((unused, throwable) -> {
if (logger.isDebugEnabled()) {
logger.debug(
"Successful async creation of partitioned topic \"{} (partition_num: {}\")",
topicName, partNum);
}
})
.exceptionally(ex -> {
if (logger.isDebugEnabled()) {
logger.error(
"Successful async creation of partitioned topic \"{} (partition_num: {}\")",
topicName, partNum);
}
return null;
});
}
}
}
// Delete the topic
else {
if (!enablePart) {
if (!asyncApi) {
topics.delete(topicName);
if (logger.isDebugEnabled()) {
logger.debug(
"Successful sync deletion of non-partitioned topic \"{}\"",
topicName);
}
} else {
CompletableFuture<Void> future = topics.deleteAsync(topicName);
future.whenComplete((unused, throwable) -> {
if (logger.isDebugEnabled()) {
logger.debug(
"Successful async deletion of non-partitioned topic \"{}\"",
topicName);
}
})
.exceptionally(ex -> {
if (logger.isDebugEnabled()) {
logger.error(
"Failed async deletion of non-partitioned topic \"{}\"",
topicName);
}
return null;
});
}
} else {
if (!asyncApi) {
topics.deletePartitionedTopic(topicName);
if (logger.isDebugEnabled()) {
logger.debug(
"Successful sync deletion of partitioned topic \"{} (partition_num: {}\")",
topicName, partNum);
}
} else {
CompletableFuture<Void> future = topics.deletePartitionedTopicAsync(topicName);
future.whenComplete((unused, throwable) -> {
if (logger.isDebugEnabled()) {
logger.debug(
"Successful async deletion of partitioned topic \"{} (partition_num: {}\")",
topicName, partNum);
}
}).exceptionally(ex -> {
if (logger.isDebugEnabled()) {
logger.error(
"Failed async deletion of partitioned topic \"{} (partition_num: {}\")",
topicName, partNum);
}
return null;
});
}
}
}
}
catch (PulsarAdminException.NotFoundException nfe) {
if (logger.isDebugEnabled()) {
logger.error("Topic \"{}\" doesn't exists - skip deletion!", topicName);
}
}
catch (PulsarAdminException e) {
String errMsg = String.format("Unexpected error when %s pulsar topic: %s (partition enabled: %b; partition number: %d)",
(!adminDelOp ? "creating" : "deleting"),
topicName,
enablePart,
partNum);
throw new PulsarAdapterUnexpectedException(errMsg);
}
}
return null;
}
}

31
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageConsumerOp.java Normal file
View File

@@ -0,0 +1,31 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.ops;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.Schema;
public class MessageConsumerOp extends PulsarClientOp {
public MessageConsumerOp(PulsarClient pulsarClient, Schema<?> pulsarSchema) {
super(pulsarClient, pulsarSchema);
}
@Override
public Object apply(long value) {
return null;
}
}

32
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageProducerOp.java Normal file
View File

@@ -0,0 +1,32 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.ops;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.Schema;
public class MessageProducerOp extends PulsarClientOp {
public MessageProducerOp(PulsarClient pulsarClient, Schema<?> pulsarSchema) {
super(pulsarClient, pulsarSchema);
}
@Override
public Object apply(long value) {
return null;
}
}

23
driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOp.java → adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/MessageReaderOp.java
View File

@@ -14,16 +14,19 @@
* limitations under the License.
*/
package io.nosqlbench.driver.jms.ops;
package io.nosqlbench.adapter.pulsar.ops;
/**
* Base type of all Pulsar Operations including Producers and Consumers.
*/
public interface JmsOp {
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.Schema;
/**
* Execute the operation, invoke the timeTracker when the operation ended.
* The timeTracker can be invoked in a separate thread, it is only used for metrics.
*/
void run(Runnable timeTracker);
public class MessageReaderOp extends PulsarClientOp {
public MessageReaderOp(PulsarClient pulsarClient, Schema<?> pulsarSchema) {
super(pulsarClient, pulsarSchema);
}
@Override
public Object apply(long value) {
return null;
}
}

32
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarAdminOp.java Normal file
View File

@@ -0,0 +1,32 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.ops;
import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp;
import org.apache.pulsar.client.admin.PulsarAdmin;
public abstract class PulsarAdminOp extends PulsarOp {
protected PulsarAdmin pulsarAdmin;
protected boolean asyncApi;
protected boolean adminDelOp;
public PulsarAdminOp(PulsarAdmin pulsarAdmin, boolean asyncApi, boolean adminDelOp) {
this.pulsarAdmin = pulsarAdmin;
this.asyncApi = asyncApi;
this.adminDelOp = adminDelOp;
}
}

31
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarClientOp.java Normal file
View File

@@ -0,0 +1,31 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.ops;
import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.Schema;
public abstract class PulsarClientOp extends PulsarOp {
protected PulsarClient pulsarClient;
protected Schema<?> pulsarScheam;
public PulsarClientOp(PulsarClient pulsarClient, Schema<?> pulsarScheam) {
this.pulsarClient = pulsarClient;
this.pulsarScheam = pulsarScheam;
}
}

23
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/ops/PulsarOp.java Normal file
View File

@@ -0,0 +1,23 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.nosqlbench.adapter.pulsar.ops;
import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp;
public abstract class PulsarOp implements CycleOp<Object> {
}

123
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/AvroUtil.java Normal file
View File

@@ -0,0 +1,123 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.util;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.JsonDecoder;
import org.apache.avro.io.BinaryDecoder;
import org.apache.pulsar.client.api.schema.Field;
import org.apache.pulsar.client.api.schema.GenericRecord;
import org.apache.pulsar.client.api.schema.GenericRecordBuilder;
import org.apache.pulsar.client.impl.schema.SchemaInfoImpl;
import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema;
import org.apache.pulsar.common.schema.SchemaInfo;
import org.apache.pulsar.common.schema.SchemaType;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
public class AvroUtil {
////////////////////////
// Get an OSS Apache Avro schema from a string definition
public static org.apache.avro.Schema GetSchema_ApacheAvro(String avroSchemDef) {
return new org.apache.avro.Schema.Parser().parse(avroSchemDef);
}
// Get an OSS Apache Avro schema record from a JSON string that matches a specific OSS Apache Avro schema
public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, String jsonData) {
org.apache.avro.generic.GenericRecord record = null;
try {
org.apache.avro.generic.GenericDatumReader<org.apache.avro.generic.GenericData.Record> reader;
reader = new org.apache.avro.generic.GenericDatumReader<>(schema);
JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, jsonData);
record = reader.read(null, decoder);
} catch (IOException ioe) {
ioe.printStackTrace();
}
return record;
}
// Get an OSS Apache Avro schema record from a byte array that matches a specific OSS Apache Avro schema
public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, byte[] bytesData) {
org.apache.avro.generic.GenericRecord record = null;
try {
org.apache.avro.generic.GenericDatumReader<org.apache.avro.generic.GenericData.Record> reader;
reader = new org.apache.avro.generic.GenericDatumReader<>(schema);
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytesData, null);
record = reader.read(null, decoder);
} catch (IOException ioe) {
ioe.printStackTrace();
}
return record;
}
////////////////////////
// Get a Pulsar Avro schema from a string definition
public static GenericAvroSchema GetSchema_PulsarAvro(String schemaName, String avroSchemDef) {
SchemaInfo schemaInfo = SchemaInfoImpl.builder()
.schema(avroSchemDef.getBytes(StandardCharsets.UTF_8))
.type(SchemaType.AVRO)
.properties(new HashMap<>())
.name(schemaName)
.build();
return new GenericAvroSchema(schemaInfo);
}
// Get a Pulsar Avro record from an OSS Avro schema record, matching a specific Pulsar Avro schema
public static GenericRecord GetGenericRecord_PulsarAvro(
GenericAvroSchema pulsarGenericAvroSchema,
org.apache.avro.generic.GenericRecord apacheAvroGenericRecord)
{
GenericRecordBuilder recordBuilder = pulsarGenericAvroSchema.newRecordBuilder();
List<Field> fieldList = pulsarGenericAvroSchema.getFields();
for (Field field : fieldList) {
String fieldName = field.getName();
recordBuilder.set(fieldName, apacheAvroGenericRecord.get(fieldName));
}
return recordBuilder.build();
}
// Get a Pulsar Avro record (GenericRecord) from a JSON string that matches a specific Pulsar Avro schema
public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, String avroSchemDefStr, String jsonData) {
org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr);
return GetGenericRecord_PulsarAvro(genericAvroSchema, avroSchema, jsonData);
}
public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, org.apache.avro.Schema avroSchema, String jsonData) {
org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData);
return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord);
}
public static GenericRecord GetGenericRecord_PulsarAvro(String schemaName, String avroSchemDefStr, String jsonData) {
GenericAvroSchema genericAvroSchema = GetSchema_PulsarAvro(schemaName, avroSchemDefStr);
org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr);
org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData);
return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord);
}
}

538
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarAdapterUtil.java Normal file
View File

@@ -0,0 +1,538 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.util;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.api.Schema;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class PulsarAdapterUtil {
private final static Logger logger = LogManager.getLogger(PulsarAdapterUtil.class);
// Supported message operation types
// TODO: websocket-producer and managed-ledger
public enum OP_TYPES {
ADMIN_TENANT("admin-tenant"),
ADMIN_NAMESPACE("admin-namespace"),
ADMIN_TOPIC("admin-topic"),
E2E_MSG_PROC_SEND("e22-msg-proc-send"),
E2E_MSG_PROC_CONSUME("e22-msg-proc-consume"),
// BATCH_MSG_SEND_START("batch-msg-send-start"),
// BATCH_MSG_SEND("batch-msg-send"),
// BATCH_MSG_SEND_END("batch-msg-send-end"),
MSG_SEND("msg-send"),
MSG_CONSUME("msg-consume"),
MSG_READ("msg-read"),
MSG_MULTI_CONSUME("msg-mt-consume");
public final String label;
OP_TYPES(String label) {
this.label = label;
}
}
public static boolean isValidClientType(String type) {
return Arrays.stream(OP_TYPES.values()).anyMatch(t -> t.label.equals(type));
}
public static final String MSG_SEQUENCE_NUMBER = "sequence_number";
///////
// Valid document level parameters for Pulsar NB yaml file
public enum DOC_LEVEL_PARAMS {
TOPIC_URI("topic_uri"),
ASYNC_API("async_api"),
USE_TRANSACTION("use_transaction"),
ADMIN_DELOP("admin_delop"),
SEQ_TRACKING("seq_tracking"),
MSG_DEDUP_BROKER("msg_dedup_broker"),
E2E_STARTING_TIME_SOURCE("e2e_starting_time_source");
public final String label;
DOC_LEVEL_PARAMS(String label) {
this.label = label;
}
}
public static boolean isValidDocLevelParam(String param) {
return Arrays.stream(DOC_LEVEL_PARAMS.values()).anyMatch(t -> t.label.equals(param));
}
///////
// Valid Pulsar API type
public enum PULSAR_API_TYPE {
PRODUCER("producer"),
CONSUMER("consumer"),
READER("reader");
public final String label;
PULSAR_API_TYPE(String label) {
this.label = label;
}
}
public static boolean isValidPulsarApiType(String param) {
return Arrays.stream(PULSAR_API_TYPE.values()).anyMatch(t -> t.label.equals(param));
}
public static String getValidPulsarApiTypeList() {
return Arrays.stream(PULSAR_API_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", "));
}
///////
// Valid persistence type
public enum PERSISTENT_TYPES {
PERSISTENT("persistent"),
NON_PERSISTENT("non-persistent")
;
public final String label;
PERSISTENT_TYPES(String label) {
this.label = label;
}
}
public static boolean isValidPersistenceType(String type) {
return Arrays.stream(PERSISTENT_TYPES.values()).anyMatch(t -> t.label.equals(type));
}
///////
// Valid Pulsar client configuration (activity-level settings)
// - https://pulsar.apache.org/docs/en/client-libraries-java/#client
public enum CLNT_CONF_KEY {
serviceUrl("serviceUrl"),
authPulginClassName("authPluginClassName"),
authParams("authParams"),
pperationTimeoutMs("operationTimeoutMs"),
statsIntervalSeconds("statsIntervalSeconds"),
numIoThreads("numIoThreads"),
numListenerThreads("numListenerThreads"),
useTcpNoDelay("useTcpNoDelay"),
enableTls("enableTls"),
tlsTrustCertsFilePath("tlsTrustCertsFilePath"),
tlsAllowInsecureConnection("tlsAllowInsecureConnection"),
tlsHostnameVerificationEnable("tlsHostnameVerificationEnable"),
concurrentLookupRequest("concurrentLookupRequest"),
maxLookupRequest("maxLookupRequest"),
maxNumberOfRejectedRequestPerConnection("maxNumberOfRejectedRequestPerConnection"),
keepAliveIntervalSeconds("keepAliveIntervalSeconds"),
connectionTimeoutMs("connectionTimeoutMs"),
requestTimeoutMs("requestTimeoutMs"),
defaultBackoffIntervalNanos("defaultBackoffIntervalNanos"),
maxBackoffIntervalNanos("maxBackoffIntervalNanos"),
socks5ProxyAddress("socks5ProxyAddress"),
socks5ProxyUsername("socks5ProxyUsername"),
socks5ProxyPassword("socks5ProxyPassword")
;
public final String label;
CLNT_CONF_KEY(String label) {
this.label = label;
}
}
public static boolean isValidClientConfItem(String item) {
return Arrays.stream(CLNT_CONF_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Standard producer configuration (activity-level settings)
// - https://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer
public enum PRODUCER_CONF_STD_KEY {
topicName("topicName"),
producerName("producerName"),
sendTimeoutMs("sendTimeoutMs"),
blockIfQueueFull("blockIfQueueFull"),
maxPendingMessages("maxPendingMessages"),
maxPendingMessagesAcrossPartitions("maxPendingMessagesAcrossPartitions"),
messageRoutingMode("messageRoutingMode"),
hashingScheme("hashingScheme"),
cryptoFailureAction("cryptoFailureAction"),
batchingMaxPublishDelayMicros("batchingMaxPublishDelayMicros"),
batchingMaxMessages("batchingMaxMessages"),
batchingEnabled("batchingEnabled"),
compressionType("compressionType");
public final String label;
PRODUCER_CONF_STD_KEY(String label) {
this.label = label;
}
}
public static boolean isStandardProducerConfItem(String item) {
return Arrays.stream(PRODUCER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Standard consumer configuration (activity-level settings)
// - https://pulsar.apache.org/docs/en/client-libraries-java/#consumer
public enum CONSUMER_CONF_STD_KEY {
topicNames("topicNames"),
topicsPattern("topicsPattern"),
subscriptionName("subscriptionName"),
subscriptionType("subscriptionType"),
receiverQueueSize("receiverQueueSize"),
acknowledgementsGroupTimeMicros("acknowledgementsGroupTimeMicros"),
negativeAckRedeliveryDelayMicros("negativeAckRedeliveryDelayMicros"),
maxTotalReceiverQueueSizeAcrossPartitions("maxTotalReceiverQueueSizeAcrossPartitions"),
consumerName("consumerName"),
ackTimeoutMillis("ackTimeoutMillis"),
tickDurationMillis("tickDurationMillis"),
priorityLevel("priorityLevel"),
cryptoFailureAction("cryptoFailureAction"),
properties("properties"),
readCompacted("readCompacted"),
subscriptionInitialPosition("subscriptionInitialPosition"),
patternAutoDiscoveryPeriod("patternAutoDiscoveryPeriod"),
regexSubscriptionMode("regexSubscriptionMode"),
deadLetterPolicy("deadLetterPolicy"),
autoUpdatePartitions("autoUpdatePartitions"),
replicateSubscriptionState("replicateSubscriptionState");
public final String label;
CONSUMER_CONF_STD_KEY(String label) {
this.label = label;
}
}
public static boolean isStandardConsumerConfItem(String item) {
return Arrays.stream(CONSUMER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Custom consumer configuration (activity-level settings)
// - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#consumer
// - NB Pulsar driver consumer operation specific
public enum CONSUMER_CONF_CUSTOM_KEY {
timeout("timeout");
public final String label;
CONSUMER_CONF_CUSTOM_KEY(String label) {
this.label = label;
}
}
public static boolean isCustomConsumerConfItem(String item) {
return Arrays.stream(CONSUMER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Pulsar subscription type
public enum SUBSCRIPTION_TYPE {
Exclusive("Exclusive"),
Failover("Failover"),
Shared("Shared"),
Key_Shared("Key_Shared");
public final String label;
SUBSCRIPTION_TYPE(String label) {
this.label = label;
}
}
public static boolean isValidSubscriptionType(String item) {
return Arrays.stream(SUBSCRIPTION_TYPE.values()).anyMatch(t -> t.label.equals(item));
}
public static String getValidSubscriptionTypeList() {
return Arrays.stream(SUBSCRIPTION_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", "));
}
///////
// Standard reader configuration (activity-level settings)
// - https://pulsar.apache.org/docs/en/client-libraries-java/#reader
public enum READER_CONF_STD_KEY {
topicName("topicName"),
receiverQueueSize("receiverQueueSize"),
readerListener("readerListener"),
readerName("readerName"),
subscriptionRolePrefix("subscriptionRolePrefix"),
cryptoKeyReader("cryptoKeyReader"),
cryptoFailureAction("cryptoFailureAction"),
readCompacted("readCompacted"),
resetIncludeHead("resetIncludeHead");
public final String label;
READER_CONF_STD_KEY(String label) {
this.label = label;
}
}
public static boolean isStandardReaderConfItem(String item) {
return Arrays.stream(READER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Custom reader configuration (activity-level settings)
// - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#reader
// - NB Pulsar driver reader operation specific
public enum READER_CONF_CUSTOM_KEY {
startMessagePos("startMessagePos");
public final String label;
READER_CONF_CUSTOM_KEY(String label) {
this.label = label;
}
}
public static boolean isCustomReaderConfItem(String item) {
return Arrays.stream(READER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Valid read positions for a Pulsar reader
public enum READER_MSG_POSITION_TYPE {
earliest("earliest"),
latest("latest"),
custom("custom");
public final String label;
READER_MSG_POSITION_TYPE(String label) {
this.label = label;
}
}
public static boolean isValideReaderStartPosition(String item) {
return Arrays.stream(READER_MSG_POSITION_TYPE.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Pulsar subscription type
public enum SEQ_ERROR_SIMU_TYPE {
OutOfOrder("out_of_order"),
MsgLoss("msg_loss"),
MsgDup("msg_dup");
public final String label;
SEQ_ERROR_SIMU_TYPE(String label) {
this.label = label;
}
private static final Map<String, SEQ_ERROR_SIMU_TYPE> MAPPING = new HashMap<>();
static {
for (SEQ_ERROR_SIMU_TYPE simuType : values()) {
MAPPING.put(simuType.label, simuType);
MAPPING.put(simuType.label.toLowerCase(), simuType);
MAPPING.put(simuType.label.toUpperCase(), simuType);
MAPPING.put(simuType.name(), simuType);
MAPPING.put(simuType.name().toLowerCase(), simuType);
MAPPING.put(simuType.name().toUpperCase(), simuType);
}
}
public static Optional<SEQ_ERROR_SIMU_TYPE> parseSimuType(String simuTypeString) {
return Optional.ofNullable(MAPPING.get(simuTypeString.trim()));
}
}
public static boolean isValidSeqErrSimuType(String item) {
return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).anyMatch(t -> t.label.equals(item));
}
public static String getValidSeqErrSimuTypeList() {
return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", "));
}
///////
// Valid websocket-producer configuration (activity-level settings)
// TODO: to be added
public enum WEBSKT_PRODUCER_CONF_KEY {
;
public final String label;
WEBSKT_PRODUCER_CONF_KEY(String label) {
this.label = label;
}
}
///////
// Valid managed-ledger configuration (activity-level settings)
// TODO: to be added
public enum MANAGED_LEDGER_CONF_KEY {
;
public final String label;
MANAGED_LEDGER_CONF_KEY(String label) {
this.label = label;
}
}
///////
// Primitive Schema type
public static boolean isPrimitiveSchemaTypeStr(String typeStr) {
boolean isPrimitive = false;
// Use "BYTES" as the default type if the type string is not explicitly specified
if (StringUtils.isBlank(typeStr)) {
typeStr = "BYTES";
}
if (typeStr.equalsIgnoreCase("BOOLEAN") || typeStr.equalsIgnoreCase("INT8") ||
typeStr.equalsIgnoreCase("INT16") || typeStr.equalsIgnoreCase("INT32") ||
typeStr.equalsIgnoreCase("INT64") || typeStr.equalsIgnoreCase("FLOAT") ||
typeStr.equalsIgnoreCase("DOUBLE") || typeStr.equalsIgnoreCase("BYTES") ||
typeStr.equalsIgnoreCase("DATE") || typeStr.equalsIgnoreCase("TIME") ||
typeStr.equalsIgnoreCase("TIMESTAMP") || typeStr.equalsIgnoreCase("INSTANT") ||
typeStr.equalsIgnoreCase("LOCAL_DATE") || typeStr.equalsIgnoreCase("LOCAL_TIME") ||
typeStr.equalsIgnoreCase("LOCAL_DATE_TIME")) {
isPrimitive = true;
}
return isPrimitive;
}
public static Schema<?> getPrimitiveTypeSchema(String typeStr) {
Schema<?> schema;
switch (typeStr.toUpperCase()) {
case "BOOLEAN":
schema = Schema.BOOL;
break;
case "INT8":
schema = Schema.INT8;
break;
case "INT16":
schema = Schema.INT16;
break;
case "INT32":
schema = Schema.INT32;
break;
case "INT64":
schema = Schema.INT64;
break;
case "FLOAT":
schema = Schema.FLOAT;
break;
case "DOUBLE":
schema = Schema.DOUBLE;
break;
case "DATE":
schema = Schema.DATE;
break;
case "TIME":
schema = Schema.TIME;
break;
case "TIMESTAMP":
schema = Schema.TIMESTAMP;
break;
case "INSTANT":
schema = Schema.INSTANT;
break;
case "LOCAL_DATE":
schema = Schema.LOCAL_DATE;
break;
case "LOCAL_TIME":
schema = Schema.LOCAL_TIME;
break;
case "LOCAL_DATE_TIME":
schema = Schema.LOCAL_DATE_TIME;
break;
// Use BYTES as the default schema type if the type string is not specified
case "":
case "BYTES":
schema = Schema.BYTES;
break;
// Report an error if non-valid, non-empty schema type string is provided
default:
throw new RuntimeException("Invalid Pulsar primitive schema type string : " + typeStr);
}
return schema;
}
///////
// Complex strut type: Avro or Json
public static boolean isAvroSchemaTypeStr(String typeStr) {
return typeStr.equalsIgnoreCase("AVRO");
}
public static boolean isKeyValueTypeStr(String typeStr) {
return typeStr.equalsIgnoreCase("KEY_VALUE");
}
// automatic decode the type from the Registry
public static boolean isAutoConsumeSchemaTypeStr(String typeStr) {
return typeStr.equalsIgnoreCase("AUTO_CONSUME");
}
public static Schema<?> getAvroSchema(String typeStr, String definitionStr) {
String schemaDefinitionStr = definitionStr;
String filePrefix = "file://";
Schema<?> schema;
// Check if payloadStr points to a file (e.g. "file:///path/to/a/file")
if (isAvroSchemaTypeStr(typeStr)) {
if (StringUtils.isBlank(schemaDefinitionStr)) {
throw new RuntimeException("Schema definition must be provided for \"Avro\" schema type!");
} else if (schemaDefinitionStr.startsWith(filePrefix)) {
try {
Path filePath = Paths.get(URI.create(schemaDefinitionStr));
schemaDefinitionStr = Files.readString(filePath, StandardCharsets.US_ASCII);
} catch (IOException ioe) {
throw new RuntimeException("Error reading the specified \"Avro\" schema definition file: " + definitionStr + ": " + ioe.getMessage());
}
}
schema = AvroUtil.GetSchema_PulsarAvro("NBAvro", schemaDefinitionStr);
} else {
throw new RuntimeException("Trying to create a \"Avro\" schema for a non-Avro schema type string: " + typeStr);
}
return schema;
}
///////
// Generate effective key string
public static String buildCacheKey(String... keyParts) {
// Ignore blank keyPart
String joinedKeyStr =
Stream.of(keyParts)
.filter(s -> !StringUtils.isBlank(s))
.collect(Collectors.joining(","));
return Base64.getEncoder().encodeToString(joinedKeyStr.getBytes());
}
///////
// Convert JSON string to a key/value map
public static Map<String, String> convertJsonToMap(String jsonStr) throws Exception {
ObjectMapper mapper = new ObjectMapper();
return mapper.readValue(jsonStr, Map.class);
}
///////
// Get full namespace name (<tenant>/<namespace>) from a Pulsar topic URI
public static String getFullNamespaceName(String topicUri) {
// Get tenant/namespace string
// - topicUri : persistent://<tenant>/<namespace>/<topic>
// - tmpStr : <tenant>/<namespace>/<topic>
// - fullNsName : <tenant>/<namespace>
String tmpStr = StringUtils.substringAfter(topicUri,"://");
return StringUtils.substringBeforeLast(tmpStr, "/");
}
}

336
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/util/PulsarNBClientConf.java Normal file
View File

@@ -0,0 +1,336 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.pulsar.util;
import org.apache.commons.configuration2.Configuration;
import org.apache.commons.configuration2.FileBasedConfiguration;
import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder;
import org.apache.commons.configuration2.builder.fluent.Parameters;
import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
public class PulsarNBClientConf {
private final static Logger logger = LogManager.getLogger(PulsarNBClientConf.class);
private String canonicalFilePath = "";
public static final String SCHEMA_CONF_PREFIX = "schema";
public static final String CLIENT_CONF_PREFIX = "client";
public static final String PRODUCER_CONF_PREFIX = "producer";
public static final String CONSUMER_CONF_PREFIX = "consumer";
public static final String READER_CONF_PREFIX = "reader";
private final HashMap<String, Object> schemaConfMap = new HashMap<>();
private final HashMap<String, Object> clientConfMap = new HashMap<>();
private final HashMap<String, Object> producerConfMap = new HashMap<>();
private final HashMap<String, Object> consumerConfMap = new HashMap<>();
private final HashMap<String, Object> readerConfMap = new HashMap<>();
// TODO: add support for other operation types: websocket-producer, managed-ledger
public PulsarNBClientConf(String fileName) {
File file = new File(fileName);
try {
canonicalFilePath = file.getCanonicalPath();
Parameters params = new Parameters();
FileBasedConfigurationBuilder<FileBasedConfiguration> builder =
new FileBasedConfigurationBuilder<FileBasedConfiguration>(PropertiesConfiguration.class)
.configure(params.properties()
.setFileName(fileName));
Configuration config = builder.getConfiguration();
// Get schema specific configuration settings
for (Iterator<String> it = config.getKeys(SCHEMA_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
schemaConfMap.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
// Get client connection specific configuration settings
for (Iterator<String> it = config.getKeys(CLIENT_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
clientConfMap.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
// Get producer specific configuration settings
for (Iterator<String> it = config.getKeys(PRODUCER_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
producerConfMap.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
// Get consumer specific configuration settings
for (Iterator<String> it = config.getKeys(CONSUMER_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
consumerConfMap.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
// Get reader specific configuration settings
for (Iterator<String> it = config.getKeys(READER_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
readerConfMap.put(confKey.substring(READER_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
} catch (IOException ioe) {
logger.error("Can't read the specified config properties file!");
ioe.printStackTrace();
} catch (ConfigurationException cex) {
logger.error("Error loading configuration items from the specified config properties file: " + canonicalFilePath);
cex.printStackTrace();
}
}
//////////////////
// Get Schema related config
public Map<String, Object> getSchemaConfMap() {
return this.schemaConfMap;
}
public boolean hasSchemaConfKey(String key) {
if (key.contains(SCHEMA_CONF_PREFIX))
return schemaConfMap.containsKey(key.substring(SCHEMA_CONF_PREFIX.length() + 1));
else
return schemaConfMap.containsKey(key);
}
public Object getSchemaConfValue(String key) {
if (key.contains(SCHEMA_CONF_PREFIX))
return schemaConfMap.get(key.substring(SCHEMA_CONF_PREFIX.length()+1));
else
return schemaConfMap.get(key);
}
public void setSchemaConfValue(String key, Object value) {
if (key.contains(SCHEMA_CONF_PREFIX))
schemaConfMap.put(key.substring(SCHEMA_CONF_PREFIX.length() + 1), value);
else
schemaConfMap.put(key, value);
}
//////////////////
// Get Pulsar client related config
public Map<String, Object> getClientConfMap() {
return this.clientConfMap;
}
public boolean hasClientConfKey(String key) {
if (key.contains(CLIENT_CONF_PREFIX))
return clientConfMap.containsKey(key.substring(CLIENT_CONF_PREFIX.length() + 1));
else
return clientConfMap.containsKey(key);
}
public Object getClientConfValue(String key) {
if (key.contains(CLIENT_CONF_PREFIX))
return clientConfMap.get(key.substring(CLIENT_CONF_PREFIX.length()+1));
else
return clientConfMap.get(key);
}
public void setClientConfValue(String key, Object value) {
if (key.contains(CLIENT_CONF_PREFIX))
clientConfMap.put(key.substring(CLIENT_CONF_PREFIX.length() + 1), value);
else
clientConfMap.put(key, value);
}
//////////////////
// Get Pulsar producer related config
public Map<String, Object> getProducerConfMap() {
return this.producerConfMap;
}
public boolean hasProducerConfKey(String key) {
if (key.contains(PRODUCER_CONF_PREFIX))
return producerConfMap.containsKey(key.substring(PRODUCER_CONF_PREFIX.length() + 1));
else
return producerConfMap.containsKey(key);
}
public Object getProducerConfValue(String key) {
if (key.contains(PRODUCER_CONF_PREFIX))
return producerConfMap.get(key.substring(PRODUCER_CONF_PREFIX.length()+1));
else
return producerConfMap.get(key);
}
public void setProducerConfValue(String key, Object value) {
if (key.contains(PRODUCER_CONF_PREFIX))
producerConfMap.put(key.substring(PRODUCER_CONF_PREFIX.length()+1), value);
else
producerConfMap.put(key, value);
}
// other producer helper functions ...
public String getProducerName() {
Object confValue = getProducerConfValue(
"producer." + PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.producerName.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getProducerTopicName() {
Object confValue = getProducerConfValue(
"producer." + PulsarAdapterUtil.PRODUCER_CONF_STD_KEY.topicName);
if (confValue == null)
return "";
else
return confValue.toString();
}
//////////////////
// Get Pulsar consumer related config
public Map<String, Object> getConsumerConfMap() {
return this.consumerConfMap;
}
public boolean hasConsumerConfKey(String key) {
if (key.contains(CONSUMER_CONF_PREFIX))
return consumerConfMap.containsKey(key.substring(CONSUMER_CONF_PREFIX.length() + 1));
else
return consumerConfMap.containsKey(key);
}
public Object getConsumerConfValue(String key) {
if (key.contains(CONSUMER_CONF_PREFIX))
return consumerConfMap.get(key.substring(CONSUMER_CONF_PREFIX.length() + 1));
else
return consumerConfMap.get(key);
}
public void setConsumerConfValue(String key, Object value) {
if (key.contains(CONSUMER_CONF_PREFIX))
consumerConfMap.put(key.substring(CONSUMER_CONF_PREFIX.length() + 1), value);
else
consumerConfMap.put(key, value);
}
// Other consumer helper functions ...
public String getConsumerTopicNames() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicNames.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getConsumerTopicPattern() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getConsumerSubscriptionName() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getConsumerSubscriptionType() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getConsumerName() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarAdapterUtil.CONSUMER_CONF_STD_KEY.consumerName.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
// NOTE: Below are not a standard Pulsar consumer configuration parameter as
// listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer"
// They're custom-made configuration properties for NB pulsar driver consumer.
public int getConsumerTimeoutSeconds() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarAdapterUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label);
if (confValue == null)
return -1; // infinite
else
return Integer.parseInt(confValue.toString());
}
//////////////////
// Get Pulsar reader related config
public Map<String, Object> getReaderConfMap() {
return this.readerConfMap;
}
public boolean hasReaderConfKey(String key) {
if (key.contains(READER_CONF_PREFIX))
return readerConfMap.containsKey(key.substring(READER_CONF_PREFIX.length() + 1));
else
return readerConfMap.containsKey(key);
}
public Object getReaderConfValue(String key) {
if (key.contains(READER_CONF_PREFIX))
return readerConfMap.get(key.substring(READER_CONF_PREFIX.length() + 1));
else
return readerConfMap.get(key);
}
public void setReaderConfValue(String key, Object value) {
if (key.contains(READER_CONF_PREFIX))
readerConfMap.put(key.substring(READER_CONF_PREFIX.length() + 1), value);
else
readerConfMap.put(key, value);
}
// Other reader helper functions ...
public String getReaderTopicName() {
Object confValue = getReaderConfValue(
"reader." + PulsarAdapterUtil.READER_CONF_STD_KEY.topicName.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getReaderName() {
Object confValue = getReaderConfValue(
"reader." + PulsarAdapterUtil.READER_CONF_STD_KEY.readerName.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
// NOTE: Below are not a standard Pulsar reader configuration parameter as
// listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#reader"
// They're custom-made configuration properties for NB pulsar driver reader.
public String getStartMsgPosStr() {
Object confValue = getReaderConfValue(
"reader." + PulsarAdapterUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
}

16
adapter-pulsar/src/main/resources/admin-namespace.yaml Normal file
View File

@@ -0,0 +1,16 @@
bindings:
# 20 topics: 10 tenants, 2 namespaces/tenant
tenant: Mod(20); Div(2); ToString(); Prefix("tnt")
namespace: Mod(2); ToString(); Prefix("ns")
params:
async_api: "false"
admin_delop: "true"
blocks:
admin-namespace-block:
tags:
phase: admin-namespace
ops:
op1:
AdminNamespace: "{tenant}/{namespace}"

17
adapter-pulsar/src/main/resources/admin-tenant.yaml Normal file
View File

@@ -0,0 +1,17 @@
bindings:
# 10 tenants
tenant: Mod(10); ToString(); Prefix("tnt")
params:
async_api: "false"
admin_delop: "true"
blocks:
admin-tenant-block:
tags:
phase: admin-tenant
ops:
op1:
AdminTenant: "{tenant}"
admin_roles: ""
allowed_clusters: ""

19
adapter-pulsar/src/main/resources/admin-topic.yaml Normal file
View File

@@ -0,0 +1,19 @@
bindings:
# 100 topics: 10 tenants, 2 namespaces/tenant, 5 topics/namespace
tenant: Mod(100); Div(10); ToString(); Prefix("tnt")
namespace: Mod(10); Div(5); ToString(); Prefix("ns")
topic: Mod(5); ToString(); Prefix("tp")
params:
async_api: "false"
admin_delop: "true"
blocks:
admin-topic-block:
tags:
phase: admin-topic
ops:
op1:
AdminTopic: "{tenant}/{namespace}/{topic}"
enable_partition: "true"
partition_num: "5"

4
adapter-pulsar/src/main/resources/bindingtest.yaml Normal file
View File

@@ -0,0 +1,4 @@
bindings:
tenant: Mod(100); Div(10); ToString(); Prefix("tnt")
namespace: Mod(10); Div(5); ToString(); Prefix("ns")
topic: Mod(5); ToString(); Prefix("tp")

52
adapter-pulsar/src/main/resources/config.properties Normal file
View File

@@ -0,0 +1,52 @@
### Schema related configurations - schema.xxx
# valid types:
# - primitive type (https://pulsar.apache.org/docs/en/schema-understand/#primitive-type)
# - keyvalue (https://pulsar.apache.org/docs/en/schema-understand/#keyvalue)
# - strut (complex type) (https://pulsar.apache.org/docs/en/schema-understand/#struct)
# avro, json, protobuf
#
# TODO: as a starting point, only supports the following types
# 1) primitive types, including bytearray (byte[]) which is default, for messages without schema
# 2) Avro for messages with schema
#schema.type=avro
#schema.definition=file:///Users/yabinmeng/DataStax/MyNoSQLBench/nosqlbench/driver-pulsar/src/main/resources/activities/iot-example.avsc
schema.type=
schema.definition=
### Pulsar client related configurations - client.xxx
# http://pulsar.apache.org/docs/en/client-libraries-java/#client
client.connectionTimeoutMs=5000
client.authPluginClassName=org.apache.pulsar.client.impl.auth.AuthenticationToken
# Cluster admin
client.authParams=
client.tlsAllowInsecureConnection=true
### Producer related configurations (global) - producer.xxx
# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer
producer.producerName=
producer.topicName=
producer.sendTimeoutMs=
producer.blockIfQueueFull=true
producer.maxPendingMessages=5000
producer.batchingMaxMessages=5000
### Consumer related configurations (global) - consumer.xxx
# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer
consumer.topicNames=
consumer.topicsPattern=
consumer.subscriptionName=
consumer.subscriptionType=
consumer.consumerName=
consumer.receiverQueueSize=
### Reader related configurations (global) - reader.xxx
# https://pulsar.apache.org/docs/en/client-libraries-java/#reader
# - valid Pos: earliest, latest, custom::file://<path>/<to>/<message_id_file>
reader.topicName=
reader.receiverQueueSize=
reader.readerName=
reader.startMessagePos=earliest

84
driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsAction.java
View File

@@ -1,84 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms;
import com.codahale.metrics.Timer;
import io.nosqlbench.driver.jms.ops.JmsOp;
import io.nosqlbench.engine.api.activityapi.core.SyncAction;
import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorDetail;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.function.LongFunction;
public class JmsAction implements SyncAction {
private final static Logger logger = LogManager.getLogger(JmsAction.class);
private final JmsActivity activity;
private final int slot;
int maxTries;
public JmsAction(JmsActivity activity, int slot) {
this.activity = activity;
this.slot = slot;
this.maxTries = activity.getActivityDef().getParams().getOptionalInteger("maxtries").orElse(10);
}
@Override
public void init() { }
@Override
public int runCycle(long cycle) {
// let's fail the action if some async operation failed
activity.failOnAsyncOperationFailure();
long start = System.nanoTime();
JmsOp jmsOp;
try (Timer.Context ctx = activity.getBindTimer().time()) {
LongFunction<? extends JmsOp> readyJmsOp = activity.getSequencer().apply(cycle);
jmsOp = readyJmsOp.apply(cycle);
} catch (Exception bindException) {
// if diagnostic mode ...
activity.getErrorhandler().handleError(bindException, cycle, 0);
throw new RuntimeException(
"while binding request in cycle " + cycle + ": " + bindException.getMessage(), bindException
);
}
for (int i = 0; i < maxTries; i++) {
Timer.Context ctx = activity.getExecuteTimer().time();
try {
// it is up to the jmsOp to call Context#close when the activity is executed
// this allows us to track time for async operations
jmsOp.run(ctx::close);
break;
} catch (RuntimeException err) {
ErrorDetail errorDetail = activity
.getErrorhandler()
.handleError(err, cycle, System.nanoTime() - start);
if (!errorDetail.isRetryable()) {
break;
}
}
}
return 0;
}
}

179
driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivity.java
View File

@@ -1,179 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Timer;
import com.datastax.oss.pulsar.jms.PulsarConnectionFactory;
import io.nosqlbench.driver.jms.conn.JmsConnInfo;
import io.nosqlbench.driver.jms.conn.JmsPulsarConnInfo;
import io.nosqlbench.driver.jms.ops.JmsOp;
import io.nosqlbench.driver.jms.util.JmsUtil;
import io.nosqlbench.driver.jms.util.PulsarConfig;
import io.nosqlbench.engine.api.activityapi.errorhandling.modular.NBErrorHandler;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.OpDispenser;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.api.engine.metrics.ActivityMetrics;
import org.apache.commons.lang3.StringUtils;
import javax.jms.Destination;
import javax.jms.JMSContext;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
public class JmsActivity extends SimpleActivity {
private final ConcurrentHashMap<String, Destination> jmsDestinations = new ConcurrentHashMap<>();
private String jmsProviderType;
private JmsConnInfo jmsConnInfo;
private JMSContext jmsContext;
private OpSequence<OpDispenser<? extends JmsOp>> sequence;
private volatile Throwable asyncOperationFailure;
private NBErrorHandler errorhandler;
private Timer bindTimer;
private Timer executeTimer;
private Counter bytesCounter;
private Histogram messagesizeHistogram;
public JmsActivity(ActivityDef activityDef) {
super(activityDef);
}
@Override
public void initActivity() {
super.initActivity();
// default JMS type: Pulsar
// - currently this is the only supported JMS provider
jmsProviderType =
activityDef.getParams()
.getOptionalString(JmsUtil.JMS_PROVIDER_TYPE_KEY_STR)
.orElse(JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label);
// "Pulsar" as the JMS provider
if (StringUtils.equalsIgnoreCase(jmsProviderType, JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label )) {
String webSvcUrl =
activityDef.getParams()
.getOptionalString(JmsUtil.JMS_PULSAR_PROVIDER_WEB_URL_KEY_STR)
.orElse("http://localhost:8080");
String pulsarSvcUrl =
activityDef.getParams()
.getOptionalString(JmsUtil.JMS_PULSAR_PROVIDER_SVC_URL_KEY_STR)
.orElse("pulsar://localhost:6650");
if (StringUtils.isAnyBlank(webSvcUrl, pulsarSvcUrl)) {
throw new RuntimeException("For \"" + JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label + "\" type, " +
"\"" + JmsUtil.JMS_PULSAR_PROVIDER_WEB_URL_KEY_STR + "\" and " +
"\"" + JmsUtil.JMS_PULSAR_PROVIDER_SVC_URL_KEY_STR + "\" parameters are manadatory!");
}
// Check if extra Pulsar config. file is in place
// - default file: "pulsar_config.properties" under the current directory
String pulsarCfgFile =
activityDef.getParams()
.getOptionalString(JmsUtil.JMS_PULSAR_PROVIDER_CFG_FILE_KEY_STR)
.orElse(JmsUtil.JMS_PULSAR_PROVIDER_DFT_CFG_FILE_NAME);
PulsarConfig pulsarConfig = new PulsarConfig(pulsarCfgFile);
jmsConnInfo = new JmsPulsarConnInfo(jmsProviderType, webSvcUrl, pulsarSvcUrl, pulsarConfig);
}
else {
throw new RuntimeException("Unsupported JMS driver type : " + jmsProviderType);
}
PulsarConnectionFactory factory;
factory = new PulsarConnectionFactory(jmsConnInfo.getJmsConnConfig());
this.jmsContext = factory.createContext();
bindTimer = ActivityMetrics.timer(activityDef, "bind", this.getHdrDigits());
executeTimer = ActivityMetrics.timer(activityDef, "execute", this.getHdrDigits());
bytesCounter = ActivityMetrics.counter(activityDef, "bytes");
messagesizeHistogram = ActivityMetrics.histogram(activityDef, "messagesize", this.getHdrDigits());
if (StringUtils.equalsIgnoreCase(jmsProviderType, JmsUtil.JMS_PROVIDER_TYPES.PULSAR.label )) {
this.sequence = createOpSequence((ot) -> new ReadyPulsarJmsOp(ot, this), false, Optional.empty());
}
setDefaultsFromOpSequence(sequence);
onActivityDefUpdate(activityDef);
this.errorhandler = new NBErrorHandler(
() -> activityDef.getParams().getOptionalString("errors").orElse("stop"),
this::getExceptionMetrics
);
}
private static String buildCacheKey(String... keyParts) {
return String.join("::", keyParts);
}
/**
* If the JMS destination that corresponds to a topic exists, reuse it; Otherwise, create it
*/
public Destination getOrCreateJmsDestination(String jmsDestinationType, String destName) {
String destinationCacheKey = buildCacheKey(jmsDestinationType, destName);
Destination destination = jmsDestinations.get(destinationCacheKey);
if ( destination == null ) {
// TODO: should we match Persistent/Non-peristent JMS Delivery mode with
// Pulsar Persistent/Non-prsistent topic?
if (StringUtils.equalsIgnoreCase(jmsDestinationType, JmsUtil.JMS_DESTINATION_TYPES.QUEUE.label)) {
destination = jmsContext.createQueue(destName);
} else if (StringUtils.equalsIgnoreCase(jmsDestinationType, JmsUtil.JMS_DESTINATION_TYPES.TOPIC.label)) {
destination = jmsContext.createTopic(destName);
}
jmsDestinations.put(destinationCacheKey, destination);
}
return destination;
}
@Override
public synchronized void onActivityDefUpdate(ActivityDef activityDef) { super.onActivityDefUpdate(activityDef); }
public OpSequence<OpDispenser<? extends JmsOp>> getSequencer() { return sequence; }
public String getJmsProviderType() { return jmsProviderType; }
public JmsConnInfo getJmsConnInfo() { return jmsConnInfo; }
public JMSContext getJmsContext() { return jmsContext; }
public Timer getBindTimer() { return bindTimer; }
public Timer getExecuteTimer() { return this.executeTimer; }
public Counter getBytesCounter() { return bytesCounter; }
public Histogram getMessagesizeHistogram() { return messagesizeHistogram; }
public NBErrorHandler getErrorhandler() { return errorhandler; }
public void failOnAsyncOperationFailure() {
if (asyncOperationFailure != null) {
throw new RuntimeException(asyncOperationFailure);
}
}
public void asyncOperationFailed(Throwable ex) {
this.asyncOperationFailure = ex;
}
}

48
driver-jms/src/main/java/io/nosqlbench/driver/jms/JmsActivityType.java
View File

@@ -1,48 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms;
import io.nosqlbench.engine.api.activityapi.core.Action;
import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.annotations.Service;
@Service(value = ActivityType.class, selector = "jms")
public class JmsActivityType implements ActivityType<JmsActivity> {
@Override
public ActionDispenser getActionDispenser(JmsActivity activity) {
return new PulsarJmsActionDispenser(activity);
}
@Override
public JmsActivity getActivity(ActivityDef activityDef) {
return new JmsActivity(activityDef);
}
private static class PulsarJmsActionDispenser implements ActionDispenser {
private final JmsActivity activity;
public PulsarJmsActionDispenser(JmsActivity activity) {
this.activity = activity;
}
@Override
public Action getAction(int slot) {
return new JmsAction(activity, slot);
}
}
}

78
driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyJmsOp.java
View File

@@ -1,78 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms;
import io.nosqlbench.driver.jms.ops.JmsOp;
import io.nosqlbench.driver.jms.util.JmsUtil;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import org.apache.commons.lang3.BooleanUtils;
import java.util.function.LongFunction;
abstract public class ReadyJmsOp extends BaseOpDispenser<JmsOp> {
protected final OpTemplate optpl;
protected final CommandTemplate cmdTpl;
protected final JmsActivity jmsActivity;
protected final String stmtOpType;
protected LongFunction<Boolean> asyncApiFunc;
protected LongFunction<String> jmsDestinationTypeFunc;
protected final LongFunction<JmsOp> opFunc;
public ReadyJmsOp(OpTemplate opTemplate, JmsActivity jmsActivity) {
super(opTemplate);
this.optpl = opTemplate;
this.cmdTpl = new CommandTemplate(optpl);
this.jmsActivity = jmsActivity;
if (!cmdTpl.containsKey("optype") || !cmdTpl.isStatic("optype")) {
throw new RuntimeException("Statement parameter \"optype\" must be static and have a valid value!");
}
this.stmtOpType = cmdTpl.getStatic("optype");
// Global/Doc-level parameter: async_api
if (cmdTpl.containsKey(JmsUtil.ASYNC_API_KEY_STR)) {
if (cmdTpl.isStatic(JmsUtil.ASYNC_API_KEY_STR)) {
boolean value = BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.ASYNC_API_KEY_STR));
this.asyncApiFunc = (l) -> value;
} else {
throw new RuntimeException("\"" + JmsUtil.ASYNC_API_KEY_STR + "\" parameter cannot be dynamic!");
}
}
// Global/Doc-level parameter: jms_desitation_type
// - queue: point-to-point
// - topic: pub/sub
if (cmdTpl.containsKey(JmsUtil.JMS_DESTINATION_TYPE_KEY_STR)) {
if (cmdTpl.isStatic(JmsUtil.JMS_DESTINATION_TYPE_KEY_STR)) {
jmsDestinationTypeFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_DESTINATION_TYPE_KEY_STR);
} else {
throw new RuntimeException("\"" + JmsUtil.JMS_DESTINATION_TYPE_KEY_STR + "\" parameter cannot be dynamic!");
}
}
this.opFunc = resolveJms();
}
public JmsOp apply(long value) { return opFunc.apply(value); }
abstract LongFunction<JmsOp> resolveJms();
}

264
driver-jms/src/main/java/io/nosqlbench/driver/jms/ReadyPulsarJmsOp.java
View File

@@ -1,264 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms;
import io.nosqlbench.driver.jms.ops.JmsMsgReadMapper;
import io.nosqlbench.driver.jms.ops.JmsMsgSendMapper;
import io.nosqlbench.driver.jms.ops.JmsOp;
import io.nosqlbench.driver.jms.util.JmsHeaderLongFunc;
import io.nosqlbench.driver.jms.util.JmsUtil;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import javax.jms.DeliveryMode;
import javax.jms.Destination;
import javax.jms.JMSRuntimeException;
import javax.jms.Message;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.function.LongFunction;
import java.util.stream.Collectors;
public class ReadyPulsarJmsOp extends ReadyJmsOp {
public ReadyPulsarJmsOp(OpTemplate opTemplate, JmsActivity jmsActivity) {
super(opTemplate, jmsActivity);
}
public LongFunction<JmsOp> resolveJms() {
// Global/Doc-level parameter: topic_uri
LongFunction<String> topicUriFunc = (l) -> null;
if (cmdTpl.containsKey(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR)) {
if (cmdTpl.isStatic(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR)) {
topicUriFunc = (l) -> cmdTpl.getStatic(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR);
} else {
topicUriFunc = (l) -> cmdTpl.getDynamic(JmsUtil.PULSAR_JMS_TOPIC_URI_KEY_STR, l);
}
}
// Global: JMS destination
LongFunction<Destination> jmsDestinationFunc;
try {
LongFunction<String> finalTopicUriFunc = topicUriFunc;
jmsDestinationFunc = (l) -> jmsActivity.getOrCreateJmsDestination(
jmsDestinationTypeFunc.apply(l),
finalTopicUriFunc.apply(l));
}
catch (JMSRuntimeException ex) {
throw new RuntimeException("Unable to create JMS destination!");
}
if (StringUtils.equalsIgnoreCase(stmtOpType, JmsUtil.OP_TYPES.MSG_SEND.label)) {
return resolveMsgSend(asyncApiFunc, jmsDestinationFunc);
} else if (StringUtils.equalsIgnoreCase(stmtOpType, JmsUtil.OP_TYPES.MSG_READ.label)) {
return resolveMsgRead(asyncApiFunc, jmsDestinationFunc);
} else {
throw new RuntimeException("Unsupported JMS operation type");
}
}
private LongFunction<JmsOp> resolveMsgSend(
LongFunction<Boolean> async_api_func,
LongFunction<Destination> jmsDestinationFunc
) {
JmsHeaderLongFunc jmsHeaderLongFunc = new JmsHeaderLongFunc();
// JMS header: delivery mode
LongFunction<Integer> msgDeliveryModeFunc = (l) -> DeliveryMode.PERSISTENT;
if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label)) {
if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label)) {
msgDeliveryModeFunc = (l) -> NumberUtils.toInt(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label));
}
else {
msgDeliveryModeFunc = (l) -> NumberUtils.toInt(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_MODE.label, l));
}
}
jmsHeaderLongFunc.setDeliveryModeFunc(msgDeliveryModeFunc);
// JMS header: message priority
LongFunction<Integer> msgPriorityFunc = (l) -> Message.DEFAULT_PRIORITY;
if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label)) {
if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label)) {
msgPriorityFunc = (l) -> NumberUtils.toInt(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label));
}
else {
msgPriorityFunc = (l) -> NumberUtils.toInt(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.PRIORITY.label, l));
}
}
jmsHeaderLongFunc.setMsgPriorityFunc(msgPriorityFunc);
// JMS header: message TTL
LongFunction<Long> msgTtlFunc = (l) -> Message.DEFAULT_TIME_TO_LIVE;
if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label)) {
if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label)) {
msgTtlFunc = (l) -> NumberUtils.toLong(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label));
}
else {
msgTtlFunc = (l) -> NumberUtils.toLong(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.TTL.label, l));
}
}
jmsHeaderLongFunc.setMsgTtlFunc(msgTtlFunc);
// JMS header: message delivery delay
LongFunction<Long> msgDeliveryDelayFunc = (l) -> Message.DEFAULT_DELIVERY_DELAY;
if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label)) {
if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label)) {
msgDeliveryDelayFunc = (l) -> NumberUtils.toLong(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label));
}
else {
msgDeliveryDelayFunc = (l) -> NumberUtils.toLong(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DELIVERY_DELAY.label, l));
}
}
jmsHeaderLongFunc.setMsgDeliveryDelayFunc(msgDeliveryDelayFunc);
// JMS header: disable message timestamp
LongFunction<Boolean> disableMsgTimestampFunc = (l) -> false;
if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label)) {
if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label)) {
disableMsgTimestampFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label));
}
else {
disableMsgTimestampFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_TIMESTAMP.label, l));
}
}
jmsHeaderLongFunc.setDisableMsgTimestampFunc(disableMsgTimestampFunc);
// JMS header: disable message ID
LongFunction<Boolean> disableMsgIdFunc = (l) -> false;
if (cmdTpl.containsKey(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label)) {
if (cmdTpl.isStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label)) {
disableMsgIdFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label));
}
else {
disableMsgIdFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_MSG_HEADER_KEYS.DISABLE_ID.label, l));
}
}
jmsHeaderLongFunc.setDisableMsgIdFunc(disableMsgIdFunc);
// JMS message properties
String jmsMsgPropertyListStr = "";
if (cmdTpl.containsKey(JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR)) {
if (cmdTpl.isStatic(JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR)) {
jmsMsgPropertyListStr = cmdTpl.getStatic(JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR);
} else {
throw new RuntimeException("\"" + JmsUtil.JMS_PRODUCER_MSG_PROPERTY_KEY_STR + "\" parameter cannot be dynamic!");
}
}
Map<String, Object> jmsMsgProperties = new HashMap<>();
if ( !StringUtils.isEmpty(jmsMsgPropertyListStr) ) {
jmsMsgProperties = Arrays.stream(jmsMsgPropertyListStr.split(";"))
.map(s -> s.split("=", 2))
.collect(Collectors.toMap(a -> a[0], a -> a.length > 1 ? a[1] : ""));
}
LongFunction<String> msgBodyFunc;
if (cmdTpl.containsKey(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR)) {
if (cmdTpl.isStatic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR)) {
msgBodyFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR);
} else if (cmdTpl.isDynamic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR)) {
msgBodyFunc = (l) -> cmdTpl.getDynamic(JmsUtil.JMS_PRODUCER_MSG_BODY_KEY_STR, l);
} else {
msgBodyFunc = (l) -> null;
}
} else {
throw new RuntimeException("JMS message send:: \"msg_body\" field must be specified!");
}
return new JmsMsgSendMapper(
jmsActivity,
async_api_func,
jmsDestinationFunc,
jmsHeaderLongFunc,
jmsMsgProperties,
msgBodyFunc);
}
private LongFunction<JmsOp> resolveMsgRead(
LongFunction<Boolean> async_api_func,
LongFunction<Destination> jmsDestinationFunc
) {
// For Pulsar JMS, make "durable" as the default
LongFunction<Boolean> jmsConsumerDurableFunc = (l) -> true;
if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR)) {
if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR)) {
jmsConsumerDurableFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR));
} else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR)) {
jmsConsumerDurableFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_DURABLE_KEY_STR, l));
}
}
LongFunction<Boolean> jmsConsumerSharedFunc = (l) -> true;
if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR)) {
if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR)) {
jmsConsumerSharedFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR));
} else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR)) {
jmsConsumerSharedFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_SHARED_KEY_STR, l));
}
}
LongFunction<String> jmsMsgSubscriptionFunc = (l) -> "";
if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR)) {
if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR)) {
jmsMsgSubscriptionFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR);
} else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR)) {
jmsMsgSubscriptionFunc = (l) -> cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR, l);
}
}
LongFunction<String> jmsMsgReadSelectorFunc = (l) -> "";
if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR)) {
if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR)) {
jmsMsgReadSelectorFunc = (l) -> cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR);
} else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR)) {
jmsMsgReadSelectorFunc = (l) -> cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR, l);
}
}
LongFunction<Boolean> jmsMsgNoLocalFunc = (l) -> true;
if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR)) {
if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR)) {
jmsMsgNoLocalFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR));
} else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR)) {
jmsMsgNoLocalFunc = (l) -> BooleanUtils.toBoolean(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_MSG_NOLOCAL_KEY_STR, l));
}
}
LongFunction<Long> jmsReadTimeoutFunc = (l) -> 0L;
if (cmdTpl.containsKey(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR)) {
if (cmdTpl.isStatic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR)) {
jmsReadTimeoutFunc = (l) -> NumberUtils.toLong(cmdTpl.getStatic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR));
} else if (cmdTpl.isDynamic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR)) {
jmsReadTimeoutFunc = (l) -> NumberUtils.toLong(cmdTpl.getDynamic(JmsUtil.JMS_CONSUMER_READ_TIMEOUT_KEY_STR, l));
}
}
return new JmsMsgReadMapper(
jmsActivity,
async_api_func,
jmsDestinationFunc,
jmsConsumerDurableFunc,
jmsConsumerSharedFunc,
jmsMsgSubscriptionFunc,
jmsMsgReadSelectorFunc,
jmsMsgNoLocalFunc,
jmsReadTimeoutFunc);
}
}

37
driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsConnInfo.java
View File

@@ -1,37 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms.conn;
import java.util.HashMap;
import java.util.Map;
public class JmsConnInfo {
protected final String jmsProviderType;
protected final Map<String, Object> jmsConnConfig;
protected JmsConnInfo(String jmsProviderType) {
this.jmsProviderType = jmsProviderType;
this.jmsConnConfig = new HashMap<>();
}
public Map<String, Object> getJmsConnConfig() { return this.jmsConnConfig; }
public void resetJmsConnConfig() { this.jmsConnConfig.clear(); }
public void addJmsConnConfigItems(Map<String, Object> cfgItems) { this.jmsConnConfig.putAll(cfgItems); }
public void addJmsConnConfigItem(String key, Object value) { this.jmsConnConfig.put(key, value); }
public void removeJmsConnConfigItem(String key) { this.jmsConnConfig.remove(key); }
}

58
driver-jms/src/main/java/io/nosqlbench/driver/jms/conn/JmsPulsarConnInfo.java
View File

@@ -1,58 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms.conn;
import io.nosqlbench.driver.jms.util.PulsarConfig;
import java.util.Map;
public class JmsPulsarConnInfo extends JmsConnInfo {
private final String webSvcUrl;
private final String pulsarSvcUrl;
private final PulsarConfig extraPulsarConfig;
public JmsPulsarConnInfo(String jmsProviderType, String webSvcUrl, String pulsarSvcUrl, PulsarConfig pulsarConfig) {
super(jmsProviderType);
this.webSvcUrl = webSvcUrl;
this.pulsarSvcUrl = pulsarSvcUrl;
this.extraPulsarConfig = pulsarConfig;
this.addJmsConnConfigItem("webServiceUrl", this.webSvcUrl);
this.addJmsConnConfigItem("brokerServiceUrl", this.pulsarSvcUrl);
Map<String, Object> clientCfgMap = this.extraPulsarConfig.getClientConfMap();
if (!clientCfgMap.isEmpty()) {
this.addJmsConnConfigItems(clientCfgMap);
}
Map<String, Object> producerCfgMap = this.extraPulsarConfig.getProducerConfMap();
if (!producerCfgMap.isEmpty()) {
this.addJmsConnConfigItem("producerConfig", producerCfgMap);
}
Map<String, Object> consumerCfgMap = this.extraPulsarConfig.getConsumerConfMap();
if (!consumerCfgMap.isEmpty()) {
this.addJmsConnConfigItem("consumerConfig", consumerCfgMap);
}
}
public String getWebSvcUrl() { return this.webSvcUrl; }
public String getPulsarSvcUrl() { return this.pulsarSvcUrl; }
public PulsarConfig getExtraPulsarConfig() { return this.extraPulsarConfig; }
}

88
driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadMapper.java
View File

@@ -1,88 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms.ops;
import io.nosqlbench.driver.jms.JmsActivity;
import javax.jms.Destination;
import java.util.function.LongFunction;
/**
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
* retried if needed.
*
* This function doesn't act *as* the operation. It merely maps the construction logic into
* a simple functional type, given the component functions.
*
* For additional parameterization, the command template is also provided.
*/
public class JmsMsgReadMapper extends JmsOpMapper {
private final LongFunction<Boolean> jmsConsumerDurableFunc;
private final LongFunction<Boolean> jmsConsumerSharedFunc;
private final LongFunction<String> jmsMsgSubscriptionFunc;
private final LongFunction<String> jmsMsgReadSelectorFunc;
private final LongFunction<Boolean> jmsMsgNoLocalFunc;
private final LongFunction<Long> jmsReadTimeoutFunc;
public JmsMsgReadMapper(JmsActivity jmsActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Destination> jmsDestinationFunc,
LongFunction<Boolean> jmsConsumerDurableFunc,
LongFunction<Boolean> jmsConsumerSharedFunc,
LongFunction<String> jmsMsgSubscriptionFunc,
LongFunction<String> jmsMsgReadSelectorFunc,
LongFunction<Boolean> jmsMsgNoLocalFunc,
LongFunction<Long> jmsReadTimeoutFunc) {
super(jmsActivity, asyncApiFunc, jmsDestinationFunc);
this.jmsConsumerDurableFunc = jmsConsumerDurableFunc;
this.jmsConsumerSharedFunc = jmsConsumerSharedFunc;
this.jmsMsgSubscriptionFunc = jmsMsgSubscriptionFunc;
this.jmsMsgReadSelectorFunc = jmsMsgReadSelectorFunc;
this.jmsMsgNoLocalFunc = jmsMsgNoLocalFunc;
this.jmsReadTimeoutFunc = jmsReadTimeoutFunc;
}
@Override
public JmsOp apply(long value) {
boolean asyncApi = asyncApiFunc.apply(value);
Destination jmsDestination = jmsDestinationFunc.apply(value);
boolean jmsConsumerDurable = jmsConsumerDurableFunc.apply(value);
boolean jmsConsumerShared = jmsConsumerSharedFunc.apply(value);
String jmsMsgSubscription = jmsMsgSubscriptionFunc.apply(value);
String jmsMsgReadSelector = jmsMsgReadSelectorFunc.apply(value);
boolean jmsMsgNoLocal = jmsMsgNoLocalFunc.apply(value);
long jmsReadTimeout = jmsReadTimeoutFunc.apply(value);
// Default to NO read timeout
if (jmsReadTimeout < 0) jmsReadTimeout = 0;
return new JmsMsgReadOp(
jmsActivity,
asyncApi,
jmsDestination,
jmsConsumerDurable,
jmsConsumerShared,
jmsMsgSubscription,
jmsMsgReadSelector,
jmsMsgNoLocal,
jmsReadTimeout
);
}
}

130
driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgReadOp.java
View File

@@ -1,130 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms.ops;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Histogram;
import io.nosqlbench.driver.jms.JmsActivity;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import javax.jms.*;
public class JmsMsgReadOp extends JmsTimeTrackOp {
private final static Logger logger = LogManager.getLogger(JmsMsgReadOp.class);
private final JmsActivity jmsActivity;
private final boolean asyncJmsOp;
private final Destination jmsDestination;
private final JMSContext jmsContext;
private final JMSConsumer jmsConsumer;
private final boolean jmsConsumerDurable;
private final boolean jmsConsumerShared;
private final String jmsMsgSubscrption;
private final String jmsMsgReadSelector;
private final boolean jmsMsgNoLocal;
private final long jmsReadTimeout;
private final Counter bytesCounter;
private final Histogram messagesizeHistogram;
public JmsMsgReadOp(JmsActivity jmsActivity,
boolean asyncJmsOp,
Destination jmsDestination,
boolean jmsConsumerDurable,
boolean jmsConsumerShared,
String jmsMsgSubscrption,
String jmsMsgReadSelector,
boolean jmsMsgNoLocal,
long jmsReadTimeout) {
this.jmsActivity = jmsActivity;
this.asyncJmsOp = asyncJmsOp;
this.jmsDestination = jmsDestination;
this.jmsConsumerDurable = jmsConsumerDurable;
this.jmsConsumerShared = jmsConsumerShared;
this.jmsMsgReadSelector = jmsMsgReadSelector;
this.jmsMsgSubscrption = jmsMsgSubscrption;
this.jmsMsgNoLocal = jmsMsgNoLocal;
this.jmsReadTimeout = jmsReadTimeout;
this.jmsContext = jmsActivity.getJmsContext();
this.jmsConsumer = createJmsConsumer();
this.bytesCounter = jmsActivity.getBytesCounter();
this.messagesizeHistogram = jmsActivity.getMessagesizeHistogram();
}
private JMSConsumer createJmsConsumer() {
JMSConsumer jmsConsumer;
try {
if (jmsConsumerDurable) {
if (jmsConsumerShared)
jmsConsumer = jmsContext.createSharedDurableConsumer((Topic) jmsDestination, jmsMsgSubscrption, jmsMsgReadSelector);
else
jmsConsumer = jmsContext.createDurableConsumer((Topic) jmsDestination, jmsMsgSubscrption, jmsMsgReadSelector, jmsMsgNoLocal);
} else {
if (jmsConsumerShared)
jmsConsumer = jmsContext.createSharedConsumer((Topic) jmsDestination, jmsMsgSubscrption, jmsMsgReadSelector);
else
jmsConsumer = jmsContext.createConsumer(jmsDestination, jmsMsgReadSelector, jmsMsgNoLocal);
}
}
catch (InvalidDestinationRuntimeException invalidDestinationRuntimeException) {
throw new RuntimeException("Failed to create JMS consumer: invalid destination!");
}
catch (InvalidSelectorRuntimeException invalidSelectorRuntimeException) {
throw new RuntimeException("Failed to create JMS consumer: invalid message selector!");
}
catch (JMSRuntimeException jmsRuntimeException) {
jmsRuntimeException.printStackTrace();
throw new RuntimeException("Failed to create JMS consumer: runtime internal error!");
}
// TODO: async consumer
// if (this.asyncJmsOp) {
// jmsConsumer.setMessageListener();
// }
return jmsConsumer;
}
@Override
public void run() {
// FIXME: jmsReadTimeout being 0 behaves like receiveNoWait() instead of waiting indefinitley
Message receivedMsg = jmsConsumer.receive(jmsReadTimeout);
try {
if (receivedMsg != null) {
receivedMsg.acknowledge();
byte[] receivedMsgBody = receivedMsg.getBody(byte[].class);
if (logger.isDebugEnabled()) {
logger.debug("received msg-payload={}", new String(receivedMsgBody));
}
int messagesize = receivedMsgBody.length;
bytesCounter.inc(messagesize);
messagesizeHistogram.update(messagesize);
}
} catch (JMSException e) {
e.printStackTrace();
throw new RuntimeException("Failed to acknowledge the received JMS message.");
}
}
}

71
driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendMapper.java
View File

@@ -1,71 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms.ops;
import io.nosqlbench.driver.jms.JmsActivity;
import io.nosqlbench.driver.jms.util.JmsHeader;
import io.nosqlbench.driver.jms.util.JmsHeaderLongFunc;
import javax.jms.Destination;
import java.util.Map;
import java.util.function.LongFunction;
/**
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
* retried if needed.
*
* This function doesn't act *as* the operation. It merely maps the construction logic into
* a simple functional type, given the component functions.
*
* For additional parameterization, the command template is also provided.
*/
public class JmsMsgSendMapper extends JmsOpMapper {
private final JmsHeaderLongFunc jmsHeaderLongFunc;
private final Map<String, Object> jmsMsgProperties;
private final LongFunction<String> msgBodyFunc;
public JmsMsgSendMapper(JmsActivity jmsActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Destination> jmsDestinationFunc,
JmsHeaderLongFunc jmsHeaderLongFunc,
Map<String, Object> jmsMsgProperties,
LongFunction<String> msgBodyFunc) {
super(jmsActivity, asyncApiFunc, jmsDestinationFunc);
this.jmsHeaderLongFunc = jmsHeaderLongFunc;
this.jmsMsgProperties = jmsMsgProperties;
this.msgBodyFunc = msgBodyFunc;
}
@Override
public JmsOp apply(long value) {
boolean asyncApi = asyncApiFunc.apply(value);
Destination jmsDestination = jmsDestinationFunc.apply(value);
JmsHeader jmsHeader = (JmsHeader)jmsHeaderLongFunc.apply(value);
String msgBody = msgBodyFunc.apply(value);
return new JmsMsgSendOp(
jmsActivity,
asyncApi,
jmsDestination,
jmsHeader,
jmsMsgProperties,
msgBody
);
}
}

139
driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsMsgSendOp.java
View File

@@ -1,139 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms.ops;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Histogram;
import io.nosqlbench.driver.jms.JmsActivity;
import io.nosqlbench.driver.jms.util.JmsHeader;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import javax.jms.*;
import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.Map;
public class JmsMsgSendOp extends JmsTimeTrackOp {
private final static Logger logger = LogManager.getLogger(JmsMsgSendOp.class);
private final JmsActivity jmsActivity;
private final boolean asyncJmsOp;
private final Destination jmsDestination;
private final JmsHeader jmsHeader;
private final Map<String, Object> jmsMsgProperties;
private final JMSContext jmsContext;
private final JMSProducer jmsProducer;
private final String msgBody;
private final Counter bytesCounter;
private final Histogram messagesizeHistogram;
public JmsMsgSendOp(JmsActivity jmsActivity,
boolean asyncJmsOp,
Destination jmsDestination,
JmsHeader jmsHeader,
Map<String, Object> jmsMsgProperties,
String msgBody) {
this.jmsActivity = jmsActivity;
this.asyncJmsOp = asyncJmsOp;
this.jmsDestination = jmsDestination;
this.jmsHeader = jmsHeader;
this.jmsMsgProperties = jmsMsgProperties;
this.msgBody = msgBody;
if (!jmsHeader.isValidHeader()) {
throw new RuntimeException(jmsHeader.getInvalidJmsHeaderMsgText());
}
if ((msgBody == null) || msgBody.isEmpty()) {
throw new RuntimeException("JMS message body can't be empty!");
}
this.jmsContext = jmsActivity.getJmsContext();
this.jmsProducer = createJmsProducer();
this.bytesCounter = jmsActivity.getBytesCounter();
this.messagesizeHistogram = jmsActivity.getMessagesizeHistogram();
}
private JMSProducer createJmsProducer() {
JMSProducer jmsProducer = this.jmsContext.createProducer();
jmsProducer.setDeliveryMode(this.jmsHeader.getDeliveryMode());
jmsProducer.setPriority(this.jmsHeader.getMsgPriority());
jmsProducer.setDeliveryDelay(this.jmsHeader.getMsgDeliveryDelay());
jmsProducer.setDisableMessageTimestamp(this.jmsHeader.isDisableMsgTimestamp());
jmsProducer.setDisableMessageID(this.jmsHeader.isDisableMsgId());
if (this.asyncJmsOp) {
jmsProducer.setAsync(new CompletionListener() {
@Override
public void onCompletion(Message msg) {
try {
byte[] msgBody = msg.getBody(byte[].class);
if (logger.isTraceEnabled()) {
logger.trace("Async message send success - message body: " + new String(msgBody));
}
}
catch (JMSException jmsException) {
jmsException.printStackTrace();
logger.warn("Unexpected error when parsing message body: " + jmsException.getMessage());
}
}
@Override
public void onException(Message msg, Exception e) {
try {
byte[] msgBody = msg.getBody(byte[].class);
if (logger.isTraceEnabled()) {
logger.trace("Async message send failure - message body: " + new String(msgBody));
}
}
catch (JMSException jmsException) {
jmsException.printStackTrace();
logger.warn("Unexpected error when parsing message body: " + jmsException.getMessage());
}
}
});
}
for (Map.Entry<String, Object> entry : jmsMsgProperties.entrySet()) {
jmsProducer.setProperty(entry.getKey(), entry.getValue());
}
return jmsProducer;
}
@Override
public void run() {
try {
byte[] msgBytes = msgBody.getBytes(StandardCharsets.UTF_8);
int messageSize = msgBytes.length;
jmsProducer.send(jmsDestination, msgBytes);
messagesizeHistogram.update(messageSize);
bytesCounter.inc(messageSize);
}
catch (Exception ex) {
logger.error("Failed to send JMS message - " + msgBody);
}
}
}

39
driver-jms/src/main/java/io/nosqlbench/driver/jms/ops/JmsOpMapper.java
View File

@@ -1,39 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms.ops;
import io.nosqlbench.driver.jms.JmsActivity;
import io.nosqlbench.driver.jms.util.JmsHeaderLongFunc;
import javax.jms.Destination;
import java.util.Map;
import java.util.function.LongFunction;
public abstract class JmsOpMapper implements LongFunction<JmsOp> {
protected final JmsActivity jmsActivity;
protected final LongFunction<Boolean> asyncApiFunc;
protected final LongFunction<Destination> jmsDestinationFunc;
public JmsOpMapper(JmsActivity jmsActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Destination> jmsDestinationFunc)
{
this.jmsActivity = jmsActivity;
this.asyncApiFunc = asyncApiFunc;
this.jmsDestinationFunc = jmsDestinationFunc;
}
}

82
driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeader.java
View File

@@ -1,82 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms.util;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.apache.commons.lang.StringUtils;
import javax.jms.DeliveryMode;
@Setter
@Getter
@AllArgsConstructor
@ToString
public class JmsHeader {
private int deliveryMode;
private int msgPriority;
private long msgTtl;
private long msgDeliveryDelay;
private boolean disableMsgTimestamp;
private boolean disableMsgId;
public boolean isValidDeliveryMode() {
return (deliveryMode == DeliveryMode.NON_PERSISTENT) || (deliveryMode == DeliveryMode.PERSISTENT);
}
public boolean isValidPriority() {
return (msgPriority >= 0) && (msgPriority <= 9);
}
public boolean isValidTtl() {
return msgTtl >= 0;
}
public boolean isValidDeliveryDelay() {
return msgTtl >= 0;
}
public boolean isValidHeader() {
return isValidDeliveryMode()
&& isValidPriority()
&& isValidTtl()
&& isValidDeliveryDelay();
}
public String getInvalidJmsHeaderMsgText() {
StringBuilder sb = new StringBuilder();
if (!isValidDeliveryMode())
sb.append("delivery mode - " + deliveryMode + "; ");
if (!isValidPriority())
sb.append("message priority - " + msgPriority + "; ");
if (!isValidTtl())
sb.append("message TTL - " + msgTtl + "; ");
if (!isValidDeliveryDelay())
sb.append("message delivery delay - " + msgDeliveryDelay + "; ");
String invalidMsgText = sb.toString();
if (StringUtils.length(invalidMsgText) > 0)
invalidMsgText = StringUtils.substringBeforeLast(invalidMsgText, ";");
else
invalidMsgText = "none";
return "Invalid JMS header values: " + invalidMsgText;
}
}

47
driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsHeaderLongFunc.java
View File

@@ -1,47 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms.util;
import lombok.*;
import javax.jms.DeliveryMode;
import javax.jms.Message;
import java.util.function.LongFunction;
@Setter
@Getter
@NoArgsConstructor
public class JmsHeaderLongFunc implements LongFunction {
private LongFunction<Integer> deliveryModeFunc;
private LongFunction<Integer> msgPriorityFunc;
private LongFunction<Long> msgTtlFunc;
private LongFunction<Long> msgDeliveryDelayFunc;
private LongFunction<Boolean> disableMsgTimestampFunc;
private LongFunction<Boolean> disableMsgIdFunc;
@Override
public Object apply(long value) {
return new JmsHeader(
(deliveryModeFunc != null) ? deliveryModeFunc.apply(value) : DeliveryMode.PERSISTENT,
(msgPriorityFunc != null) ? msgPriorityFunc.apply(value) : Message.DEFAULT_PRIORITY,
(msgTtlFunc != null) ? msgTtlFunc.apply(value) : Message.DEFAULT_TIME_TO_LIVE,
(msgTtlFunc != null) ? msgTtlFunc.apply(value) : Message.DEFAULT_DELIVERY_DELAY,
(disableMsgTimestampFunc != null) ? disableMsgTimestampFunc.apply(value) : false,
(disableMsgIdFunc != null) ? disableMsgIdFunc.apply(value) : false
);
}
}

120
driver-jms/src/main/java/io/nosqlbench/driver/jms/util/JmsUtil.java
View File

@@ -1,120 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms.util;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Arrays;
public class JmsUtil {
private final static Logger logger = LogManager.getLogger(JmsUtil.class);
// Supported JMS provider type
public enum JMS_PROVIDER_TYPES {
PULSAR("pulsar");
public final String label;
JMS_PROVIDER_TYPES(String label) {
this.label = label;
}
}
public static boolean isValidJmsProviderType(String type) {
return Arrays.stream(JMS_PROVIDER_TYPES.values()).anyMatch(t -> t.label.equals(type));
}
/////
// NB command line parameters
// - JMS provider type
public final static String JMS_PROVIDER_TYPE_KEY_STR = "provider_type";
/// Only applicable when the provider is "Pulsar"
// - Pulsar configuration properties file
public final static String JMS_PULSAR_PROVIDER_CFG_FILE_KEY_STR = "pulsar_cfg_file";
public final static String JMS_PULSAR_PROVIDER_DFT_CFG_FILE_NAME = "pulsar_config.properties";
// - Pulsar web url
public final static String JMS_PULSAR_PROVIDER_WEB_URL_KEY_STR = "web_url";
// - Pulsar service url
public final static String JMS_PULSAR_PROVIDER_SVC_URL_KEY_STR = "service_url";
public final static String ASYNC_API_KEY_STR = "async_api";
public final static String JMS_DESTINATION_TYPE_KEY_STR = "jms_desitation_type";
///// JMS Producer
// Supported JMS provider type
public enum JMS_MSG_HEADER_KEYS {
DELIVERY_MODE("jms_producer_header_msg_delivery_mode"),
PRIORITY("jms_producer_header_msg_priority"),
TTL("jms_producer_header_msg_ttl"),
DELIVERY_DELAY("jms_producer_header_msg_delivery_delay"),
DISABLE_TIMESTAMP("jms_producer_header_disable_msg_timestamp"),
DISABLE_ID("jms_producer_header_disable_msg_id");
public final String label;
JMS_MSG_HEADER_KEYS(String label) {
this.label = label;
}
}
public static boolean isValidJmsHeaderKey(String type) {
return Arrays.stream(JMS_MSG_HEADER_KEYS.values()).anyMatch(t -> t.label.equals(type));
}
public final static String JMS_PRODUCER_MSG_PROPERTY_KEY_STR = "jms_producer_msg_properties";
public final static String JMS_PRODUCER_MSG_BODY_KEY_STR = "msg_body";
///// JMS Consumer
public final static String JMS_CONSUMER_DURABLE_KEY_STR = "jms_consumer_msg_durable";
public final static String JMS_CONSUMER_SHARED_KEY_STR = "jms_consumer_msg_shared";
public final static String JMS_CONSUMER_MSG_SUBSCRIPTIOn_KEY_STR = "jms_consumer_subscription";
public final static String JMS_CONSUMER_MSG_READ_SELECTOR_KEY_STR = "jms_consumer_msg_read_selector";
public final static String JMS_CONSUMER_MSG_NOLOCAL_KEY_STR = "jms_consumer_msg_nolocal";
public final static String JMS_CONSUMER_READ_TIMEOUT_KEY_STR = "jms_consumer_msg_read_timeout";
// Only applicable to Pulsar JMS provider
public final static String PULSAR_JMS_TOPIC_URI_KEY_STR = "pulsar_topic_uri";
// Supported message operation types
public enum OP_TYPES {
MSG_SEND("msg_send"),
MSG_READ("msg_read");
public final String label;
OP_TYPES(String label) {
this.label = label;
}
}
public static boolean isValidClientType(String type) {
return Arrays.stream(OP_TYPES.values()).anyMatch(t -> t.label.equals(type));
}
// JMS Destination Types
public enum JMS_DESTINATION_TYPES {
QUEUE("queue"),
TOPIC("topic");
public final String label;
JMS_DESTINATION_TYPES(String label) {
this.label = label;
}
}
public static boolean isValidJmsDestinationType(String type) {
return Arrays.stream(JMS_DESTINATION_TYPES.values()).anyMatch(t -> t.label.equals(type));
}
}

115
driver-jms/src/main/java/io/nosqlbench/driver/jms/util/PulsarConfig.java
View File

@@ -1,115 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.jms.util;
import org.apache.commons.configuration2.Configuration;
import org.apache.commons.configuration2.FileBasedConfiguration;
import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder;
import org.apache.commons.configuration2.builder.fluent.Parameters;
import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
public class PulsarConfig {
private final static Logger logger = LogManager.getLogger(PulsarConfig.class);
public static final String SCHEMA_CONF_PREFIX = "schema";
public static final String CLIENT_CONF_PREFIX = "client";
public static final String PRODUCER_CONF_PREFIX = "producer";
public static final String CONSUMER_CONF_PREFIX = "consumer";
private final Map<String, Object> schemaConfMap = new HashMap<>();
private final Map<String, Object> clientConfMap = new HashMap<>();
private final Map<String, Object> producerConfMap = new HashMap<>();
private final Map<String, Object> consumerConfMap = new HashMap<>();
public PulsarConfig(String fileName) {
File file = new File(fileName);
try {
String canonicalFilePath = file.getCanonicalPath();
Parameters params = new Parameters();
FileBasedConfigurationBuilder<FileBasedConfiguration> builder =
new FileBasedConfigurationBuilder<FileBasedConfiguration>(PropertiesConfiguration.class)
.configure(params.properties()
.setFileName(fileName));
Configuration config = builder.getConfiguration();
// Get schema specific configuration settings
for (Iterator<String> it = config.getKeys(SCHEMA_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
schemaConfMap.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
// Get client connection specific configuration settings
for (Iterator<String> it = config.getKeys(CLIENT_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
clientConfMap.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
// Get producer specific configuration settings
for (Iterator<String> it = config.getKeys(PRODUCER_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
producerConfMap.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
// Get consumer specific configuration settings
for (Iterator<String> it = config.getKeys(CONSUMER_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
consumerConfMap.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
} catch (IOException ioe) {
logger.error("Can't read the specified config properties file: " + fileName);
ioe.printStackTrace();
} catch (ConfigurationException cex) {
logger.error("Error loading configuration items from the specified config properties file: " + fileName + ":" + cex.getMessage());
cex.printStackTrace();
}
}
public Map<String, Object> getSchemaConfMap() {
return this.schemaConfMap;
}
public Map<String, Object> getClientConfMap() {
return this.clientConfMap;
}
public Map<String, Object> getProducerConfMap() {
return this.producerConfMap;
}
public Map<String, Object> getConsumerConfMap() {
return this.consumerConfMap;
}
}

1
driver-jms/src/main/resources/jms.md
View File

@@ -1 +0,0 @@
# Overview

33
driver-jms/src/main/resources/pulsar_config.properties
View File

@@ -1,33 +0,0 @@
### Schema related configurations - schema.xxx
# valid types:
# - primitive type (https://pulsar.apache.org/docs/en/schema-understand/#primitive-type)
# - keyvalue (https://pulsar.apache.org/docs/en/schema-understand/#keyvalue)
# - strut (complex type) (https://pulsar.apache.org/docs/en/schema-understand/#struct)
# avro, json, protobuf
#
# NOTE: for JMS client, Pulsar "schema" is NOT supported yet
schema.type=
schema.definition=
### Pulsar client related configurations - client.xxx
# http://pulsar.apache.org/docs/en/client-libraries-java/#client
client.connectionTimeoutMs=5000
#client.authPluginClassName=org.apache.pulsar.client.impl.auth.AuthenticationToken
#client.authParams=
#client.tlsAllowInsecureConnection=true
client.numIoThreads=10
client.numListenerThreads=10
### Producer related configurations (global) - producer.xxx
# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer
producer.sendTimeoutMs=
producer.blockIfQueueFull=true
producer.maxPendingMessages=10000
producer.batchingMaxMessages=10000
### Consumer related configurations (global) - consumer.xxx
# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer
consumer.receiverQueueSize=2000

89
driver-jms/src/main/resources/pulsar_jms.yaml
View File

@@ -1,89 +0,0 @@
bindings:
payload: NumberNameToString() #AlphaNumericString(20)
tenant: Mod(10000); Div(10L); ToString(); Prefix("tnt")
namespace: Mod(10); Div(5L); ToString(); Prefix("ns")
core_topic_name: Mod(5); ToString(); Prefix("t")
# document level parameters that apply to all Pulsar client types:
params:
### static only
async_api: "true"
### Static only
# Valid values: queue (point-to-point) or topic (pub-sub)
jms_desitation_type: "topic"
### Static Only
# NOTE: ONLY relevant when the JMS provider is Pulsar
#pulsar_topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}"
#pulsar_topic_uri: "persistent://public/default/pt100"
#pulsar_topic_uri: "persistent://public/default/t0"
pulsar_topic_uri: "persistent://public/default/pt100_10"
#pulsar_topic_uri: "persistent://public/default/pt200_10"
#pulsar_topic_uri: "persistent://public/default/pt300_10"
#pulsar_topic_uri: "persistent://public/default/pt400_10"
blocks:
- name: "producer-block"
tags:
phase: "jms_producer"
statements:
- name: "s1"
optype: "msg_send"
### JMS PRODUCER message header
### https://docs.oracle.com/javaee/7/api/constant-values.html#javax.jms.DeliveryMode.NON_PERSISTENT
# - static or dynamic
# - Producer only
# Valid values: non-persistent(1), or persistent(2) - default
jms_producer_header_msg_delivery_mode: "2"
# Valid values: 0~9 (4 as default)
jms_producer_header_msg_priority: "4"
# Valid values: non-negative long; default 0 (never expires)
jms_producer_header_msg_ttl: "0"
# Valid values: non-negative long; default 0 (no delay)
jms_producer_header_msg_delivery_delay: "0"
# Valid values: true/false; default false (message timestamp is enabled)
jms_producer_header_disable_msg_timestamp: "false"
# Valid values: true/false; default false (message ID is enabled)
jms_producer_header_disable_msg_id: "false"
### JMS PRODUCER message properties
# - static only
# - Producer only
# - In format: "key1=value1;key2=value2;..."
jms_producer_msg_properties: "key1=value1;key2=value2"
### JMS PRODUCER message body
msg_body: "{payload}"
- name: "consumer-block"
tags:
phase: "jms_consumer"
statements:
- name: "s1"
optype: "msg_read"
### JMS CONSUMER durable and shared
jms_consumer_msg_durable: "true"
jms_consumer_msg_shared: "true"
### JMS CONSUMER subscription name
# - only relevant for durable consumer
jms_consumer_subscription: "mysub"
### JMS CONSUMER subscription name
# - only relevant for unshared consumer
jms_consumer_nolocal: "false"
### JMS CONSUMER message read timeout
# - unit: milliseconds
# - 0 means call blocks indefinitely
# - FIXME: 0 supposes to wait indefinitly; but
# it actually behaves like no wait at all
jms_consumer_msg_read_timeout: "10000"
### JMS CONSUMER message selector
# - empty string means no message selector
# - https://docs.oracle.com/cd/E19798-01/821-1841/bncer/index.html
jms_consumer_msg_read_selector: ""

6
nb/pom.xml
View File

@@ -67,6 +67,12 @@
<version>4.17.22-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-pulsar</artifactId>
<version>4.17.22-SNAPSHOT</version>
</dependency>
<!-- Everything below this line constitutes the delta between nb and nb5 -->
<!-- All driver-* modules should be migrated to adapter-* modules and added to nb5 -->

6
nb5/pom.xml
View File

@@ -88,6 +88,12 @@
<version>4.17.31-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-pulsar</artifactId>
<version>4.17.31-SNAPSHOT</version>
</dependency>
</dependencies>
<build>